2024-08-29T14:43:27.326 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-08-29T14:43:27.353 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-29T14:43:27.440 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940 branch: wip-yuri4-testing-2024-08-28-1359 description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: null first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7879940' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-yuri4-testing-2024-08-28-1359 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-yuri4-testing-2024-08-28-1359 sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 owner: scheduled_yuriw@teuthology priority: 99 repo: https://github.com/ceph/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 8916 sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 111/120000 suite: upgrade suite_branch: wip-yuri4-testing-2024-08-28-1359 suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_1f4056d968930e12926c25a8f13ee8ed6047c791/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 targets: smithi027.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAmAK1zZpI+rFUsTXA8NSFB5vwAu2aZxN1kpLDP8yyeWoctd3FmOf7ZDDC9ZOB53QXPxpHCqvbHX65KfcRurze0= smithi171.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE/AXPmKh6iMCUTht/C80qC1dfPQ2JNa++2k19h8MPQwXZoIiDUvzJJ56jUyqFEnaGK5g07BuVZty0kH0NN+WCY= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install nvmetcli nvme-cli -y - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-08-29_14:13:22 tube: smithi user: yuriw verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-29T14:43:27.441 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_1f4056d968930e12926c25a8f13ee8ed6047c791/qa; will attempt to use it 2024-08-29T14:43:27.441 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_1f4056d968930e12926c25a8f13ee8ed6047c791/qa/tasks 2024-08-29T14:43:27.442 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-29T14:43:27.443 INFO:teuthology.task.internal:Checking packages... 2024-08-29T14:43:27.464 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '1f4056d968930e12926c25a8f13ee8ed6047c791' 2024-08-29T14:43:27.464 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-29T14:43:27.465 INFO:teuthology.packaging:ref: None 2024-08-29T14:43:27.465 INFO:teuthology.packaging:tag: None 2024-08-29T14:43:27.465 INFO:teuthology.packaging:branch: wip-yuri4-testing-2024-08-28-1359 2024-08-29T14:43:27.465 INFO:teuthology.packaging:sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:43:27.465 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-yuri4-testing-2024-08-28-1359 2024-08-29T14:43:27.611 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4579.g1f4056d9 2024-08-29T14:43:27.612 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-29T14:43:27.618 INFO:teuthology.task.internal:no buildpackages task found 2024-08-29T14:43:27.619 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-29T14:43:27.632 INFO:teuthology.task.internal:Saving configuration 2024-08-29T14:43:27.653 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-29T14:43:27.659 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-29T14:43:27.680 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-29 14:37:21.424575', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAmAK1zZpI+rFUsTXA8NSFB5vwAu2aZxN1kpLDP8yyeWoctd3FmOf7ZDDC9ZOB53QXPxpHCqvbHX65KfcRurze0='} 2024-08-29T14:43:27.698 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi171.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-29 14:37:21.426643', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE/AXPmKh6iMCUTht/C80qC1dfPQ2JNa++2k19h8MPQwXZoIiDUvzJJ56jUyqFEnaGK5g07BuVZty0kH0NN+WCY='} 2024-08-29T14:43:27.699 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-29T14:43:27.704 INFO:teuthology.task.internal:roles: ubuntu@smithi027.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-08-29T14:43:27.704 INFO:teuthology.task.internal:roles: ubuntu@smithi171.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-08-29T14:43:27.705 INFO:teuthology.run_tasks:Running task console_log... 2024-08-29T14:43:27.753 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f369a657be0>, signals=[15]) 2024-08-29T14:43:27.753 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-29T14:43:27.759 INFO:teuthology.task.internal:Opening connections... 2024-08-29T14:43:27.759 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi027.front.sepia.ceph.com 2024-08-29T14:43:27.760 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-29T14:43:27.851 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi171.front.sepia.ceph.com 2024-08-29T14:43:27.852 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-29T14:43:27.925 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-29T14:43:27.931 DEBUG:teuthology.orchestra.run.smithi027:> uname -m 2024-08-29T14:43:27.951 INFO:teuthology.orchestra.run.smithi027.stdout:x86_64 2024-08-29T14:43:27.951 DEBUG:teuthology.orchestra.run.smithi027:> cat /etc/os-release 2024-08-29T14:43:28.010 INFO:teuthology.orchestra.run.smithi027.stdout:NAME="CentOS Stream" 2024-08-29T14:43:28.010 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION="9" 2024-08-29T14:43:28.010 INFO:teuthology.orchestra.run.smithi027.stdout:ID="centos" 2024-08-29T14:43:28.011 INFO:teuthology.orchestra.run.smithi027.stdout:ID_LIKE="rhel fedora" 2024-08-29T14:43:28.011 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION_ID="9" 2024-08-29T14:43:28.011 INFO:teuthology.orchestra.run.smithi027.stdout:PLATFORM_ID="platform:el9" 2024-08-29T14:43:28.011 INFO:teuthology.orchestra.run.smithi027.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-29T14:43:28.011 INFO:teuthology.orchestra.run.smithi027.stdout:ANSI_COLOR="0;31" 2024-08-29T14:43:28.012 INFO:teuthology.orchestra.run.smithi027.stdout:LOGO="fedora-logo-icon" 2024-08-29T14:43:28.012 INFO:teuthology.orchestra.run.smithi027.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-29T14:43:28.012 INFO:teuthology.orchestra.run.smithi027.stdout:HOME_URL="https://centos.org/" 2024-08-29T14:43:28.012 INFO:teuthology.orchestra.run.smithi027.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-08-29T14:43:28.012 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-29T14:43:28.013 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-29T14:43:28.013 INFO:teuthology.lock.ops:Updating smithi027.front.sepia.ceph.com on lock server 2024-08-29T14:43:28.033 DEBUG:teuthology.orchestra.run.smithi171:> uname -m 2024-08-29T14:43:28.052 INFO:teuthology.orchestra.run.smithi171.stdout:x86_64 2024-08-29T14:43:28.052 DEBUG:teuthology.orchestra.run.smithi171:> cat /etc/os-release 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:NAME="CentOS Stream" 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION="9" 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:ID="centos" 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:ID_LIKE="rhel fedora" 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:VERSION_ID="9" 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:PLATFORM_ID="platform:el9" 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-29T14:43:28.110 INFO:teuthology.orchestra.run.smithi171.stdout:ANSI_COLOR="0;31" 2024-08-29T14:43:28.111 INFO:teuthology.orchestra.run.smithi171.stdout:LOGO="fedora-logo-icon" 2024-08-29T14:43:28.111 INFO:teuthology.orchestra.run.smithi171.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-29T14:43:28.111 INFO:teuthology.orchestra.run.smithi171.stdout:HOME_URL="https://centos.org/" 2024-08-29T14:43:28.111 INFO:teuthology.orchestra.run.smithi171.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-08-29T14:43:28.111 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-29T14:43:28.111 INFO:teuthology.orchestra.run.smithi171.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-29T14:43:28.111 INFO:teuthology.lock.ops:Updating smithi171.front.sepia.ceph.com on lock server 2024-08-29T14:43:28.133 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-29T14:43:28.141 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-29T14:43:28.146 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-29T14:43:28.146 DEBUG:teuthology.orchestra.run.smithi027:> test '!' -e /home/ubuntu/cephtest 2024-08-29T14:43:28.148 DEBUG:teuthology.orchestra.run.smithi171:> test '!' -e /home/ubuntu/cephtest 2024-08-29T14:43:28.166 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-29T14:43:28.172 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-29T14:43:28.172 DEBUG:teuthology.orchestra.run.smithi027:> test -z $(ls -A /var/lib/ceph) 2024-08-29T14:43:28.206 DEBUG:teuthology.orchestra.run.smithi171:> test -z $(ls -A /var/lib/ceph) 2024-08-29T14:43:28.312 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-29T14:43:28.364 INFO:teuthology.run_tasks:Running task kernel... 2024-08-29T14:43:28.379 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-08-29T14:43:28.379 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-08-29T14:43:28.379 DEBUG:teuthology.orchestra.run.smithi027:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-29T14:43:28.380 DEBUG:teuthology.orchestra.run.smithi171:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-29T14:43:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T14:43:28.397 DEBUG:teuthology.orchestra.run.smithi171:> uname -r 2024-08-29T14:43:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T14:43:28.398 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-08-29T14:43:28.455 INFO:teuthology.orchestra.run.smithi171.stdout:5.14.0-503.el9.x86_64 2024-08-29T14:43:28.456 INFO:teuthology.task.kernel:Running kernel on smithi171: 5.14.0-503.el9.x86_64 2024-08-29T14:43:28.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum install -y kernel 2024-08-29T14:43:28.457 INFO:teuthology.orchestra.run.smithi027.stdout:5.14.0-503.el9.x86_64 2024-08-29T14:43:28.457 INFO:teuthology.task.kernel:Running kernel on smithi027: 5.14.0-503.el9.x86_64 2024-08-29T14:43:28.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum install -y kernel 2024-08-29T14:43:32.337 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - BaseOS 3.9 MB/s | 8.2 MB 00:02 2024-08-29T14:43:33.052 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - BaseOS 2.9 MB/s | 8.2 MB 00:02 2024-08-29T14:43:35.511 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - AppStream 16 MB/s | 20 MB 00:01 2024-08-29T14:43:35.540 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - AppStream 9.8 MB/s | 20 MB 00:02 2024-08-29T14:43:40.428 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - CRB 7.3 MB/s | 6.6 MB 00:00 2024-08-29T14:43:41.280 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - CRB 3.5 MB/s | 6.6 MB 00:01 2024-08-29T14:43:42.204 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - Extras packages 38 kB/s | 18 kB 00:00 2024-08-29T14:43:42.975 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 35 MB/s | 22 MB 00:00 2024-08-29T14:43:43.539 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - Extras packages 18 kB/s | 18 kB 00:01 2024-08-29T14:43:44.586 INFO:teuthology.orchestra.run.smithi171.stdout:Extra Packages for Enterprise Linux 25 MB/s | 22 MB 00:00 2024-08-29T14:43:48.672 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-29T14:43:50.008 INFO:teuthology.orchestra.run.smithi171.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-29T14:43:50.779 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-29T14:43:50.779 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-08-29T14:43:50.855 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:43:50.856 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:43:50.856 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:43:50.967 DEBUG:teuthology.orchestra.run.smithi027:> echo no | sudo yum reinstall kernel || true 2024-08-29T14:43:51.537 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:03 ago on Thu 29 Aug 2024 02:43:48 PM UTC. 2024-08-29T14:43:51.692 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout:Reinstalling: 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 1.9 M 2024-08-29T14:43:51.700 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 0 2024-08-29T14:43:51.701 INFO:teuthology.orchestra.run.smithi027.stderr:Operation aborted. 2024-08-29T14:43:51.759 INFO:teuthology.orchestra.run.smithi027.stdout:Is this ok [y/N]: 2024-08-29T14:43:51.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum reinstall -y kernel || true 2024-08-29T14:43:52.130 INFO:teuthology.orchestra.run.smithi171.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-29T14:43:52.131 INFO:teuthology.orchestra.run.smithi171.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-08-29T14:43:52.215 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:43:52.216 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:43:52.216 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:43:52.325 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:04 ago on Thu 29 Aug 2024 02:43:48 PM UTC. 2024-08-29T14:43:52.330 DEBUG:teuthology.orchestra.run.smithi171:> echo no | sudo yum reinstall kernel || true 2024-08-29T14:43:52.457 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:43:52.458 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:43:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout:Reinstalling: 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 1.9 M 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 0 2024-08-29T14:43:52.459 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:43:52.877 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:03 ago on Thu 29 Aug 2024 02:43:49 PM UTC. 2024-08-29T14:43:53.022 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout:Reinstalling: 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:43:53.030 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:43:53.031 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:43:53.031 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 1.9 M 2024-08-29T14:43:53.031 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 0 2024-08-29T14:43:53.031 INFO:teuthology.orchestra.run.smithi171.stderr:Operation aborted. 2024-08-29T14:43:53.086 INFO:teuthology.orchestra.run.smithi171.stdout:Is this ok [y/N]: 2024-08-29T14:43:53.086 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum reinstall -y kernel || true 2024-08-29T14:43:53.200 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-503.el9.x86_64.rpm 3.9 MB/s | 1.9 MB 00:00 2024-08-29T14:43:53.200 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:43:53.200 INFO:teuthology.orchestra.run.smithi027.stdout:Total 2.5 MB/s | 1.9 MB 00:00 2024-08-29T14:43:53.230 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:43:53.471 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:43:53.471 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:43:53.503 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:43:53.504 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:43:53.628 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:04 ago on Thu 29 Aug 2024 02:43:49 PM UTC. 2024-08-29T14:43:53.687 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:43:53.756 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:43:53.757 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:43:53.757 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-08-29T14:43:53.757 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:43:53.757 INFO:teuthology.orchestra.run.smithi171.stdout:Reinstalling: 2024-08-29T14:43:53.757 INFO:teuthology.orchestra.run.smithi171.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-29T14:43:53.758 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:43:53.758 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:43:53.758 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:43:53.758 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:43:53.758 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 1.9 M 2024-08-29T14:43:53.758 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 0 2024-08-29T14:43:53.758 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:43:53.793 INFO:teuthology.orchestra.run.smithi027.stdout: Reinstalling : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-29T14:43:53.794 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-29T14:43:54.551 INFO:teuthology.orchestra.run.smithi171.stdout:kernel-5.14.0-503.el9.x86_64.rpm 3.5 MB/s | 1.9 MB 00:00 2024-08-29T14:43:54.551 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:43:54.551 INFO:teuthology.orchestra.run.smithi171.stdout:Total 2.4 MB/s | 1.9 MB 00:00 2024-08-29T14:43:54.581 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:43:54.583 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-5.14.0-503.el9.x86_64 2/2 2024-08-29T14:43:54.583 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-29T14:43:54.807 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:43:54.808 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:43:54.838 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:43:54.839 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:43:55.009 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:43:55.097 INFO:teuthology.orchestra.run.smithi171.stdout: Reinstalling : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-29T14:43:55.097 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-29T14:43:55.296 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-29T14:43:55.297 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:43:55.297 INFO:teuthology.orchestra.run.smithi027.stdout:Reinstalled: 2024-08-29T14:43:55.297 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-5.14.0-503.el9.x86_64 2024-08-29T14:43:55.297 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:43:55.297 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:43:55.499 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-08-29T14:43:55.560 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-503.el9.x86_64 2024-08-29T14:43:55.560 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-503.el9.x86_64 2024-08-29T14:43:55.560 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-08-29T14:43:55.561 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-08-29T14:43:55.561 DEBUG:teuthology.orchestra.run.smithi027:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-29T14:43:55.595 INFO:teuthology.orchestra.run.smithi027.stdout:ttyS1 2024-08-29T14:43:55.615 DEBUG:teuthology.parallel:result is None 2024-08-29T14:43:55.888 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: kernel-5.14.0-503.el9.x86_64 2/2 2024-08-29T14:43:55.888 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-29T14:43:56.616 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-29T14:43:56.616 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:43:56.616 INFO:teuthology.orchestra.run.smithi171.stdout:Reinstalled: 2024-08-29T14:43:56.616 INFO:teuthology.orchestra.run.smithi171.stdout: kernel-5.14.0-503.el9.x86_64 2024-08-29T14:43:56.616 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:43:56.616 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:43:56.834 DEBUG:teuthology.orchestra.run.smithi171:> rpm -q kernel | sort -rV | head -n 1 2024-08-29T14:43:56.893 INFO:teuthology.orchestra.run.smithi171.stdout:kernel-5.14.0-503.el9.x86_64 2024-08-29T14:43:56.893 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-503.el9.x86_64 2024-08-29T14:43:56.893 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-08-29T14:43:56.894 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-08-29T14:43:56.894 DEBUG:teuthology.orchestra.run.smithi171:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-29T14:43:56.923 INFO:teuthology.orchestra.run.smithi171.stdout:ttyS1 2024-08-29T14:43:56.950 DEBUG:teuthology.parallel:result is None 2024-08-29T14:43:56.951 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-29T14:43:56.959 INFO:teuthology.task.internal:Creating test directory... 2024-08-29T14:43:56.960 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-29T14:43:56.962 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-29T14:43:57.006 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-29T14:43:57.012 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-29T14:43:57.046 INFO:teuthology.task.internal:Creating archive directory... 2024-08-29T14:43:57.046 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-29T14:43:57.049 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-29T14:43:57.077 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-29T14:43:57.084 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-29T14:43:57.084 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-29T14:43:57.115 DEBUG:teuthology.orchestra.run.smithi171:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-29T14:43:57.155 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-29T14:43:57.165 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-29T14:43:57.179 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-29T14:43:57.191 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-29T14:43:57.193 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-29T14:43:57.200 INFO:teuthology.task.internal:Configuring sudo... 2024-08-29T14:43:57.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-29T14:43:57.203 DEBUG:teuthology.orchestra.run.smithi171:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-29T14:43:57.231 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-29T14:43:57.240 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-29T14:43:57.241 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-29T14:43:57.272 DEBUG:teuthology.orchestra.run.smithi171:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-29T14:43:57.287 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-29T14:43:57.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-29T14:43:57.422 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-29T14:43:57.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-29T14:43:57.548 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:43:57.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-29T14:43:57.615 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-29T14:43:57.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-29T14:43:57.709 DEBUG:teuthology.orchestra.run.smithi171:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-29T14:43:57.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-29T14:43:57.834 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:43:57.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-29T14:43:57.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo service rsyslog restart 2024-08-29T14:43:57.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo service rsyslog restart 2024-08-29T14:43:57.941 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-29T14:43:57.984 INFO:teuthology.orchestra.run.smithi171.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-29T14:43:58.359 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-29T14:43:58.367 INFO:teuthology.task.internal:Starting timer... 2024-08-29T14:43:58.367 INFO:teuthology.run_tasks:Running task pcp... 2024-08-29T14:43:58.378 INFO:teuthology.run_tasks:Running task selinux... 2024-08-29T14:43:58.387 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-08-29T14:43:58.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo service auditd rotate 2024-08-29T14:43:58.436 INFO:teuthology.orchestra.run.smithi027.stdout:Rotating logs: 2024-08-29T14:43:58.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo service auditd rotate 2024-08-29T14:43:58.488 INFO:teuthology.orchestra.run.smithi171.stdout:Rotating logs: 2024-08-29T14:43:58.490 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-29T14:43:58.490 DEBUG:teuthology.orchestra.run.smithi027:> /usr/sbin/getenforce 2024-08-29T14:43:58.514 INFO:teuthology.orchestra.run.smithi027.stdout:Permissive 2024-08-29T14:43:58.514 DEBUG:teuthology.orchestra.run.smithi171:> /usr/sbin/getenforce 2024-08-29T14:43:58.541 INFO:teuthology.orchestra.run.smithi171.stdout:Permissive 2024-08-29T14:43:58.542 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi027.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi171.front.sepia.ceph.com': 'permissive'} 2024-08-29T14:43:58.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-29T14:43:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T14:43:58.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-29T14:43:58.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T14:43:58.609 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-29T14:43:58.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo /usr/sbin/setenforce permissive 2024-08-29T14:43:58.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo /usr/sbin/setenforce permissive 2024-08-29T14:43:58.690 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-29T14:43:58.699 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-08-29T14:43:58.702 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-08-29T14:43:58.816 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-29T14:43:58.832 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-29T14:43:58.833 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi027.front.sepia.ceph.com,smithi171.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-29T14:45:44.310 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi027.front.sepia.ceph.com'), Remote(name='ubuntu@smithi171.front.sepia.ceph.com')] 2024-08-29T14:45:44.312 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-08-29T14:45:44.313 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-29T14:45:44.400 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-08-29T14:45:44.483 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-08-29T14:45:44.483 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi171.front.sepia.ceph.com' 2024-08-29T14:45:44.484 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi171.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-29T14:45:44.560 DEBUG:teuthology.orchestra.run.smithi171:> true 2024-08-29T14:45:44.656 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi171.front.sepia.ceph.com' 2024-08-29T14:45:44.656 INFO:teuthology.run_tasks:Running task clock... 2024-08-29T14:45:45.462 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-29T14:45:45.462 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-29T14:45:45.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-29T14:45:45.467 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-29T14:45:45.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-29T14:45:45.500 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-29T14:45:45.503 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-29T14:45:45.517 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-29T14:45:45.519 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-29T14:45:45.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: ntpd: command not found 2024-08-29T14:45:45.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: ntpd: command not found 2024-08-29T14:45:45.557 INFO:teuthology.orchestra.run.smithi027.stdout:506 Cannot talk to daemon 2024-08-29T14:45:45.557 INFO:teuthology.orchestra.run.smithi171.stdout:506 Cannot talk to daemon 2024-08-29T14:45:45.572 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-29T14:45:45.572 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-29T14:45:45.587 INFO:teuthology.orchestra.run.smithi171.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-29T14:45:45.587 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-29T14:45:45.647 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-08-29T14:45:45.648 INFO:teuthology.orchestra.run.smithi171.stderr:bash: line 1: ntpq: command not found 2024-08-29T14:45:45.650 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-29T14:45:45.651 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-08-29T14:45:45.651 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.651 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.651 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.651 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.654 INFO:teuthology.orchestra.run.smithi171.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-29T14:45:45.654 INFO:teuthology.orchestra.run.smithi171.stdout:=============================================================================== 2024-08-29T14:45:45.654 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.654 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.654 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.654 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T14:45:45.654 INFO:teuthology.run_tasks:Running task pexec... 2024-08-29T14:45:45.754 INFO:teuthology.task.pexec:Executing custom commands... 2024-08-29T14:45:45.754 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi027.front.sepia.ceph.com 2024-08-29T14:45:45.754 DEBUG:teuthology.orchestra.run.smithi027:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-29T14:45:45.755 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi171.front.sepia.ceph.com 2024-08-29T14:45:45.755 DEBUG:teuthology.orchestra.run.smithi171:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-29T14:45:46.099 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:Removing: 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:Removing unused dependencies: 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:Remove 2 Packages 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:46.100 INFO:teuthology.orchestra.run.smithi171.stdout:Freed space: 5.5 M 2024-08-29T14:45:46.101 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:45:46.104 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:45:46.104 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:45:46.114 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout:Removing: 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout:Removing unused dependencies: 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout:Remove 2 Packages 2024-08-29T14:45:46.115 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:46.116 INFO:teuthology.orchestra.run.smithi027.stdout:Freed space: 5.5 M 2024-08-29T14:45:46.116 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:45:46.119 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:45:46.120 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:45:46.153 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:45:46.153 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:45:46.159 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:45:46.159 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:45:46.280 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:45:46.295 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:45:46.365 INFO:teuthology.orchestra.run.smithi171.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-08-29T14:45:46.366 INFO:teuthology.orchestra.run.smithi027.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-08-29T14:45:46.422 INFO:teuthology.orchestra.run.smithi171.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-08-29T14:45:46.438 INFO:teuthology.orchestra.run.smithi027.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-08-29T14:45:46.803 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-08-29T14:45:46.803 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-08-29T14:45:46.827 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-08-29T14:45:46.827 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-08-29T14:45:47.072 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-08-29T14:45:47.073 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:47.073 INFO:teuthology.orchestra.run.smithi171.stdout:Removed: 2024-08-29T14:45:47.073 INFO:teuthology.orchestra.run.smithi171.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-08-29T14:45:47.073 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:47.073 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:45:47.092 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-08-29T14:45:47.092 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:47.092 INFO:teuthology.orchestra.run.smithi027.stdout:Removed: 2024-08-29T14:45:47.092 INFO:teuthology.orchestra.run.smithi027.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-08-29T14:45:47.092 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:47.092 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:45:47.861 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:45 ago on Thu 29 Aug 2024 02:45:02 PM UTC. 2024-08-29T14:45:47.948 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:45 ago on Thu 29 Aug 2024 02:45:02 PM UTC. 2024-08-29T14:45:48.001 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:45:48.001 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:45:48.001 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:45:48.002 INFO:teuthology.orchestra.run.smithi171.stdout:Install 6 Packages 2024-08-29T14:45:48.003 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:48.003 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 2.0 M 2024-08-29T14:45:48.003 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 9.2 M 2024-08-29T14:45:48.003 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:45:48.112 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:45:48.113 INFO:teuthology.orchestra.run.smithi027.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout:Install 6 Packages 2024-08-29T14:45:48.114 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:48.115 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 2.0 M 2024-08-29T14:45:48.115 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 9.2 M 2024-08-29T14:45:48.115 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:45:48.252 INFO:teuthology.orchestra.run.smithi171.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 297 kB/s | 44 kB 00:00 2024-08-29T14:45:48.286 INFO:teuthology.orchestra.run.smithi171.stdout:(2/6): libnvme-1.9-3.el9.x86_64.rpm 579 kB/s | 106 kB 00:00 2024-08-29T14:45:48.306 INFO:teuthology.orchestra.run.smithi027.stdout:(1/6): nvmetcli-0.7-3.el9.noarch.rpm 486 kB/s | 44 kB 00:00 2024-08-29T14:45:48.331 INFO:teuthology.orchestra.run.smithi027.stdout:(2/6): libnvme-1.9-3.el9.x86_64.rpm 907 kB/s | 106 kB 00:00 2024-08-29T14:45:48.344 INFO:teuthology.orchestra.run.smithi171.stdout:(3/6): python3-kmod-0.9-32.el9.x86_64.rpm 1.4 MB/s | 84 kB 00:00 2024-08-29T14:45:48.356 INFO:teuthology.orchestra.run.smithi027.stdout:(3/6): python3-configshell-1.1.30-1.el9.noarch. 1.4 MB/s | 72 kB 00:00 2024-08-29T14:45:48.378 INFO:teuthology.orchestra.run.smithi171.stdout:(4/6): python3-configshell-1.1.30-1.el9.noarch. 570 kB/s | 72 kB 00:00 2024-08-29T14:45:48.381 INFO:teuthology.orchestra.run.smithi027.stdout:(4/6): python3-kmod-0.9-32.el9.x86_64.rpm 1.6 MB/s | 84 kB 00:00 2024-08-29T14:45:48.423 INFO:teuthology.orchestra.run.smithi027.stdout:(5/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 4.2 MB/s | 906 kB 00:00 2024-08-29T14:45:48.437 INFO:teuthology.orchestra.run.smithi171.stdout:(5/6): nvme-cli-2.9.1-6.el9.x86_64.rpm 2.6 MB/s | 906 kB 00:00 2024-08-29T14:45:48.508 INFO:teuthology.orchestra.run.smithi027.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 5.4 MB/s | 837 kB 00:00 2024-08-29T14:45:48.509 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:45:48.509 INFO:teuthology.orchestra.run.smithi027.stdout:Total 5.1 MB/s | 2.0 MB 00:00 2024-08-29T14:45:48.555 INFO:teuthology.orchestra.run.smithi171.stdout:(6/6): python3-urwid-2.1.2-4.el9.x86_64.rpm 3.9 MB/s | 837 kB 00:00 2024-08-29T14:45:48.555 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:45:48.556 INFO:teuthology.orchestra.run.smithi171.stdout:Total 3.6 MB/s | 2.0 MB 00:00 2024-08-29T14:45:48.584 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:45:48.599 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:45:48.599 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:45:48.630 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:45:48.644 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:45:48.644 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:45:48.709 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:45:48.709 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:45:48.746 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:45:48.746 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:45:49.288 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:45:49.312 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:45:49.410 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-08-29T14:45:49.440 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/6 2024-08-29T14:45:49.452 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-08-29T14:45:49.491 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/6 2024-08-29T14:45:49.507 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-08-29T14:45:49.545 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/6 2024-08-29T14:45:49.747 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-08-29T14:45:49.783 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-08-29T14:45:49.789 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/6 2024-08-29T14:45:49.847 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-08-29T14:45:50.314 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-08-29T14:45:50.321 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/6 2024-08-29T14:45:50.359 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-08-29T14:45:50.363 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/6 2024-08-29T14:45:51.437 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-08-29T14:45:51.437 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-08-29T14:45:51.437 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-08-29T14:45:51.437 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-08-29T14:45:51.437 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-08-29T14:45:51.437 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-08-29T14:45:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/6 2024-08-29T14:45:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/6 2024-08-29T14:45:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/6 2024-08-29T14:45:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/6 2024-08-29T14:45:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/6 2024-08-29T14:45:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/6 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:45:51.754 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:45:51.794 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/6 2024-08-29T14:45:51.794 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:51.794 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:45:51.795 INFO:teuthology.orchestra.run.smithi171.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-08-29T14:45:51.795 INFO:teuthology.orchestra.run.smithi171.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-29T14:45:51.795 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-29T14:45:51.795 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:45:51.795 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:45:51.986 DEBUG:teuthology.parallel:result is None 2024-08-29T14:45:52.115 DEBUG:teuthology.parallel:result is None 2024-08-29T14:45:52.116 INFO:teuthology.run_tasks:Running task install... 2024-08-29T14:45:52.124 DEBUG:teuthology.task.install:project ceph 2024-08-29T14:45:52.124 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791'}} 2024-08-29T14:45:52.124 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791'} 2024-08-29T14:45:52.125 INFO:teuthology.task.install:Using flavor: default 2024-08-29T14:45:52.130 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-08-29T14:45:52.130 INFO:teuthology.task.install:extra packages: [] 2024-08-29T14:45:52.131 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791', 'tag': None, 'wait_for_package': False} 2024-08-29T14:45:52.131 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-29T14:45:52.131 INFO:teuthology.packaging:ref: None 2024-08-29T14:45:52.131 INFO:teuthology.packaging:tag: None 2024-08-29T14:45:52.131 INFO:teuthology.packaging:branch: reef 2024-08-29T14:45:52.131 INFO:teuthology.packaging:sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:45:52.131 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-29T14:45:52.134 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791', 'tag': None, 'wait_for_package': False} 2024-08-29T14:45:52.135 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-29T14:45:52.135 INFO:teuthology.packaging:ref: None 2024-08-29T14:45:52.135 INFO:teuthology.packaging:tag: None 2024-08-29T14:45:52.135 INFO:teuthology.packaging:branch: reef 2024-08-29T14:45:52.135 INFO:teuthology.packaging:sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:45:52.135 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-29T14:45:52.376 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/ 2024-08-29T14:45:52.376 INFO:teuthology.task.install.rpm:Package version is 18.2.4-774.gfe43f1c9 2024-08-29T14:45:52.401 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/ 2024-08-29T14:45:52.402 INFO:teuthology.task.install.rpm:Package version is 18.2.4-774.gfe43f1c9 2024-08-29T14:45:52.491 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-29T14:45:52.491 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:45:52.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-29T14:45:52.510 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-29T14:45:52.510 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:45:52.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-29T14:45:52.522 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-29T14:45:52.522 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-29T14:45:52.522 INFO:teuthology.packaging:ref: None 2024-08-29T14:45:52.522 INFO:teuthology.packaging:tag: None 2024-08-29T14:45:52.522 INFO:teuthology.packaging:branch: reef 2024-08-29T14:45:52.522 INFO:teuthology.packaging:sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:45:52.522 DEBUG:teuthology.orchestra.run.smithi171:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-29T14:45:52.541 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-29T14:45:52.541 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-29T14:45:52.541 INFO:teuthology.packaging:ref: None 2024-08-29T14:45:52.542 INFO:teuthology.packaging:tag: None 2024-08-29T14:45:52.542 INFO:teuthology.packaging:branch: reef 2024-08-29T14:45:52.542 INFO:teuthology.packaging:sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:45:52.542 DEBUG:teuthology.orchestra.run.smithi027:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-29T14:45:52.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-29T14:45:52.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-29T14:45:52.684 DEBUG:teuthology.orchestra.run.smithi171:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-29T14:45:52.689 DEBUG:teuthology.orchestra.run.smithi027:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-29T14:45:52.709 INFO:teuthology.orchestra.run.smithi171.stdout:check_obsoletes = 1 2024-08-29T14:45:52.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum clean all 2024-08-29T14:45:52.756 INFO:teuthology.orchestra.run.smithi027.stdout:check_obsoletes = 1 2024-08-29T14:45:52.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum clean all 2024-08-29T14:45:52.937 INFO:teuthology.orchestra.run.smithi171.stdout:45 files removed 2024-08-29T14:45:52.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-radosgw 2024-08-29T14:45:52.994 INFO:teuthology.orchestra.run.smithi027.stdout:45 files removed 2024-08-29T14:45:53.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-radosgw 2024-08-29T14:45:53.516 INFO:teuthology.orchestra.run.smithi171.stdout:ceph packages for x86_64 279 kB/s | 77 kB 00:00 2024-08-29T14:45:53.591 INFO:teuthology.orchestra.run.smithi027.stdout:ceph packages for x86_64 277 kB/s | 77 kB 00:00 2024-08-29T14:45:53.800 INFO:teuthology.orchestra.run.smithi171.stdout:ceph noarch packages 48 kB/s | 11 kB 00:00 2024-08-29T14:45:53.874 INFO:teuthology.orchestra.run.smithi027.stdout:ceph noarch packages 48 kB/s | 11 kB 00:00 2024-08-29T14:45:54.076 INFO:teuthology.orchestra.run.smithi171.stdout:ceph source packages 7.8 kB/s | 1.9 kB 00:00 2024-08-29T14:45:54.150 INFO:teuthology.orchestra.run.smithi027.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-08-29T14:45:54.833 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.2 MB 00:00 2024-08-29T14:45:56.186 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - BaseOS 4.0 MB/s | 8.2 MB 00:02 2024-08-29T14:45:56.626 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - AppStream 32 MB/s | 20 MB 00:00 2024-08-29T14:46:02.452 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - CRB 3.5 MB/s | 6.6 MB 00:01 2024-08-29T14:46:04.244 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 18 kB 00:00 2024-08-29T14:46:04.445 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - AppStream 2.8 MB/s | 20 MB 00:07 2024-08-29T14:46:05.299 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 24 MB/s | 22 MB 00:00 2024-08-29T14:46:08.738 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.6 MB 00:00 2024-08-29T14:46:11.013 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 37 kB/s | 1.7 kB 00:00 2024-08-29T14:46:11.064 INFO:teuthology.orchestra.run.smithi171.stdout:CentOS Stream 9 - Extras packages 17 kB/s | 18 kB 00:01 2024-08-29T14:46:12.086 INFO:teuthology.orchestra.run.smithi171.stdout:Extra Packages for Enterprise Linux 25 MB/s | 22 MB 00:00 2024-08-29T14:46:13.426 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-radosgw x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 7.7 M 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout:Upgrading: 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout: librados2 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 3.3 M 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout: librbd1 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 3.0 M 2024-08-29T14:46:13.428 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-base x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 5.1 M 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-common x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 18 M 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-selinux x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 25 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs2 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 711 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-29T14:46:13.429 INFO:teuthology.orchestra.run.smithi027.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: libradosstriper1 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 477 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: librgw2 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 4.5 M 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-argparse x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 45 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-common x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 129 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cephfs x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 162 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rados x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 321 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rbd x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 300 k 2024-08-29T14:46:13.430 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rgw x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 99 k 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout:Install 28 Packages 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout:Upgrade 2 Packages 2024-08-29T14:46:13.431 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:13.432 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 53 M 2024-08-29T14:46:13.432 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:46:14.006 INFO:teuthology.orchestra.run.smithi027.stdout:(1/30): ceph-base-18.2.4-774.gfe43f1c9.el9.x86_ 10 MB/s | 5.1 MB 00:00 2024-08-29T14:46:14.056 INFO:teuthology.orchestra.run.smithi027.stdout:(2/30): ceph-selinux-18.2.4-774.gfe43f1c9.el9.x 505 kB/s | 25 kB 00:00 2024-08-29T14:46:14.140 INFO:teuthology.orchestra.run.smithi027.stdout:(3/30): libcephfs2-18.2.4-774.gfe43f1c9.el9.x86 8.3 MB/s | 711 kB 00:00 2024-08-29T14:46:14.215 INFO:teuthology.orchestra.run.smithi027.stdout:(4/30): libradosstriper1-18.2.4-774.gfe43f1c9.e 6.2 MB/s | 477 kB 00:00 2024-08-29T14:46:14.298 INFO:teuthology.orchestra.run.smithi027.stdout:(5/30): ceph-radosgw-18.2.4-774.gfe43f1c9.el9.x 9.9 MB/s | 7.7 MB 00:00 2024-08-29T14:46:14.351 INFO:teuthology.orchestra.run.smithi027.stdout:(6/30): python3-ceph-argparse-18.2.4-774.gfe43f 864 kB/s | 45 kB 00:00 2024-08-29T14:46:14.418 INFO:teuthology.orchestra.run.smithi027.stdout:(7/30): librgw2-18.2.4-774.gfe43f1c9.el9.x86_64 22 MB/s | 4.5 MB 00:00 2024-08-29T14:46:14.454 INFO:teuthology.orchestra.run.smithi027.stdout:(8/30): python3-ceph-common-18.2.4-774.gfe43f1c 1.4 MB/s | 129 kB 00:00 2024-08-29T14:46:14.647 INFO:teuthology.orchestra.run.smithi027.stdout:(9/30): ceph-common-18.2.4-774.gfe43f1c9.el9.x8 16 MB/s | 18 MB 00:01 2024-08-29T14:46:14.677 INFO:teuthology.orchestra.run.smithi027.stdout:(10/30): python3-cephfs-18.2.4-774.gfe43f1c9.el 627 kB/s | 162 kB 00:00 2024-08-29T14:46:14.705 INFO:teuthology.orchestra.run.smithi027.stdout:(11/30): python3-rados-18.2.4-774.gfe43f1c9.el9 1.2 MB/s | 321 kB 00:00 2024-08-29T14:46:14.728 INFO:teuthology.orchestra.run.smithi027.stdout:(12/30): python3-rbd-18.2.4-774.gfe43f1c9.el9.x 3.6 MB/s | 300 kB 00:00 2024-08-29T14:46:14.745 INFO:teuthology.orchestra.run.smithi027.stdout:(13/30): python3-rgw-18.2.4-774.gfe43f1c9.el9.x 1.4 MB/s | 99 kB 00:00 2024-08-29T14:46:14.853 INFO:teuthology.orchestra.run.smithi027.stdout:(14/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 417 kB/s | 45 kB 00:00 2024-08-29T14:46:14.878 INFO:teuthology.orchestra.run.smithi027.stdout:(15/30): boost-program-options-1.75.0-8.el9.x86 608 kB/s | 107 kB 00:00 2024-08-29T14:46:14.904 INFO:teuthology.orchestra.run.smithi027.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 912 kB/s | 160 kB 00:00 2024-08-29T14:46:14.954 INFO:teuthology.orchestra.run.smithi027.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 849 kB/s | 42 kB 00:00 2024-08-29T14:46:14.979 INFO:teuthology.orchestra.run.smithi027.stdout:(18/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.8 MB/s | 292 kB 00:00 2024-08-29T14:46:15.021 INFO:teuthology.orchestra.run.smithi027.stdout:(19/30): librdkafka-1.6.1-102.el9.x86_64.rpm 3.9 MB/s | 662 kB 00:00 2024-08-29T14:46:15.054 INFO:teuthology.orchestra.run.smithi027.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 1.1 MB/s | 111 kB 00:00 2024-08-29T14:46:15.080 INFO:teuthology.orchestra.run.smithi027.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.1 MB/s | 26 kB 00:00 2024-08-29T14:46:15.105 INFO:teuthology.orchestra.run.smithi027.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-08-29T14:46:15.130 INFO:teuthology.orchestra.run.smithi027.stdout:(23/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 2.0 MB/s | 309 kB 00:00 2024-08-29T14:46:15.156 INFO:teuthology.orchestra.run.smithi027.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-08-29T14:46:15.189 INFO:teuthology.orchestra.run.smithi027.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 5.7 MB/s | 191 kB 00:00 2024-08-29T14:46:15.223 INFO:teuthology.orchestra.run.smithi027.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.9 MB/s | 839 kB 00:00 2024-08-29T14:46:15.290 INFO:teuthology.orchestra.run.smithi027.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-08-29T14:46:15.377 INFO:teuthology.orchestra.run.smithi027.stdout:(28/30): libarrow-9.0.0-11.el9.x86_64.rpm 13 MB/s | 4.4 MB 00:00 2024-08-29T14:46:15.448 INFO:teuthology.orchestra.run.smithi027.stdout:(29/30): librados2-18.2.4-774.gfe43f1c9.el9.x86 15 MB/s | 3.3 MB 00:00 2024-08-29T14:46:15.515 INFO:teuthology.orchestra.run.smithi027.stdout:(30/30): librbd1-18.2.4-774.gfe43f1c9.el9.x86_6 13 MB/s | 3.0 MB 00:00 2024-08-29T14:46:15.521 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:46:15.521 INFO:teuthology.orchestra.run.smithi027.stdout:Total 25 MB/s | 53 MB 00:02 2024-08-29T14:46:15.595 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:46:15.634 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:46:15.634 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:46:16.082 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:46:16.083 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:46:17.533 INFO:teuthology.orchestra.run.smithi171.stdout:lab-extras 24 kB/s | 1.7 kB 00:00 2024-08-29T14:46:17.566 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:46:17.658 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-08-29T14:46:17.687 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-08-29T14:46:17.737 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-08-29T14:46:18.066 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-08-29T14:46:18.178 INFO:teuthology.orchestra.run.smithi027.stdout: Upgrading : librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/32 2024-08-29T14:46:18.271 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/32 2024-08-29T14:46:18.357 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 6/32 2024-08-29T14:46:18.388 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 7/32 2024-08-29T14:46:18.442 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 7/32 2024-08-29T14:46:18.471 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-08-29T14:46:18.515 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-ceph-argparse-2:18.2.4-774.gfe43f1c9.el9.x 9/32 2024-08-29T14:46:18.582 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 10/32 2024-08-29T14:46:18.607 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 11/32 2024-08-29T14:46:18.655 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 11/32 2024-08-29T14:46:18.763 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-08-29T14:46:18.797 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-08-29T14:46:18.837 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-08-29T14:46:18.871 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-08-29T14:46:18.898 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-08-29T14:46:18.942 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-08-29T14:46:18.975 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-08-29T14:46:19.289 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-08-29T14:46:19.329 INFO:teuthology.orchestra.run.smithi027.stdout: Upgrading : librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 20/32 2024-08-29T14:46:19.420 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 20/32 2024-08-29T14:46:19.451 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 21/32 2024-08-29T14:46:19.486 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-08-29T14:46:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-08-29T14:46:19.945 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-radosgw x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 7.7 M 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout:Upgrading: 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout: librados2 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 3.3 M 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout: librbd1 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 3.0 M 2024-08-29T14:46:19.947 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-base x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 5.1 M 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-common x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 18 M 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-selinux x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 25 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs2 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 711 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-29T14:46:19.948 INFO:teuthology.orchestra.run.smithi171.stdout: libradosstriper1 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 477 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: librgw2 x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 4.5 M 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-argparse x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 45 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-common x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 129 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cephfs x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 162 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rados x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 321 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rbd x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 300 k 2024-08-29T14:46:19.949 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rgw x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 99 k 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout:Install 28 Packages 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout:Upgrade 2 Packages 2024-08-29T14:46:19.950 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:19.951 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 24/32 2024-08-29T14:46:19.951 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 53 M 2024-08-29T14:46:19.951 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:46:20.003 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 24/32 2024-08-29T14:46:20.082 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 25/32 2024-08-29T14:46:20.109 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86 26/32 2024-08-29T14:46:20.702 INFO:teuthology.orchestra.run.smithi171.stdout:(1/30): ceph-base-18.2.4-774.gfe43f1c9.el9.x86_ 9.9 MB/s | 5.1 MB 00:00 2024-08-29T14:46:20.744 INFO:teuthology.orchestra.run.smithi171.stdout:(2/30): ceph-selinux-18.2.4-774.gfe43f1c9.el9.x 603 kB/s | 25 kB 00:00 2024-08-29T14:46:20.838 INFO:teuthology.orchestra.run.smithi171.stdout:(3/30): libcephfs2-18.2.4-774.gfe43f1c9.el9.x86 7.6 MB/s | 711 kB 00:00 2024-08-29T14:46:20.936 INFO:teuthology.orchestra.run.smithi171.stdout:(4/30): ceph-radosgw-18.2.4-774.gfe43f1c9.el9.x 10 MB/s | 7.7 MB 00:00 2024-08-29T14:46:20.961 INFO:teuthology.orchestra.run.smithi171.stdout:(5/30): libradosstriper1-18.2.4-774.gfe43f1c9.e 3.7 MB/s | 477 kB 00:00 2024-08-29T14:46:21.162 INFO:teuthology.orchestra.run.smithi171.stdout:(6/30): ceph-common-18.2.4-774.gfe43f1c9.el9.x8 19 MB/s | 18 MB 00:00 2024-08-29T14:46:21.187 INFO:teuthology.orchestra.run.smithi171.stdout:(7/30): python3-ceph-argparse-18.2.4-774.gfe43f 199 kB/s | 45 kB 00:00 2024-08-29T14:46:21.212 INFO:teuthology.orchestra.run.smithi171.stdout:(8/30): python3-ceph-common-18.2.4-774.gfe43f1c 2.5 MB/s | 129 kB 00:00 2024-08-29T14:46:21.238 INFO:teuthology.orchestra.run.smithi171.stdout:(9/30): python3-cephfs-18.2.4-774.gfe43f1c9.el9 3.1 MB/s | 162 kB 00:00 2024-08-29T14:46:21.263 INFO:teuthology.orchestra.run.smithi171.stdout:(10/30): python3-rados-18.2.4-774.gfe43f1c9.el9 6.2 MB/s | 321 kB 00:00 2024-08-29T14:46:21.297 INFO:teuthology.orchestra.run.smithi171.stdout:(11/30): python3-rbd-18.2.4-774.gfe43f1c9.el9.x 5.0 MB/s | 300 kB 00:00 2024-08-29T14:46:21.322 INFO:teuthology.orchestra.run.smithi171.stdout:(12/30): python3-rgw-18.2.4-774.gfe43f1c9.el9.x 1.6 MB/s | 99 kB 00:00 2024-08-29T14:46:21.398 INFO:teuthology.orchestra.run.smithi171.stdout:(13/30): librgw2-18.2.4-774.gfe43f1c9.el9.x86_6 9.7 MB/s | 4.5 MB 00:00 2024-08-29T14:46:21.507 INFO:teuthology.orchestra.run.smithi171.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 510 kB/s | 107 kB 00:00 2024-08-29T14:46:21.532 INFO:teuthology.orchestra.run.smithi171.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 337 kB/s | 45 kB 00:00 2024-08-29T14:46:21.557 INFO:teuthology.orchestra.run.smithi171.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 683 kB/s | 160 kB 00:00 2024-08-29T14:46:21.599 INFO:teuthology.orchestra.run.smithi171.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-08-29T14:46:21.641 INFO:teuthology.orchestra.run.smithi171.stdout:(18/30): fmt-8.1.1-5.el9.x86_64.rpm 2.6 MB/s | 111 kB 00:00 2024-08-29T14:46:21.667 INFO:teuthology.orchestra.run.smithi171.stdout:(19/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.1 MB/s | 292 kB 00:00 2024-08-29T14:46:21.701 INFO:teuthology.orchestra.run.smithi171.stdout:(20/30): librdkafka-1.6.1-102.el9.x86_64.rpm 3.3 MB/s | 662 kB 00:00 2024-08-29T14:46:21.743 INFO:teuthology.orchestra.run.smithi171.stdout:(21/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 3.0 MB/s | 309 kB 00:00 2024-08-29T14:46:21.768 INFO:teuthology.orchestra.run.smithi171.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 2.1 MB/s | 49 kB 00:00 2024-08-29T14:46:21.793 INFO:teuthology.orchestra.run.smithi171.stdout:(23/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 304 kB/s | 26 kB 00:00 2024-08-29T14:46:21.827 INFO:teuthology.orchestra.run.smithi171.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2024-08-29T14:46:21.852 INFO:teuthology.orchestra.run.smithi171.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 7.5 MB/s | 191 kB 00:00 2024-08-29T14:46:21.903 INFO:teuthology.orchestra.run.smithi171.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 7.5 MB/s | 839 kB 00:00 2024-08-29T14:46:21.953 INFO:teuthology.orchestra.run.smithi171.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-08-29T14:46:22.045 INFO:teuthology.orchestra.run.smithi171.stdout:(28/30): libarrow-9.0.0-11.el9.x86_64.rpm 12 MB/s | 4.4 MB 00:00 2024-08-29T14:46:22.129 INFO:teuthology.orchestra.run.smithi171.stdout:(29/30): librados2-18.2.4-774.gfe43f1c9.el9.x86 15 MB/s | 3.3 MB 00:00 2024-08-29T14:46:22.196 INFO:teuthology.orchestra.run.smithi171.stdout:(30/30): librbd1-18.2.4-774.gfe43f1c9.el9.x86_6 12 MB/s | 3.0 MB 00:00 2024-08-29T14:46:22.200 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:46:22.201 INFO:teuthology.orchestra.run.smithi171.stdout:Total 24 MB/s | 53 MB 00:02 2024-08-29T14:46:22.271 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:46:22.306 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:46:22.306 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:46:22.306 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 27/32 2024-08-29T14:46:22.347 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 27/32 2024-08-29T14:46:22.730 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:46:22.730 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:46:22.897 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 27/32 2024-08-29T14:46:22.934 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 28/32 2024-08-29T14:46:23.017 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 28/32 2024-08-29T14:46:23.017 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-29T14:46:23.017 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-29T14:46:23.018 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:23.052 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/32 2024-08-29T14:46:24.034 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:46:24.114 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-08-29T14:46:24.167 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-08-29T14:46:24.217 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-08-29T14:46:24.528 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-08-29T14:46:24.643 INFO:teuthology.orchestra.run.smithi171.stdout: Upgrading : librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/32 2024-08-29T14:46:24.735 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/32 2024-08-29T14:46:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 6/32 2024-08-29T14:46:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 7/32 2024-08-29T14:46:24.941 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 7/32 2024-08-29T14:46:24.978 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-08-29T14:46:25.021 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ceph-argparse-2:18.2.4-774.gfe43f1c9.el9.x 9/32 2024-08-29T14:46:25.087 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 10/32 2024-08-29T14:46:25.123 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 11/32 2024-08-29T14:46:25.189 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 11/32 2024-08-29T14:46:25.296 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-08-29T14:46:25.330 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-08-29T14:46:25.370 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-08-29T14:46:25.405 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-08-29T14:46:25.432 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-08-29T14:46:25.468 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-08-29T14:46:25.501 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-08-29T14:46:25.799 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-08-29T14:46:25.848 INFO:teuthology.orchestra.run.smithi171.stdout: Upgrading : librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 20/32 2024-08-29T14:46:25.938 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 20/32 2024-08-29T14:46:25.969 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 21/32 2024-08-29T14:46:26.006 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-08-29T14:46:26.409 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-08-29T14:46:26.443 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 24/32 2024-08-29T14:46:26.514 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 24/32 2024-08-29T14:46:26.592 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 25/32 2024-08-29T14:46:26.621 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86 26/32 2024-08-29T14:46:28.836 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 27/32 2024-08-29T14:46:28.878 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 27/32 2024-08-29T14:46:29.397 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 27/32 2024-08-29T14:46:29.441 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 28/32 2024-08-29T14:46:29.522 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 28/32 2024-08-29T14:46:29.522 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-29T14:46:29.523 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-29T14:46:29.523 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:29.551 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/32 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/32 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /sys 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /proc 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /mnt 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /var/tmp 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /home 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /root 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /tmp 2024-08-29T14:46:33.682 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:34.459 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/32 2024-08-29T14:46:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/32 2024-08-29T14:46:34.487 INFO:teuthology.orchestra.run.smithi027.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:46:34.487 INFO:teuthology.orchestra.run.smithi027.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-29T14:46:34.487 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-29T14:46:34.487 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-29T14:46:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-29T14:46:34.579 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-29T14:46:34.579 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-29T14:46:35.674 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-29T14:46:35.674 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/32 2024-08-29T14:46:35.674 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/32 2024-08-29T14:46:35.674 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/32 2024-08-29T14:46:35.674 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/32 2024-08-29T14:46:35.674 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 6/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 7/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-ceph-argparse-2:18.2.4-774.gfe43f1c9.el9.x 8/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86 9/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 10/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 11/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 12/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 13/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-08-29T14:46:35.675 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-08-29T14:46:35.676 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 31/32 2024-08-29T14:46:36.413 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout:Upgraded: 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: fmt-8.1.1-5.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-29T14:46:36.414 INFO:teuthology.orchestra.run.smithi027.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-argparse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.415 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout: re2-1:20211101-20.el9.x86_64 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:36.416 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:46:36.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-test 2024-08-29T14:46:37.374 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:27 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:46:37.514 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-test x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 40 M 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout: libcephsqlite x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 166 k 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:37.515 INFO:teuthology.orchestra.run.smithi027.stdout:Install 4 Packages 2024-08-29T14:46:37.516 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:37.516 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 40 M 2024-08-29T14:46:37.516 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 164 M 2024-08-29T14:46:37.516 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:46:37.777 INFO:teuthology.orchestra.run.smithi027.stdout:(1/4): libcephsqlite-18.2.4-774.gfe43f1c9.el9.x 695 kB/s | 166 kB 00:00 2024-08-29T14:46:38.161 INFO:teuthology.orchestra.run.smithi027.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 166 kB/s | 64 kB 00:00 2024-08-29T14:46:38.211 INFO:teuthology.orchestra.run.smithi027.stdout:(3/4): socat-1.7.4.1-6.el9.x86_64.rpm 452 kB/s | 304 kB 00:00 2024-08-29T14:46:38.820 INFO:teuthology.orchestra.run.smithi027.stdout:(4/4): ceph-test-18.2.4-774.gfe43f1c9.el9.x86_6 31 MB/s | 40 MB 00:01 2024-08-29T14:46:38.822 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:46:38.822 INFO:teuthology.orchestra.run.smithi027.stdout:Total 31 MB/s | 40 MB 00:01 2024-08-29T14:46:38.851 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:46:38.873 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:46:38.873 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:46:39.208 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:46:39.208 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:46:39.699 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/32 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /sys 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /proc 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /mnt 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /var/tmp 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /home 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /root 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout:skipping the directory /tmp 2024-08-29T14:46:39.742 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:39.779 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-29T14:46:39.819 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-29T14:46:39.846 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/4 2024-08-29T14:46:40.506 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/32 2024-08-29T14:46:40.531 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/32 2024-08-29T14:46:40.532 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:46:40.532 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-29T14:46:40.532 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-29T14:46:40.532 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-29T14:46:40.532 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:40.533 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-29T14:46:40.635 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-29T14:46:40.635 INFO:teuthology.orchestra.run.smithi171.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 6/32 2024-08-29T14:46:41.752 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 7/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ceph-argparse-2:18.2.4-774.gfe43f1c9.el9.x 8/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86 9/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 10/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 11/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 12/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 13/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-08-29T14:46:41.753 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-08-29T14:46:41.754 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 31/32 2024-08-29T14:46:42.774 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-08-29T14:46:42.774 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:42.774 INFO:teuthology.orchestra.run.smithi171.stdout:Upgraded: 2024-08-29T14:46:42.774 INFO:teuthology.orchestra.run.smithi171.stdout: librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.774 INFO:teuthology.orchestra.run.smithi171.stdout: librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.774 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-radosgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-selinux-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: fmt-8.1.1-5.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-29T14:46:42.775 INFO:teuthology.orchestra.run.smithi171.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: libradosstriper1-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: librgw2-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-argparse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ceph-common-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.776 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:42.777 INFO:teuthology.orchestra.run.smithi171.stdout: re2-1:20211101-20.el9.x86_64 2024-08-29T14:46:42.777 INFO:teuthology.orchestra.run.smithi171.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-29T14:46:42.777 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:42.777 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:46:43.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-test 2024-08-29T14:46:43.713 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:26 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:46:43.734 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/4 2024-08-29T14:46:43.813 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/4 2024-08-29T14:46:43.847 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:46:43.847 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:43.847 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:46:43.847 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:43.847 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-test x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 40 M 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout: libcephsqlite x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 166 k 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout:Install 4 Packages 2024-08-29T14:46:43.848 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:43.849 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 40 M 2024-08-29T14:46:43.849 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 164 M 2024-08-29T14:46:43.849 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:46:44.101 INFO:teuthology.orchestra.run.smithi171.stdout:(1/4): libcephsqlite-18.2.4-774.gfe43f1c9.el9.x 806 kB/s | 166 kB 00:00 2024-08-29T14:46:44.443 INFO:teuthology.orchestra.run.smithi171.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 186 kB/s | 64 kB 00:00 2024-08-29T14:46:44.560 INFO:teuthology.orchestra.run.smithi171.stdout:(3/4): socat-1.7.4.1-6.el9.x86_64.rpm 457 kB/s | 304 kB 00:00 2024-08-29T14:46:45.185 INFO:teuthology.orchestra.run.smithi171.stdout:(4/4): ceph-test-18.2.4-774.gfe43f1c9.el9.x86_6 31 MB/s | 40 MB 00:01 2024-08-29T14:46:45.188 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:46:45.188 INFO:teuthology.orchestra.run.smithi171.stdout:Total 30 MB/s | 40 MB 00:01 2024-08-29T14:46:45.216 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:46:45.236 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:46:45.236 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:46:45.554 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:46:45.554 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:46:45.727 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/4 2024-08-29T14:46:45.727 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/4 2024-08-29T14:46:45.727 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/4 2024-08-29T14:46:45.727 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-29T14:46:46.024 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:46:46.034 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-29T14:46:46.034 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:46.035 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:46:46.035 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:46.035 INFO:teuthology.orchestra.run.smithi027.stdout: libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:46.035 INFO:teuthology.orchestra.run.smithi027.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-29T14:46:46.035 INFO:teuthology.orchestra.run.smithi027.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-29T14:46:46.035 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:46.035 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:46:46.102 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-29T14:46:46.143 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-29T14:46:46.171 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/4 2024-08-29T14:46:46.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph 2024-08-29T14:46:46.907 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:36 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:46:47.047 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: ceph x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 6.5 k 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mds x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 2.1 M 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 1.5 M 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-modules-core noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 246 k 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mon x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 4.7 M 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-osd x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 17 M 2024-08-29T14:46:47.050 INFO:teuthology.orchestra.run.smithi027.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-29T14:46:47.051 INFO:teuthology.orchestra.run.smithi027.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-29T14:46:47.052 INFO:teuthology.orchestra.run.smithi027.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-29T14:46:47.053 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:47.053 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:46:47.053 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:46:47.053 INFO:teuthology.orchestra.run.smithi027.stdout:Install 33 Packages 2024-08-29T14:46:47.053 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:47.053 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 29 M 2024-08-29T14:46:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 104 M 2024-08-29T14:46:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:46:47.437 INFO:teuthology.orchestra.run.smithi027.stdout:(1/33): ceph-18.2.4-774.gfe43f1c9.el9.x86_64.rp 52 kB/s | 6.5 kB 00:00 2024-08-29T14:46:47.612 INFO:teuthology.orchestra.run.smithi027.stdout:(2/33): ceph-mgr-18.2.4-774.gfe43f1c9.el9.x86_6 4.9 MB/s | 1.5 MB 00:00 2024-08-29T14:46:47.662 INFO:teuthology.orchestra.run.smithi027.stdout:(3/33): ceph-mds-18.2.4-774.gfe43f1c9.el9.x86_6 6.0 MB/s | 2.1 MB 00:00 2024-08-29T14:46:47.762 INFO:teuthology.orchestra.run.smithi027.stdout:(4/33): ceph-mgr-modules-core-18.2.4-774.gfe43f 2.4 MB/s | 246 kB 00:00 2024-08-29T14:46:47.838 INFO:teuthology.orchestra.run.smithi027.stdout:(5/33): ceph-mon-18.2.4-774.gfe43f1c9.el9.x86_6 12 MB/s | 4.7 MB 00:00 2024-08-29T14:46:47.988 INFO:teuthology.orchestra.run.smithi027.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-08-29T14:46:48.197 INFO:teuthology.orchestra.run.smithi027.stdout:(7/33): ceph-osd-18.2.4-774.gfe43f1c9.el9.x86_6 28 MB/s | 17 MB 00:00 2024-08-29T14:46:48.222 INFO:teuthology.orchestra.run.smithi027.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 456 kB/s | 106 kB 00:00 2024-08-29T14:46:48.256 INFO:teuthology.orchestra.run.smithi027.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 2.9 MB/s | 1.2 MB 00:00 2024-08-29T14:46:48.281 INFO:teuthology.orchestra.run.smithi027.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 2.1 MB/s | 125 kB 00:00 2024-08-29T14:46:48.323 INFO:teuthology.orchestra.run.smithi027.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 3.1 MB/s | 215 kB 00:00 2024-08-29T14:46:48.348 INFO:teuthology.orchestra.run.smithi027.stdout:(12/33): python3-pycparser-2.20-6.el9.noarch.rp 897 kB/s | 135 kB 00:00 2024-08-29T14:46:48.898 INFO:teuthology.orchestra.run.smithi027.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 60 kB/s | 35 kB 00:00 2024-08-29T14:46:48.949 INFO:teuthology.orchestra.run.smithi027.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 85 kB/s | 51 kB 00:00 2024-08-29T14:46:48.990 INFO:teuthology.orchestra.run.smithi027.stdout:(15/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.0 MB/s | 43 kB 00:00 2024-08-29T14:46:49.016 INFO:teuthology.orchestra.run.smithi027.stdout:(16/33): python3-mako-1.1.4-6.el9.noarch.rpm 234 kB/s | 172 kB 00:00 2024-08-29T14:46:49.058 INFO:teuthology.orchestra.run.smithi027.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 2.5 MB/s | 172 kB 00:00 2024-08-29T14:46:49.091 INFO:teuthology.orchestra.run.smithi027.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 341 kB/s | 11 kB 00:00 2024-08-29T14:46:49.117 INFO:teuthology.orchestra.run.smithi027.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 713 kB/s | 18 kB 00:00 2024-08-29T14:46:49.142 INFO:teuthology.orchestra.run.smithi027.stdout:(20/33): python3-jaraco-collections-3.0.0-8.el9 932 kB/s | 23 kB 00:00 2024-08-29T14:46:49.167 INFO:teuthology.orchestra.run.smithi027.stdout:(21/33): python3-jaraco-functools-3.5.0-2.el9.n 771 kB/s | 19 kB 00:00 2024-08-29T14:46:49.193 INFO:teuthology.orchestra.run.smithi027.stdout:(22/33): python3-jaraco-text-3.2.0-6.el9.noarch 1.1 MB/s | 20 kB 00:00 2024-08-29T14:46:49.218 INFO:teuthology.orchestra.run.smithi027.stdout:(23/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.7 MB/s | 358 kB 00:00 2024-08-29T14:46:49.234 INFO:teuthology.orchestra.run.smithi027.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 1.1 MB/s | 46 kB 00:00 2024-08-29T14:46:49.260 INFO:teuthology.orchestra.run.smithi027.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.8 MB/s | 79 kB 00:00 2024-08-29T14:46:49.277 INFO:teuthology.orchestra.run.smithi027.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 1.0 MB/s | 16 kB 00:00 2024-08-29T14:46:49.302 INFO:teuthology.orchestra.run.smithi027.stdout:(27/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 3.5 MB/s | 90 kB 00:00 2024-08-29T14:46:49.328 INFO:teuthology.orchestra.run.smithi027.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.9 MB/s | 272 kB 00:00 2024-08-29T14:46:49.353 INFO:teuthology.orchestra.run.smithi027.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 712 kB/s | 36 kB 00:00 2024-08-29T14:46:49.378 INFO:teuthology.orchestra.run.smithi027.stdout:(30/33): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2024-08-29T14:46:49.403 INFO:teuthology.orchestra.run.smithi027.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 805 kB/s | 20 kB 00:00 2024-08-29T14:46:49.428 INFO:teuthology.orchestra.run.smithi027.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.6 MB/s | 427 kB 00:00 2024-08-29T14:46:49.554 INFO:teuthology.orchestra.run.smithi027.stdout:(33/33): python3-toml-0.10.2-6.el9.noarch.rpm 64 kB/s | 42 kB 00:00 2024-08-29T14:46:49.559 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:46:49.559 INFO:teuthology.orchestra.run.smithi027.stdout:Total 12 MB/s | 29 MB 00:02 2024-08-29T14:46:49.686 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:46:49.728 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:46:49.728 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:46:49.814 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/4 2024-08-29T14:46:49.885 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/4 2024-08-29T14:46:50.029 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:46:50.029 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:46:51.255 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:46:51.315 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-08-29T14:46:51.348 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-08-29T14:46:51.381 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-08-29T14:46:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-08-29T14:46:51.554 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-08-29T14:46:51.603 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-08-29T14:46:51.660 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-08-29T14:46:51.690 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-08-29T14:46:51.724 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-08-29T14:46:51.758 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-08-29T14:46:51.807 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-08-29T14:46:51.850 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-08-29T14:46:51.909 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-08-29T14:46:51.948 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-08-29T14:46:51.949 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/4 2024-08-29T14:46:51.950 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/4 2024-08-29T14:46:51.950 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/4 2024-08-29T14:46:51.950 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-29T14:46:52.025 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-08-29T14:46:52.175 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-08-29T14:46:52.241 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-29T14:46:52.241 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:52.241 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:46:52.241 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-test-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:52.242 INFO:teuthology.orchestra.run.smithi171.stdout: libcephsqlite-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:52.242 INFO:teuthology.orchestra.run.smithi171.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-29T14:46:52.242 INFO:teuthology.orchestra.run.smithi171.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-29T14:46:52.242 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:52.242 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:46:52.258 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-08-29T14:46:52.310 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-08-29T14:46:52.348 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-08-29T14:46:52.407 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-08-29T14:46:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-08-29T14:46:52.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph 2024-08-29T14:46:52.666 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-08-29T14:46:52.718 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-08-29T14:46:52.799 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-08-29T14:46:52.896 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-08-29T14:46:52.933 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-08-29T14:46:53.056 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-08-29T14:46:53.089 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:36 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:46:53.216 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-774.gfe43f1c9.el9.n 28/33 2024-08-29T14:46:53.227 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout: ceph x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 6.5 k 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mds x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 2.1 M 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 1.5 M 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-modules-core noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 246 k 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mon x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 4.7 M 2024-08-29T14:46:53.230 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-osd x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 17 M 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-29T14:46:53.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-08-29T14:46:53.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-29T14:46:53.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-29T14:46:53.233 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:53.233 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:46:53.233 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:46:53.233 INFO:teuthology.orchestra.run.smithi171.stdout:Install 33 Packages 2024-08-29T14:46:53.233 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:53.233 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 29 M 2024-08-29T14:46:53.234 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 104 M 2024-08-29T14:46:53.234 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:46:53.243 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/33 2024-08-29T14:46:53.270 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/33 2024-08-29T14:46:53.271 INFO:teuthology.orchestra.run.smithi027.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:46:53.271 INFO:teuthology.orchestra.run.smithi027.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-29T14:46:53.271 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-29T14:46:53.271 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-29T14:46:53.271 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:53.575 INFO:teuthology.orchestra.run.smithi171.stdout:(1/33): ceph-18.2.4-774.gfe43f1c9.el9.x86_64.rp 57 kB/s | 6.5 kB 00:00 2024-08-29T14:46:53.801 INFO:teuthology.orchestra.run.smithi171.stdout:(2/33): ceph-mds-18.2.4-774.gfe43f1c9.el9.x86_6 6.2 MB/s | 2.1 MB 00:00 2024-08-29T14:46:53.842 INFO:teuthology.orchestra.run.smithi171.stdout:(3/33): ceph-mgr-18.2.4-774.gfe43f1c9.el9.x86_6 3.8 MB/s | 1.5 MB 00:00 2024-08-29T14:46:53.935 INFO:teuthology.orchestra.run.smithi171.stdout:(4/33): ceph-mon-18.2.4-774.gfe43f1c9.el9.x86_6 13 MB/s | 4.7 MB 00:00 2024-08-29T14:46:53.960 INFO:teuthology.orchestra.run.smithi171.stdout:(5/33): ceph-mgr-modules-core-18.2.4-774.gfe43f 2.1 MB/s | 246 kB 00:00 2024-08-29T14:46:54.077 INFO:teuthology.orchestra.run.smithi171.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.7 MB/s | 253 kB 00:00 2024-08-29T14:46:54.127 INFO:teuthology.orchestra.run.smithi171.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-08-29T14:46:54.177 INFO:teuthology.orchestra.run.smithi171.stdout:(8/33): python3-cryptography-36.0.1-4.el9.x86_6 5.6 MB/s | 1.2 MB 00:00 2024-08-29T14:46:54.203 INFO:teuthology.orchestra.run.smithi171.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 1.8 MB/s | 135 kB 00:00 2024-08-29T14:46:54.228 INFO:teuthology.orchestra.run.smithi171.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 2.4 MB/s | 125 kB 00:00 2024-08-29T14:46:54.254 INFO:teuthology.orchestra.run.smithi171.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-08-29T14:46:54.462 INFO:teuthology.orchestra.run.smithi171.stdout:(12/33): ceph-osd-18.2.4-774.gfe43f1c9.el9.x86_ 27 MB/s | 17 MB 00:00 2024-08-29T14:46:54.761 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/33 2024-08-29T14:46:54.790 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/33 2024-08-29T14:46:54.790 INFO:teuthology.orchestra.run.smithi027.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:46:54.790 INFO:teuthology.orchestra.run.smithi027.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-29T14:46:54.791 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-29T14:46:54.791 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-29T14:46:54.791 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:54.963 INFO:teuthology.orchestra.run.smithi171.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 49 kB/s | 35 kB 00:00 2024-08-29T14:46:55.080 INFO:teuthology.orchestra.run.smithi171.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 83 kB/s | 51 kB 00:00 2024-08-29T14:46:55.113 INFO:teuthology.orchestra.run.smithi171.stdout:(15/33): python3-mako-1.1.4-6.el9.noarch.rpm 195 kB/s | 172 kB 00:00 2024-08-29T14:46:55.147 INFO:teuthology.orchestra.run.smithi171.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 652 kB/s | 43 kB 00:00 2024-08-29T14:46:55.181 INFO:teuthology.orchestra.run.smithi171.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 2.6 MB/s | 172 kB 00:00 2024-08-29T14:46:55.206 INFO:teuthology.orchestra.run.smithi171.stdout:(18/33): python3-cherrypy-18.6.1-2.el9.noarch.r 6.2 MB/s | 358 kB 00:00 2024-08-29T14:46:55.231 INFO:teuthology.orchestra.run.smithi171.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 211 kB/s | 11 kB 00:00 2024-08-29T14:46:55.248 INFO:teuthology.orchestra.run.smithi171.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 506 kB/s | 18 kB 00:00 2024-08-29T14:46:55.262 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 31/33 2024-08-29T14:46:55.265 INFO:teuthology.orchestra.run.smithi171.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 685 kB/s | 23 kB 00:00 2024-08-29T14:46:55.283 INFO:teuthology.orchestra.run.smithi171.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 576 kB/s | 19 kB 00:00 2024-08-29T14:46:55.287 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 31/33 2024-08-29T14:46:55.288 INFO:teuthology.orchestra.run.smithi027.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:46:55.288 INFO:teuthology.orchestra.run.smithi027.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-29T14:46:55.288 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-29T14:46:55.288 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-29T14:46:55.288 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:55.299 INFO:teuthology.orchestra.run.smithi171.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 585 kB/s | 20 kB 00:00 2024-08-29T14:46:55.317 INFO:teuthology.orchestra.run.smithi171.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 1.3 MB/s | 46 kB 00:00 2024-08-29T14:46:55.343 INFO:teuthology.orchestra.run.smithi171.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.9 MB/s | 79 kB 00:00 2024-08-29T14:46:55.678 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 32/33 2024-08-29T14:46:55.678 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 32/33 2024-08-29T14:46:55.678 INFO:teuthology.orchestra.run.smithi027.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:46:55.678 INFO:teuthology.orchestra.run.smithi027.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-29T14:46:55.678 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-29T14:46:55.678 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-29T14:46:55.678 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:55.679 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 33/33 2024-08-29T14:46:55.679 INFO:teuthology.orchestra.run.smithi171.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 657 kB/s | 16 kB 00:00 2024-08-29T14:46:55.679 INFO:teuthology.orchestra.run.smithi171.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 3.5 MB/s | 272 kB 00:00 2024-08-29T14:46:55.679 INFO:teuthology.orchestra.run.smithi171.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-08-29T14:46:55.679 INFO:teuthology.orchestra.run.smithi171.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 2.1 MB/s | 36 kB 00:00 2024-08-29T14:46:55.679 INFO:teuthology.orchestra.run.smithi171.stdout:(30/33): python3-webob-1.8.8-2.el9.noarch.rpm 5.4 MB/s | 230 kB 00:00 2024-08-29T14:46:55.680 INFO:teuthology.orchestra.run.smithi171.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 1.2 MB/s | 20 kB 00:00 2024-08-29T14:46:55.680 INFO:teuthology.orchestra.run.smithi171.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 6.2 MB/s | 427 kB 00:00 2024-08-29T14:46:55.680 INFO:teuthology.orchestra.run.smithi171.stdout:(33/33): python3-toml-0.10.2-6.el9.noarch.rpm 62 kB/s | 42 kB 00:00 2024-08-29T14:46:55.680 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:46:55.680 INFO:teuthology.orchestra.run.smithi171.stdout:Total 12 MB/s | 29 MB 00:02 2024-08-29T14:46:55.766 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:46:55.804 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:46:55.804 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:46:56.087 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:46:56.087 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 33/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-774.gfe43f1c9.el9.n 6/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-08-29T14:46:57.315 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-08-29T14:46:57.316 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-08-29T14:46:57.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-08-29T14:46:57.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-08-29T14:46:57.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-08-29T14:46:57.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-08-29T14:46:57.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-08-29T14:46:57.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-08-29T14:46:57.318 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-08-29T14:46:57.318 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-08-29T14:46:57.318 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-08-29T14:46:57.372 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:46:57.439 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-08-29T14:46:57.480 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-08-29T14:46:57.513 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-08-29T14:46:57.614 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-08-29T14:46:57.691 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-08-29T14:46:57.734 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-08-29T14:46:57.777 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-08-29T14:46:57.807 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-08-29T14:46:57.842 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-08-29T14:46:57.875 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-08-29T14:46:57.908 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-08-29T14:46:57.951 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-08-29T14:46:57.985 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-08-29T14:46:58.024 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-08-29T14:46:58.108 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-08-29T14:46:58.252 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-08-29T14:46:58.278 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-08-29T14:46:58.278 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:58.278 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-modules-core-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-29T14:46:58.279 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ply-3.11-14.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-29T14:46:58.280 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-webob-1.8.8-2.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:46:58.281 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:46:58.341 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-08-29T14:46:58.395 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-08-29T14:46:58.434 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-08-29T14:46:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-08-29T14:46:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-08-29T14:46:58.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-base 2024-08-29T14:46:58.746 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-08-29T14:46:58.794 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-08-29T14:46:58.873 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-08-29T14:46:58.970 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-08-29T14:46:59.010 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-08-29T14:46:59.127 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-08-29T14:46:59.236 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:49 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:46:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-774.gfe43f1c9.el9.n 28/33 2024-08-29T14:46:59.313 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/33 2024-08-29T14:46:59.328 INFO:teuthology.orchestra.run.smithi027.stdout:Package ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:46:59.339 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 29/33 2024-08-29T14:46:59.339 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:46:59.339 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-29T14:46:59.339 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-29T14:46:59.339 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-29T14:46:59.339 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:46:59.373 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:46:59.373 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:46:59.373 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:46:59.425 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install cephadm 2024-08-29T14:47:00.000 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:49 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:00.147 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:00.148 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:00.148 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:47:00.148 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:00.148 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:00.148 INFO:teuthology.orchestra.run.smithi027.stdout: cephadm noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 224 k 2024-08-29T14:47:00.148 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:00.149 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:00.149 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:00.149 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2024-08-29T14:47:00.149 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:00.149 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 224 k 2024-08-29T14:47:00.149 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 216 k 2024-08-29T14:47:00.149 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:00.352 INFO:teuthology.orchestra.run.smithi027.stdout:cephadm-18.2.4-774.gfe43f1c9.el9.noarch.rpm 1.0 MB/s | 224 kB 00:00 2024-08-29T14:47:00.353 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:00.354 INFO:teuthology.orchestra.run.smithi027.stdout:Total 1.0 MB/s | 224 kB 00:00 2024-08-29T14:47:00.354 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:00.359 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:00.359 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:00.375 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:00.375 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:00.438 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:00.791 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/33 2024-08-29T14:47:00.819 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 30/33 2024-08-29T14:47:00.820 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:47:00.820 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-29T14:47:00.820 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-29T14:47:00.820 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-29T14:47:00.820 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:01.013 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:01.094 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:01.267 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 31/33 2024-08-29T14:47:01.292 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 31/33 2024-08-29T14:47:01.292 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:47:01.292 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-29T14:47:01.292 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-29T14:47:01.292 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-29T14:47:01.292 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:01.431 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:01.501 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 32/33 2024-08-29T14:47:01.525 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 32/33 2024-08-29T14:47:01.525 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:47:01.525 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-29T14:47:01.525 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-29T14:47:01.525 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-29T14:47:01.525 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:01.588 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 33/33 2024-08-29T14:47:01.677 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:01.677 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:01.677 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:01.677 INFO:teuthology.orchestra.run.smithi027.stdout: cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:01.677 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:01.678 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:01.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-immutable-object-cache 2024-08-29T14:47:02.443 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:52 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:02.579 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:02.580 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:02.580 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2024-08-29T14:47:02.580 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 143 k 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 143 k 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 395 k 2024-08-29T14:47:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:02.788 INFO:teuthology.orchestra.run.smithi027.stdout:ceph-immutable-object-cache-18.2.4-774.gfe43f1c 692 kB/s | 143 kB 00:00 2024-08-29T14:47:02.788 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:02.788 INFO:teuthology.orchestra.run.smithi027.stdout:Total 686 kB/s | 143 kB 00:00 2024-08-29T14:47:02.789 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:02.797 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:02.797 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:02.859 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:02.859 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:03.020 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:03.105 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.e 1/1 2024-08-29T14:47:03.132 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.e 1/1 2024-08-29T14:47:03.132 INFO:teuthology.orchestra.run.smithi027.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:47:03.132 INFO:teuthology.orchestra.run.smithi027.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-29T14:47:03.133 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 33/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 3/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 4/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 5/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-774.gfe43f1c9.el9.n 6/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-08-29T14:47:03.337 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-08-29T14:47:03.338 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-08-29T14:47:03.339 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-08-29T14:47:03.339 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-08-29T14:47:03.339 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-08-29T14:47:03.340 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-08-29T14:47:03.340 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-08-29T14:47:03.340 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-08-29T14:47:03.340 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-08-29T14:47:03.340 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-08-29T14:47:03.340 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-08-29T14:47:03.340 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-08-29T14:47:04.190 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.e 1/1 2024-08-29T14:47:04.190 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:04.190 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:04.190 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:04.190 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:04.190 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:04.230 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mds-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-modules-core-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mon-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-osd-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-29T14:47:04.231 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-ply-3.11-14.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-29T14:47:04.232 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-webob-1.8.8-2.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:04.233 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:04.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr 2024-08-29T14:47:04.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-base 2024-08-29T14:47:04.980 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:54 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:05.072 INFO:teuthology.orchestra.run.smithi027.stdout:Package ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:05.117 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:05.118 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:05.118 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:05.160 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:48 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:05.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-dashboard 2024-08-29T14:47:05.250 INFO:teuthology.orchestra.run.smithi171.stdout:Package ceph-base-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:05.292 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:05.293 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:05.293 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:05.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install cephadm 2024-08-29T14:47:05.752 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:55 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:05.889 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:05.889 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:05.889 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:47:05.889 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:05.889 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-dashboard noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 3.5 M 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-grafana-dashboards noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 24 k 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-prometheus-alerts noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 16 k 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout:Install 5 Packages 2024-08-29T14:47:05.890 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:05.891 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 3.8 M 2024-08-29T14:47:05.891 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 87 M 2024-08-29T14:47:05.891 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:05.918 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:48 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:06.049 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:06.050 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:06.050 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:47:06.050 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:06.050 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:06.050 INFO:teuthology.orchestra.run.smithi171.stdout: cephadm noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 224 k 2024-08-29T14:47:06.050 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:06.051 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:06.051 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:06.051 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-08-29T14:47:06.051 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:06.051 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 224 k 2024-08-29T14:47:06.051 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 216 k 2024-08-29T14:47:06.051 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:06.208 INFO:teuthology.orchestra.run.smithi027.stdout:(1/5): ceph-grafana-dashboards-18.2.4-774.gfe43 165 kB/s | 24 kB 00:00 2024-08-29T14:47:06.241 INFO:teuthology.orchestra.run.smithi027.stdout:(2/5): ceph-prometheus-alerts-18.2.4-774.gfe43f 91 kB/s | 16 kB 00:00 2024-08-29T14:47:06.254 INFO:teuthology.orchestra.run.smithi171.stdout:cephadm-18.2.4-774.gfe43f1c9.el9.noarch.rpm 1.1 MB/s | 224 kB 00:00 2024-08-29T14:47:06.255 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:06.255 INFO:teuthology.orchestra.run.smithi171.stdout:Total 1.1 MB/s | 224 kB 00:00 2024-08-29T14:47:06.256 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:06.260 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:06.260 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:06.263 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:06.263 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:06.283 INFO:teuthology.orchestra.run.smithi027.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 409 kB/s | 31 kB 00:00 2024-08-29T14:47:06.308 INFO:teuthology.orchestra.run.smithi027.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.8 MB/s | 188 kB 00:00 2024-08-29T14:47:06.329 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:06.475 INFO:teuthology.orchestra.run.smithi027.stdout:(5/5): ceph-mgr-dashboard-18.2.4-774.gfe43f1c9. 8.5 MB/s | 3.5 MB 00:00 2024-08-29T14:47:06.478 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:06.479 INFO:teuthology.orchestra.run.smithi027.stdout:Total 6.4 MB/s | 3.8 MB 00:00 2024-08-29T14:47:06.479 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:06.486 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:06.487 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:06.554 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:06.554 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:06.794 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:06.809 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:06.883 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:06.891 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-08-29T14:47:06.921 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-08-29T14:47:06.950 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-774.gfe43f1c9.el9.no 3/5 2024-08-29T14:47:07.207 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:07.471 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/1 2024-08-29T14:47:07.472 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:07.472 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:07.472 INFO:teuthology.orchestra.run.smithi171.stdout: cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:07.472 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:07.472 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:07.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-immutable-object-cache 2024-08-29T14:47:08.064 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-774.gfe43f1c9.el9.n 4/5 2024-08-29T14:47:08.096 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 5/5 2024-08-29T14:47:08.294 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:51 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:08.431 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 143 k 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:08.432 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-08-29T14:47:08.433 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:08.433 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 143 k 2024-08-29T14:47:08.433 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 395 k 2024-08-29T14:47:08.433 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:08.647 INFO:teuthology.orchestra.run.smithi171.stdout:ceph-immutable-object-cache-18.2.4-774.gfe43f1c 665 kB/s | 143 kB 00:00 2024-08-29T14:47:08.648 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:08.648 INFO:teuthology.orchestra.run.smithi171.stdout:Total 660 kB/s | 143 kB 00:00 2024-08-29T14:47:08.649 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:08.655 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:08.656 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:08.714 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:08.715 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:08.861 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:08.932 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.e 1/1 2024-08-29T14:47:08.958 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.e 1/1 2024-08-29T14:47:08.958 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:47:08.958 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-29T14:47:08.958 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:09.236 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 5/5 2024-08-29T14:47:09.236 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-774.gfe43f1c9.el9.n 1/5 2024-08-29T14:47:09.236 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 2/5 2024-08-29T14:47:09.236 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-774.gfe43f1c9.el9.no 3/5 2024-08-29T14:47:09.236 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-08-29T14:47:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-08-29T14:47:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:09.505 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-grafana-dashboards-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-prometheus-alerts-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-29T14:47:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-29T14:47:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:09.506 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:09.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-29T14:47:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.e 1/1 2024-08-29T14:47:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:09.897 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-immutable-object-cache-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:09.897 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:10.111 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr 2024-08-29T14:47:10.275 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:00 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:10.413 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:10.413 INFO:teuthology.orchestra.run.smithi027.stdout:====================================================================================== 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout:====================================================================================== 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 7.4 M 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-08-29T14:47:10.414 INFO:teuthology.orchestra.run.smithi027.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout:====================================================================================== 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout:Install 12 Packages 2024-08-29T14:47:10.415 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:10.416 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 38 M 2024-08-29T14:47:10.416 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 212 M 2024-08-29T14:47:10.416 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:10.700 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:53 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:10.792 INFO:teuthology.orchestra.run.smithi171.stdout:Package ceph-mgr-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:10.834 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:10.835 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:10.835 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:10.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-dashboard 2024-08-29T14:47:11.132 INFO:teuthology.orchestra.run.smithi027.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 994 kB/s | 191 kB 00:00 2024-08-29T14:47:11.224 INFO:teuthology.orchestra.run.smithi027.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.8 MB/s | 801 kB 00:00 2024-08-29T14:47:11.482 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:54 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:11.614 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:11.615 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:11.615 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:47:11.615 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:11.615 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:11.615 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-dashboard noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 3.5 M 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-grafana-dashboards noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 24 k 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-prometheus-alerts noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 16 k 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout:Install 5 Packages 2024-08-29T14:47:11.616 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:11.617 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 3.8 M 2024-08-29T14:47:11.617 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 87 M 2024-08-29T14:47:11.617 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:11.699 INFO:teuthology.orchestra.run.smithi027.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 58 kB/s | 33 kB 00:00 2024-08-29T14:47:11.805 INFO:teuthology.orchestra.run.smithi171.stdout:(1/5): ceph-grafana-dashboards-18.2.4-774.gfe43 157 kB/s | 24 kB 00:00 2024-08-29T14:47:11.830 INFO:teuthology.orchestra.run.smithi171.stdout:(2/5): ceph-prometheus-alerts-18.2.4-774.gfe43f 91 kB/s | 16 kB 00:00 2024-08-29T14:47:11.872 INFO:teuthology.orchestra.run.smithi171.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 460 kB/s | 31 kB 00:00 2024-08-29T14:47:11.897 INFO:teuthology.orchestra.run.smithi171.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.8 MB/s | 188 kB 00:00 2024-08-29T14:47:12.183 INFO:teuthology.orchestra.run.smithi027.stdout:(4/12): ceph-mgr-diskprediction-local-18.2.4-77 5.9 MB/s | 7.4 MB 00:01 2024-08-29T14:47:12.208 INFO:teuthology.orchestra.run.smithi027.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 36 kB/s | 18 kB 00:00 2024-08-29T14:47:12.350 INFO:teuthology.orchestra.run.smithi027.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.7 MB/s | 3.0 MB 00:01 2024-08-29T14:47:12.572 INFO:teuthology.orchestra.run.smithi171.stdout:(5/5): ceph-mgr-dashboard-18.2.4-774.gfe43f1c9. 3.8 MB/s | 3.5 MB 00:00 2024-08-29T14:47:12.576 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:12.576 INFO:teuthology.orchestra.run.smithi171.stdout:Total 3.9 MB/s | 3.8 MB 00:00 2024-08-29T14:47:12.576 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:12.584 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:12.584 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:12.649 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:12.649 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:12.784 INFO:teuthology.orchestra.run.smithi027.stdout:(7/12): openblas-0.3.26-2.el9.x86_64.rpm 64 kB/s | 38 kB 00:00 2024-08-29T14:47:12.901 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:12.995 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-08-29T14:47:13.036 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-08-29T14:47:13.065 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-774.gfe43f1c9.el9.no 3/5 2024-08-29T14:47:13.109 INFO:teuthology.orchestra.run.smithi027.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 323 kB/s | 245 kB 00:00 2024-08-29T14:47:13.351 INFO:teuthology.orchestra.run.smithi027.stdout:(9/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 4.3 MB/s | 4.9 MB 00:01 2024-08-29T14:47:13.901 INFO:teuthology.orchestra.run.smithi027.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.6 MB/s | 5.1 MB 00:01 2024-08-29T14:47:13.926 INFO:teuthology.orchestra.run.smithi027.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 307 kB/s | 250 kB 00:00 2024-08-29T14:47:14.121 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-774.gfe43f1c9.el9.n 4/5 2024-08-29T14:47:14.153 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 5/5 2024-08-29T14:47:14.970 INFO:teuthology.orchestra.run.smithi027.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 10 MB/s | 16 MB 00:01 2024-08-29T14:47:14.971 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:14.971 INFO:teuthology.orchestra.run.smithi027.stdout:Total 8.4 MB/s | 38 MB 00:04 2024-08-29T14:47:15.338 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 5/5 2024-08-29T14:47:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-774.gfe43f1c9.el9.n 1/5 2024-08-29T14:47:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 2/5 2024-08-29T14:47:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-774.gfe43f1c9.el9.no 3/5 2024-08-29T14:47:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-08-29T14:47:15.365 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:15.366 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:15.596 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-08-29T14:47:15.596 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:15.596 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:15.596 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-grafana-dashboards-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:15.597 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-dashboard-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:15.597 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-prometheus-alerts-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:15.597 INFO:teuthology.orchestra.run.smithi171.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-29T14:47:15.597 INFO:teuthology.orchestra.run.smithi171.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-29T14:47:15.597 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:15.597 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:15.803 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:15.803 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:15.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-29T14:47:16.396 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:00:59 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:16.529 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout:====================================================================================== 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout:====================================================================================== 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 7.4 M 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-29T14:47:16.530 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout:====================================================================================== 2024-08-29T14:47:16.531 INFO:teuthology.orchestra.run.smithi171.stdout:Install 12 Packages 2024-08-29T14:47:16.532 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:16.532 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 38 M 2024-08-29T14:47:16.532 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 212 M 2024-08-29T14:47:16.532 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:16.701 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:16.766 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-29T14:47:16.818 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-08-29T14:47:16.865 INFO:teuthology.orchestra.run.smithi171.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.0 MB/s | 191 kB 00:00 2024-08-29T14:47:16.907 INFO:teuthology.orchestra.run.smithi171.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 3.5 MB/s | 801 kB 00:00 2024-08-29T14:47:16.923 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-08-29T14:47:16.932 INFO:teuthology.orchestra.run.smithi171.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 488 kB/s | 33 kB 00:00 2024-08-29T14:47:16.970 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-08-29T14:47:16.974 INFO:teuthology.orchestra.run.smithi171.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 438 kB/s | 18 kB 00:00 2024-08-29T14:47:17.033 INFO:teuthology.orchestra.run.smithi171.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 24 MB/s | 3.0 MB 00:00 2024-08-29T14:47:17.075 INFO:teuthology.orchestra.run.smithi171.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 383 kB/s | 38 kB 00:00 2024-08-29T14:47:17.161 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-08-29T14:47:17.184 INFO:teuthology.orchestra.run.smithi171.stdout:(7/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 33 MB/s | 4.9 MB 00:00 2024-08-29T14:47:17.193 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-08-29T14:47:17.209 INFO:teuthology.orchestra.run.smithi171.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.8 MB/s | 245 kB 00:00 2024-08-29T14:47:17.272 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-29T14:47:17.318 INFO:teuthology.orchestra.run.smithi171.stdout:(9/12): ceph-mgr-diskprediction-local-18.2.4-77 12 MB/s | 7.4 MB 00:00 2024-08-29T14:47:17.344 INFO:teuthology.orchestra.run.smithi171.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.8 MB/s | 250 kB 00:00 2024-08-29T14:47:17.419 INFO:teuthology.orchestra.run.smithi171.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 22 MB/s | 5.1 MB 00:00 2024-08-29T14:47:17.861 INFO:teuthology.orchestra.run.smithi171.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 30 MB/s | 16 MB 00:00 2024-08-29T14:47:17.862 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:17.862 INFO:teuthology.orchestra.run.smithi171.stdout:Total 29 MB/s | 38 MB 00:01 2024-08-29T14:47:17.957 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-29T14:47:18.079 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-29T14:47:18.219 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:18.245 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:18.245 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:18.659 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:18.659 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:19.383 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-29T14:47:19.435 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:19.532 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-29T14:47:19.568 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-08-29T14:47:19.668 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-08-29T14:47:19.711 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-08-29T14:47:19.896 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-08-29T14:47:19.927 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-08-29T14:47:20.003 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-29T14:47:20.448 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-29T14:47:20.476 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1 12/12 2024-08-29T14:47:20.656 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-29T14:47:20.775 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-29T14:47:22.002 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-29T14:47:22.972 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1 12/12 2024-08-29T14:47:22.972 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1 1/12 2024-08-29T14:47:22.972 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-08-29T14:47:22.972 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-29T14:47:22.973 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-29T14:47:23.056 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-29T14:47:23.085 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1 12/12 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-08-29T14:47:23.359 INFO:teuthology.orchestra.run.smithi027.stdout: openblas-0.3.26-2.el9.x86_64 2024-08-29T14:47:23.360 INFO:teuthology.orchestra.run.smithi027.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-08-29T14:47:23.360 INFO:teuthology.orchestra.run.smithi027.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-08-29T14:47:23.360 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-29T14:47:23.360 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-29T14:47:23.360 INFO:teuthology.orchestra.run.smithi027.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-29T14:47:23.360 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:23.360 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:23.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-rook 2024-08-29T14:47:24.317 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:14 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:24.457 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-rook noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 49 k 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-29T14:47:24.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout:Install 13 Packages 2024-08-29T14:47:24.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:24.460 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 2.2 M 2024-08-29T14:47:24.460 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 26 M 2024-08-29T14:47:24.460 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:24.809 INFO:teuthology.orchestra.run.smithi027.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 173 kB/s | 26 kB 00:00 2024-08-29T14:47:24.834 INFO:teuthology.orchestra.run.smithi027.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 110 kB/s | 19 kB 00:00 2024-08-29T14:47:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:(3/13): ceph-mgr-rook-18.2.4-774.gfe43f1c9.el9. 258 kB/s | 49 kB 00:00 2024-08-29T14:47:24.943 INFO:teuthology.orchestra.run.smithi027.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.6 MB/s | 222 kB 00:00 2024-08-29T14:47:24.993 INFO:teuthology.orchestra.run.smithi027.stdout:(5/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-08-29T14:47:25.027 INFO:teuthology.orchestra.run.smithi027.stdout:(6/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 824 kB/s | 159 kB 00:00 2024-08-29T14:47:25.052 INFO:teuthology.orchestra.run.smithi027.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 549 kB/s | 32 kB 00:00 2024-08-29T14:47:25.078 INFO:teuthology.orchestra.run.smithi027.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.2 MB/s | 279 kB 00:00 2024-08-29T14:47:25.103 INFO:teuthology.orchestra.run.smithi027.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 191 kB/s | 14 kB 00:00 2024-08-29T14:47:25.137 INFO:teuthology.orchestra.run.smithi027.stdout:(10/13): python3-google-auth-2.34.0-1.el9.noarc 2.6 MB/s | 220 kB 00:00 2024-08-29T14:47:25.162 INFO:teuthology.orchestra.run.smithi027.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-08-29T14:47:25.187 INFO:teuthology.orchestra.run.smithi027.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-08-29T14:47:25.237 INFO:teuthology.orchestra.run.smithi027.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.4 MB/s | 1.0 MB 00:00 2024-08-29T14:47:25.240 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:25.240 INFO:teuthology.orchestra.run.smithi027.stdout:Total 2.8 MB/s | 2.2 MB 00:00 2024-08-29T14:47:25.306 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:25.323 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:25.323 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:25.417 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:25.417 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:25.656 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1 12/12 2024-08-29T14:47:25.656 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1 1/12 2024-08-29T14:47:25.656 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-08-29T14:47:25.656 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-08-29T14:47:25.656 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-29T14:47:25.656 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-29T14:47:25.656 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-29T14:47:25.657 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-08-29T14:47:25.657 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-08-29T14:47:25.657 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-08-29T14:47:25.657 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-29T14:47:25.657 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-29T14:47:25.940 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:26.039 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-29T14:47:26.039 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:26.039 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:26.039 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-diskprediction-local-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:26.039 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-29T14:47:26.039 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-29T14:47:26.039 INFO:teuthology.orchestra.run.smithi171.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-0.3.26-2.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:26.040 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:26.125 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-29T14:47:26.182 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-29T14:47:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-29T14:47:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-29T14:47:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-29T14:47:26.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-rook 2024-08-29T14:47:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-29T14:47:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-08-29T14:47:26.592 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-29T14:47:26.953 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:09 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:27.085 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:27.086 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-rook noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 49 k 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-29T14:47:27.087 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout:Install 13 Packages 2024-08-29T14:47:27.088 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:27.089 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 2.2 M 2024-08-29T14:47:27.089 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 26 M 2024-08-29T14:47:27.089 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:27.338 INFO:teuthology.orchestra.run.smithi171.stdout:(1/13): ceph-mgr-rook-18.2.4-774.gfe43f1c9.el9. 290 kB/s | 49 kB 00:00 2024-08-29T14:47:27.351 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-29T14:47:27.397 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-29T14:47:27.431 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-29T14:47:27.480 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-29T14:47:27.500 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 13/13 2024-08-29T14:47:27.721 INFO:teuthology.orchestra.run.smithi171.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 35 kB/s | 19 kB 00:00 2024-08-29T14:47:27.747 INFO:teuthology.orchestra.run.smithi171.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 44 kB/s | 26 kB 00:00 2024-08-29T14:47:28.189 INFO:teuthology.orchestra.run.smithi171.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 261 kB/s | 222 kB 00:00 2024-08-29T14:47:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 13/13 2024-08-29T14:47:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 1/13 2024-08-29T14:47:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-29T14:47:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-29T14:47:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-29T14:47:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-29T14:47:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-29T14:47:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-29T14:47:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-29T14:47:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-29T14:47:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-08-29T14:47:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-29T14:47:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-29T14:47:28.481 INFO:teuthology.orchestra.run.smithi171.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 209 kB/s | 159 kB 00:00 2024-08-29T14:47:28.530 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-29T14:47:28.530 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-29T14:47:28.531 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-29T14:47:28.532 INFO:teuthology.orchestra.run.smithi027.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-29T14:47:28.532 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:28.532 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:28.539 INFO:teuthology.orchestra.run.smithi171.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 555 kB/s | 32 kB 00:00 2024-08-29T14:47:28.573 INFO:teuthology.orchestra.run.smithi171.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 423 kB/s | 14 kB 00:00 2024-08-29T14:47:28.599 INFO:teuthology.orchestra.run.smithi171.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 327 kB/s | 279 kB 00:00 2024-08-29T14:47:28.640 INFO:teuthology.orchestra.run.smithi171.stdout:(9/13): python3-google-auth-2.34.0-1.el9.noarch 3.2 MB/s | 220 kB 00:00 2024-08-29T14:47:28.695 INFO:teuthology.orchestra.run.smithi171.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-08-29T14:47:28.733 INFO:teuthology.orchestra.run.smithi171.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 2.3 MB/s | 90 kB 00:00 2024-08-29T14:47:28.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-cephadm 2024-08-29T14:47:28.783 INFO:teuthology.orchestra.run.smithi171.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 5.6 MB/s | 1.0 MB 00:00 2024-08-29T14:47:28.825 INFO:teuthology.orchestra.run.smithi171.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 84 kB/s | 54 kB 00:00 2024-08-29T14:47:28.829 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:28.829 INFO:teuthology.orchestra.run.smithi171.stdout:Total 1.3 MB/s | 2.2 MB 00:01 2024-08-29T14:47:28.901 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:28.917 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:28.917 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:29.006 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:29.007 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:29.360 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:19 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:29.499 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-cephadm noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 139 k 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-29T14:47:29.500 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-29T14:47:29.501 INFO:teuthology.orchestra.run.smithi027.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-29T14:47:29.501 INFO:teuthology.orchestra.run.smithi027.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-29T14:47:29.501 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:29.501 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:29.501 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:29.501 INFO:teuthology.orchestra.run.smithi027.stdout:Install 6 Packages 2024-08-29T14:47:29.501 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:29.502 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 7.0 M 2024-08-29T14:47:29.502 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 33 M 2024-08-29T14:47:29.502 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:29.509 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-29T14:47:29.736 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-29T14:47:29.792 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-29T14:47:29.814 INFO:teuthology.orchestra.run.smithi027.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.4 MB/s | 248 kB 00:00 2024-08-29T14:47:29.837 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-29T14:47:29.865 INFO:teuthology.orchestra.run.smithi027.stdout:(2/6): ceph-mgr-cephadm-18.2.4-774.gfe43f1c9.el 619 kB/s | 139 kB 00:00 2024-08-29T14:47:29.887 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-29T14:47:29.990 INFO:teuthology.orchestra.run.smithi027.stdout:(3/6): python3-babel-2.9.1-2.el9.noarch.rpm 17 MB/s | 6.0 MB 00:00 2024-08-29T14:47:29.998 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-29T14:47:30.015 INFO:teuthology.orchestra.run.smithi027.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 384 kB/s | 58 kB 00:00 2024-08-29T14:47:30.049 INFO:teuthology.orchestra.run.smithi027.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 2.3 MB/s | 548 kB 00:00 2024-08-29T14:47:30.090 INFO:teuthology.orchestra.run.smithi027.stdout:(6/6): python3-typing-extensions-4.12.2-2.el9.n 746 kB/s | 74 kB 00:00 2024-08-29T14:47:30.093 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:30.094 INFO:teuthology.orchestra.run.smithi027.stdout:Total 12 MB/s | 7.0 MB 00:00 2024-08-29T14:47:30.108 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-08-29T14:47:30.164 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-29T14:47:30.167 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:30.179 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:30.179 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:30.284 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:30.716 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:30.881 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-29T14:47:30.889 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-29T14:47:30.938 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-29T14:47:30.944 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-29T14:47:30.973 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-29T14:47:31.021 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-29T14:47:31.050 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 13/13 2024-08-29T14:47:31.500 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-29T14:47:31.590 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-29T14:47:31.660 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-29T14:47:31.688 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 6/6 2024-08-29T14:47:31.800 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 13/13 2024-08-29T14:47:31.800 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 1/13 2024-08-29T14:47:31.800 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-29T14:47:31.800 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-29T14:47:31.800 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-29T14:47:31.800 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-29T14:47:31.801 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-29T14:47:31.801 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-29T14:47:31.801 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-29T14:47:31.801 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-29T14:47:31.801 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-08-29T14:47:31.801 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-29T14:47:31.801 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-rook-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-29T14:47:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-29T14:47:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-29T14:47:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-29T14:47:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-29T14:47:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-29T14:47:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-29T14:47:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:32.131 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:32.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-mgr-cephadm 2024-08-29T14:47:32.412 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 6/6 2024-08-29T14:47:32.412 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/6 2024-08-29T14:47:32.413 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-29T14:47:32.413 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-29T14:47:32.413 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-29T14:47:32.413 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-29T14:47:32.670 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-29T14:47:32.670 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:32.670 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:32.670 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:32.670 INFO:teuthology.orchestra.run.smithi027.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-29T14:47:32.671 INFO:teuthology.orchestra.run.smithi027.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-29T14:47:32.671 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-29T14:47:32.671 INFO:teuthology.orchestra.run.smithi027.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-29T14:47:32.671 INFO:teuthology.orchestra.run.smithi027.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-29T14:47:32.671 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:32.671 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:32.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-fuse 2024-08-29T14:47:32.955 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:15 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:33.088 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:33.088 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-cephadm noarch 2:18.2.4-774.gfe43f1c9.el9 ceph-noarch 139 k 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:33.089 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:33.090 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:33.090 INFO:teuthology.orchestra.run.smithi171.stdout:Install 6 Packages 2024-08-29T14:47:33.090 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:33.090 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 7.0 M 2024-08-29T14:47:33.090 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 33 M 2024-08-29T14:47:33.090 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:33.334 INFO:teuthology.orchestra.run.smithi171.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.7 MB/s | 248 kB 00:00 2024-08-29T14:47:33.393 INFO:teuthology.orchestra.run.smithi171.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 9.2 MB/s | 548 kB 00:00 2024-08-29T14:47:33.418 INFO:teuthology.orchestra.run.smithi171.stdout:(3/6): ceph-mgr-cephadm-18.2.4-774.gfe43f1c9.el 621 kB/s | 139 kB 00:00 2024-08-29T14:47:33.436 INFO:teuthology.orchestra.run.smithi171.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 1.3 MB/s | 58 kB 00:00 2024-08-29T14:47:33.478 INFO:teuthology.orchestra.run.smithi171.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.2 MB/s | 74 kB 00:00 2024-08-29T14:47:33.508 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:23 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:33.646 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-fuse x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 851 k 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:33.647 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:33.648 INFO:teuthology.orchestra.run.smithi027.stdout:Install 2 Packages 2024-08-29T14:47:33.648 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:33.648 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 931 k 2024-08-29T14:47:33.648 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 2.7 M 2024-08-29T14:47:33.648 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:33.850 INFO:teuthology.orchestra.run.smithi027.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 772 kB/s | 80 kB 00:00 2024-08-29T14:47:34.000 INFO:teuthology.orchestra.run.smithi027.stdout:(2/2): ceph-fuse-18.2.4-774.gfe43f1c9.el9.x86_6 3.3 MB/s | 851 kB 00:00 2024-08-29T14:47:34.001 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:34.001 INFO:teuthology.orchestra.run.smithi027.stdout:Total 2.6 MB/s | 931 kB 00:00 2024-08-29T14:47:34.013 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:34.035 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:34.035 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:34.120 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:34.120 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-29T14:47:34.526 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/2 2024-08-29T14:47:34.886 INFO:teuthology.orchestra.run.smithi171.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.5 MB/s | 6.0 MB 00:01 2024-08-29T14:47:34.889 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:34.890 INFO:teuthology.orchestra.run.smithi171.stdout:Total 3.9 MB/s | 7.0 MB 00:01 2024-08-29T14:47:34.965 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:34.977 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:34.977 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:35.076 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:35.077 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:35.182 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/2 2024-08-29T14:47:35.182 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/2 2024-08-29T14:47:35.379 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:35.445 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-29T14:47:35.445 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:35.445 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:35.446 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-29T14:47:35.446 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:35.446 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:35.558 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-29T14:47:35.621 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-29T14:47:35.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install librados-devel 2024-08-29T14:47:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-29T14:47:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-29T14:47:36.304 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-29T14:47:36.320 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:26 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:36.344 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 6/6 2024-08-29T14:47:36.457 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:36.458 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:36.458 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2024-08-29T14:47:36.458 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:36.458 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout: librados-devel x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 127 k 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 127 k 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 456 k 2024-08-29T14:47:36.459 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:36.672 INFO:teuthology.orchestra.run.smithi027.stdout:librados-devel-18.2.4-774.gfe43f1c9.el9.x86_64. 596 kB/s | 127 kB 00:00 2024-08-29T14:47:36.672 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:36.673 INFO:teuthology.orchestra.run.smithi027.stdout:Total 591 kB/s | 127 kB 00:00 2024-08-29T14:47:36.673 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:36.678 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:36.679 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:36.746 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:36.746 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:36.906 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:36.996 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:37.084 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 6/6 2024-08-29T14:47:37.084 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 1/6 2024-08-29T14:47:37.084 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-29T14:47:37.085 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-29T14:47:37.085 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-29T14:47:37.085 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-mgr-cephadm-2:18.2.4-774.gfe43f1c9.el9.noarch 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-29T14:47:37.351 INFO:teuthology.orchestra.run.smithi171.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-29T14:47:37.352 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:37.352 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:37.417 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:37.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install ceph-fuse 2024-08-29T14:47:37.658 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:37.658 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:37.658 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:37.658 INFO:teuthology.orchestra.run.smithi027.stdout: librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:37.658 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:37.658 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:37.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install libcephfs2 2024-08-29T14:47:38.189 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:21 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:38.321 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repository Size 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-fuse x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 851 k 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout:Installing dependencies: 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:38.322 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:38.323 INFO:teuthology.orchestra.run.smithi171.stdout:Install 2 Packages 2024-08-29T14:47:38.323 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:38.323 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 931 k 2024-08-29T14:47:38.323 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 2.7 M 2024-08-29T14:47:38.323 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:38.461 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:28 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:38.554 INFO:teuthology.orchestra.run.smithi027.stdout:Package libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:38.600 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:38.600 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:38.601 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:38.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install libcephfs-devel 2024-08-29T14:47:38.833 INFO:teuthology.orchestra.run.smithi171.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 355 kB/s | 80 kB 00:00 2024-08-29T14:47:39.058 INFO:teuthology.orchestra.run.smithi171.stdout:(2/2): ceph-fuse-18.2.4-774.gfe43f1c9.el9.x86_6 1.8 MB/s | 851 kB 00:00 2024-08-29T14:47:39.059 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:39.059 INFO:teuthology.orchestra.run.smithi171.stdout:Total 1.2 MB/s | 931 kB 00:00 2024-08-29T14:47:39.071 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:39.094 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:39.094 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:39.175 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:39.175 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:39.276 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:29 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:39.366 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:39.413 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:39.414 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:39.414 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2024-08-29T14:47:39.414 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs-devel x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 31 k 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 31 k 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 139 k 2024-08-29T14:47:39.415 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:39.508 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-29T14:47:39.573 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/2 2024-08-29T14:47:39.608 INFO:teuthology.orchestra.run.smithi027.stdout:libcephfs-devel-18.2.4-774.gfe43f1c9.el9.x86_64 161 kB/s | 31 kB 00:00 2024-08-29T14:47:39.609 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:39.609 INFO:teuthology.orchestra.run.smithi027.stdout:Total 160 kB/s | 31 kB 00:00 2024-08-29T14:47:39.609 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:39.613 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:39.613 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:39.617 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:39.618 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:39.709 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:39.831 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:40.180 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:40.262 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2/2 2024-08-29T14:47:40.262 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/2 2024-08-29T14:47:40.412 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:40.412 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:40.412 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:40.412 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:40.412 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:40.412 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:40.519 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-29T14:47:40.519 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:40.519 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:40.519 INFO:teuthology.orchestra.run.smithi171.stdout: ceph-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-29T14:47:40.519 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:40.519 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:40.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install librados2 2024-08-29T14:47:40.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install librados-devel 2024-08-29T14:47:41.170 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:31 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:41.263 INFO:teuthology.orchestra.run.smithi027.stdout:Package librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:41.308 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:41.309 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:41.309 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:41.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install librbd1 2024-08-29T14:47:41.376 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:24 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:41.513 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout: librados-devel x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 127 k 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-08-29T14:47:41.514 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:41.515 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 127 k 2024-08-29T14:47:41.515 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 456 k 2024-08-29T14:47:41.515 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:41.701 INFO:teuthology.orchestra.run.smithi171.stdout:librados-devel-18.2.4-774.gfe43f1c9.el9.x86_64. 684 kB/s | 127 kB 00:00 2024-08-29T14:47:41.701 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:41.702 INFO:teuthology.orchestra.run.smithi171.stdout:Total 678 kB/s | 127 kB 00:00 2024-08-29T14:47:41.702 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:41.709 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:41.709 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:41.773 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:41.773 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:41.927 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:41.951 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:31 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:42.022 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:42.044 INFO:teuthology.orchestra.run.smithi027.stdout:Package librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:42.090 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:42.091 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:42.091 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:42.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-rados 2024-08-29T14:47:42.397 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:42.668 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: librados-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:42.668 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:42.726 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:32 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:42.819 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:42.864 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:42.865 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:42.865 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:42.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-rgw 2024-08-29T14:47:42.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install libcephfs2 2024-08-29T14:47:43.507 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:33 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:43.517 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:26 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:43.600 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:43.620 INFO:teuthology.orchestra.run.smithi171.stdout:Package libcephfs2-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:43.645 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:43.646 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:43.646 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:43.665 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:43.666 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:43.666 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:43.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-cephfs 2024-08-29T14:47:43.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install libcephfs-devel 2024-08-29T14:47:44.291 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:34 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:44.300 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:27 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:44.384 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:44.429 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:44.430 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:44.430 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:44.432 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:44.433 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:44.433 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-08-29T14:47:44.433 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:44.433 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:44.433 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs-devel x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 31 k 2024-08-29T14:47:44.433 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:44.433 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:44.434 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:44.434 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-08-29T14:47:44.434 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:44.434 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 31 k 2024-08-29T14:47:44.434 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 139 k 2024-08-29T14:47:44.434 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:44.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-rbd 2024-08-29T14:47:44.583 INFO:teuthology.orchestra.run.smithi171.stdout:libcephfs-devel-18.2.4-774.gfe43f1c9.el9.x86_64 209 kB/s | 31 kB 00:00 2024-08-29T14:47:44.584 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:44.584 INFO:teuthology.orchestra.run.smithi171.stdout:Total 207 kB/s | 31 kB 00:00 2024-08-29T14:47:44.584 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:44.588 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:44.588 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:44.592 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:44.592 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:44.689 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:44.795 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:45.079 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:35 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:45.145 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:45.171 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:45.216 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:45.217 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2024-08-29T14:47:45.218 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:45.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install rbd-fuse 2024-08-29T14:47:45.365 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:45.365 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:45.365 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:45.365 INFO:teuthology.orchestra.run.smithi171.stdout: libcephfs-devel-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:45.365 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:45.365 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:45.584 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install librados2 2024-08-29T14:47:45.860 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:35 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:45.998 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:45.999 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:45.999 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-08-29T14:47:45.999 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:45.999 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-fuse x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 87 k 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 87 k 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 227 k 2024-08-29T14:47:46.000 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:46.142 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:29 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:46.196 INFO:teuthology.orchestra.run.smithi027.stdout:rbd-fuse-18.2.4-774.gfe43f1c9.el9.x86_64.rpm 443 kB/s | 87 kB 00:00 2024-08-29T14:47:46.196 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:46.196 INFO:teuthology.orchestra.run.smithi027.stdout:Total 440 kB/s | 87 kB 00:00 2024-08-29T14:47:46.197 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:46.204 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:46.204 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:46.231 INFO:teuthology.orchestra.run.smithi171.stdout:Package librados2-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:46.271 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:46.271 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:46.273 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:46.275 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:46.275 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:46.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install librbd1 2024-08-29T14:47:46.416 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:46.510 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:46.867 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:46.910 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:29 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:47.000 INFO:teuthology.orchestra.run.smithi171.stdout:Package librbd1-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:47.042 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:47.043 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:47.043 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:47.102 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:47.103 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:47.103 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:47.103 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:47.103 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:47.103 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:47.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-rados 2024-08-29T14:47:47.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install rbd-mirror 2024-08-29T14:47:47.683 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:30 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:47.772 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-rados-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:47.815 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:47.816 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:47.816 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:47.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-rgw 2024-08-29T14:47:47.925 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:37 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:48.065 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:48.065 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:48.065 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2024-08-29T14:47:48.065 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:48.065 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-mirror x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 3.0 M 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 3.0 M 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 12 M 2024-08-29T14:47:48.066 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:48.448 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:31 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:48.507 INFO:teuthology.orchestra.run.smithi027.stdout:rbd-mirror-18.2.4-774.gfe43f1c9.el9.x86_64.rpm 6.8 MB/s | 3.0 MB 00:00 2024-08-29T14:47:48.507 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:48.508 INFO:teuthology.orchestra.run.smithi027.stdout:Total 6.7 MB/s | 3.0 MB 00:00 2024-08-29T14:47:48.508 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:48.518 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:48.518 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:48.537 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-rgw-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:48.579 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:48.580 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:48.580 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:48.604 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:48.604 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:48.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-cephfs 2024-08-29T14:47:49.047 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:49.099 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:49.126 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:49.126 INFO:teuthology.orchestra.run.smithi027.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:47:49.126 INFO:teuthology.orchestra.run.smithi027.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-29T14:47:49.126 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-29T14:47:49.126 INFO:teuthology.orchestra.run.smithi027.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-29T14:47:49.126 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:49.211 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:32 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:49.302 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-cephfs-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:49.344 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:49.345 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:49.345 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:49.424 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install python3-rbd 2024-08-29T14:47:49.984 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:32 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:50.074 INFO:teuthology.orchestra.run.smithi171.stdout:Package python3-rbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 is already installed. 2024-08-29T14:47:50.117 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:50.117 INFO:teuthology.orchestra.run.smithi171.stdout:Nothing to do. 2024-08-29T14:47:50.118 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:50.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install rbd-fuse 2024-08-29T14:47:50.227 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:50.227 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:50.227 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:50.228 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:50.228 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:50.228 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:50.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install rbd-nbd 2024-08-29T14:47:50.761 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:33 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:50.894 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:50.895 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:50.895 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-08-29T14:47:50.895 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:50.895 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:50.895 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-fuse x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 87 k 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 87 k 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 227 k 2024-08-29T14:47:50.896 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:51.045 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:41 ago on Thu 29 Aug 2024 02:46:10 PM UTC. 2024-08-29T14:47:51.104 INFO:teuthology.orchestra.run.smithi171.stdout:rbd-fuse-18.2.4-774.gfe43f1c9.el9.x86_64.rpm 416 kB/s | 87 kB 00:00 2024-08-29T14:47:51.105 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:51.105 INFO:teuthology.orchestra.run.smithi171.stdout:Total 413 kB/s | 87 kB 00:00 2024-08-29T14:47:51.105 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:51.112 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:51.112 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:51.174 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:51.175 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:51.183 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-08-29T14:47:51.184 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:51.184 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-08-29T14:47:51.184 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:51.184 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-08-29T14:47:51.184 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-nbd x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 172 k 2024-08-29T14:47:51.185 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:51.185 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-08-29T14:47:51.185 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-08-29T14:47:51.185 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2024-08-29T14:47:51.185 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:51.185 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 172 k 2024-08-29T14:47:51.185 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 494 k 2024-08-29T14:47:51.186 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-08-29T14:47:51.317 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:51.410 INFO:teuthology.orchestra.run.smithi027.stdout:rbd-nbd-18.2.4-774.gfe43f1c9.el9.x86_64.rpm 769 kB/s | 172 kB 00:00 2024-08-29T14:47:51.410 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:51.411 INFO:teuthology.orchestra.run.smithi027.stdout:Total 762 kB/s | 172 kB 00:00 2024-08-29T14:47:51.411 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-08-29T14:47:51.419 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-08-29T14:47:51.419 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-08-29T14:47:51.424 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:51.487 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-08-29T14:47:51.487 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-08-29T14:47:51.651 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-08-29T14:47:51.750 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:51.759 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:52.023 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:52.023 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:52.024 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:52.024 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-fuse-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:52.024 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:52.024 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:52.131 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:52.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install rbd-mirror 2024-08-29T14:47:52.386 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:52.386 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:52.386 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-08-29T14:47:52.386 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:52.386 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:47:52.386 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-08-29T14:47:52.625 DEBUG:teuthology.parallel:result is None 2024-08-29T14:47:52.824 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:35 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:52.957 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:52.958 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:52.958 INFO:teuthology.orchestra.run.smithi171.stdout: Package Arch Version Repo Size 2024-08-29T14:47:52.958 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:52.958 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-mirror x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 3.0 M 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 3.0 M 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 12 M 2024-08-29T14:47:52.959 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:53.427 INFO:teuthology.orchestra.run.smithi171.stdout:rbd-mirror-18.2.4-774.gfe43f1c9.el9.x86_64.rpm 6.4 MB/s | 3.0 MB 00:00 2024-08-29T14:47:53.428 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:53.429 INFO:teuthology.orchestra.run.smithi171.stdout:Total 6.4 MB/s | 3.0 MB 00:00 2024-08-29T14:47:53.429 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:53.437 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:53.438 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:53.518 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:53.518 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:53.943 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:54.037 INFO:teuthology.orchestra.run.smithi171.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-29T14:47:54.037 INFO:teuthology.orchestra.run.smithi171.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-29T14:47:54.037 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-29T14:47:54.037 INFO:teuthology.orchestra.run.smithi171.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-29T14:47:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:55.073 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:55.073 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:55.073 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:55.073 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-mirror-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:55.073 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:55.073 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:55.357 DEBUG:teuthology.orchestra.run.smithi171:> sudo yum -y install rbd-nbd 2024-08-29T14:47:55.936 INFO:teuthology.orchestra.run.smithi171.stdout:Last metadata expiration check: 0:01:38 ago on Thu 29 Aug 2024 02:46:17 PM UTC. 2024-08-29T14:47:56.069 INFO:teuthology.orchestra.run.smithi171.stdout:Dependencies resolved. 2024-08-29T14:47:56.070 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:56.070 INFO:teuthology.orchestra.run.smithi171.stdout: Package Architecture Version Repository Size 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:Installing: 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-nbd x86_64 2:18.2.4-774.gfe43f1c9.el9 ceph 172 k 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction Summary 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:================================================================================ 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:Install 1 Package 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:Total download size: 172 k 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:Installed size: 494 k 2024-08-29T14:47:56.071 INFO:teuthology.orchestra.run.smithi171.stdout:Downloading Packages: 2024-08-29T14:47:56.293 INFO:teuthology.orchestra.run.smithi171.stdout:rbd-nbd-18.2.4-774.gfe43f1c9.el9.x86_64.rpm 776 kB/s | 172 kB 00:00 2024-08-29T14:47:56.294 INFO:teuthology.orchestra.run.smithi171.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:47:56.294 INFO:teuthology.orchestra.run.smithi171.stdout:Total 770 kB/s | 172 kB 00:00 2024-08-29T14:47:56.294 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction check 2024-08-29T14:47:56.301 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction check succeeded. 2024-08-29T14:47:56.302 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction test 2024-08-29T14:47:56.366 INFO:teuthology.orchestra.run.smithi171.stdout:Transaction test succeeded. 2024-08-29T14:47:56.366 INFO:teuthology.orchestra.run.smithi171.stdout:Running transaction 2024-08-29T14:47:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: Preparing : 1/1 2024-08-29T14:47:56.605 INFO:teuthology.orchestra.run.smithi171.stdout: Installing : rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:57.010 INFO:teuthology.orchestra.run.smithi171.stdout: Running scriptlet: rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:57.242 INFO:teuthology.orchestra.run.smithi171.stdout: Verifying : rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 1/1 2024-08-29T14:47:57.242 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:57.242 INFO:teuthology.orchestra.run.smithi171.stdout:Installed: 2024-08-29T14:47:57.242 INFO:teuthology.orchestra.run.smithi171.stdout: rbd-nbd-2:18.2.4-774.gfe43f1c9.el9.x86_64 2024-08-29T14:47:57.242 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:47:57.242 INFO:teuthology.orchestra.run.smithi171.stdout:Complete! 2024-08-29T14:47:57.515 DEBUG:teuthology.parallel:result is None 2024-08-29T14:47:57.516 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-29T14:47:57.516 INFO:teuthology.packaging:ref: None 2024-08-29T14:47:57.516 INFO:teuthology.packaging:tag: None 2024-08-29T14:47:57.516 INFO:teuthology.packaging:branch: reef 2024-08-29T14:47:57.516 INFO:teuthology.packaging:sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:47:57.517 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-29T14:47:57.798 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-29T14:47:57.824 INFO:teuthology.orchestra.run.smithi027.stdout:18.2.4-774.gfe43f1c9.el9 2024-08-29T14:47:57.825 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-774.gfe43f1c9.el9 2024-08-29T14:47:57.825 INFO:teuthology.task.install:The correct ceph version 18.2.4-774.gfe43f1c9 is installed. 2024-08-29T14:47:57.827 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-29T14:47:57.827 INFO:teuthology.packaging:ref: None 2024-08-29T14:47:57.827 INFO:teuthology.packaging:tag: None 2024-08-29T14:47:57.827 INFO:teuthology.packaging:branch: reef 2024-08-29T14:47:57.827 INFO:teuthology.packaging:sha1: 1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:47:57.827 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-29T14:47:58.064 DEBUG:teuthology.orchestra.run.smithi171:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-29T14:47:58.086 INFO:teuthology.orchestra.run.smithi171.stdout:18.2.4-774.gfe43f1c9.el9 2024-08-29T14:47:58.087 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-774.gfe43f1c9.el9 2024-08-29T14:47:58.087 INFO:teuthology.task.install:The correct ceph version 18.2.4-774.gfe43f1c9 is installed. 2024-08-29T14:47:58.089 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-08-29T14:47:58.089 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:47:58.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-29T14:47:58.121 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:47:58.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-29T14:47:58.152 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-08-29T14:47:58.153 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:47:58.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/usr/bin/daemon-helper 2024-08-29T14:47:58.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-29T14:47:58.258 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:47:58.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/daemon-helper 2024-08-29T14:47:58.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-29T14:47:58.360 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-08-29T14:47:58.360 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:47:58.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-29T14:47:58.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-29T14:47:58.455 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:47:58.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-29T14:47:58.481 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-29T14:47:58.546 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-08-29T14:47:58.546 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:47:58.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/usr/bin/stdin-killer 2024-08-29T14:47:58.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-29T14:47:58.638 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:47:58.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/usr/bin/stdin-killer 2024-08-29T14:47:58.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-29T14:47:58.728 INFO:teuthology.run_tasks:Running task cephadm... 2024-08-29T14:47:58.836 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791'} 2024-08-29T14:47:58.836 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-08-29T14:47:58.837 INFO:tasks.cephadm:Cluster fsid is af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:47:58.837 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-08-29T14:47:58.837 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.27', 'mon.c': '[v2:172.21.15.27:3301,v1:172.21.15.27:6790]', 'mon.b': '172.21.15.171'} 2024-08-29T14:47:58.837 INFO:tasks.cephadm:First mon is mon.a on smithi027 2024-08-29T14:47:58.837 INFO:tasks.cephadm:First mgr is y 2024-08-29T14:47:58.837 INFO:tasks.cephadm:Normalizing hostnames... 2024-08-29T14:47:58.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo hostname $(hostname -s) 2024-08-29T14:47:58.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo hostname $(hostname -s) 2024-08-29T14:47:58.889 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-08-29T14:47:58.890 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:47:59.069 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/82602/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.3.0-4579-g1f4056d9', 'node_name': '172.21.5.40+adami10', 'job_name': 'ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.3.0-4579.g1f4056d9'}, 'url': 'https://4.chacra.ceph.com/r/ceph/wip-yuri4-testing-2024-08-28-1359/1f4056d968930e12926c25a8f13ee8ed6047c791/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-08-28 22:37:40.250681', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-yuri4-testing-2024-08-28-1359', 'chacra_url': 'https://4.chacra.ceph.com/repos/ceph/wip-yuri4-testing-2024-08-28-1359/1f4056d968930e12926c25a8f13ee8ed6047c791/centos/9/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2024-08-29T14:47:59.345 INFO:tasks.util.chacra:got chacra host 3.chacra.ceph.com, ref reef, sha1 fe43f1c9f9e2945dda7244d8acbd33fb921449af from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-08-29T14:47:59.347 INFO:tasks.cephadm:Discovered cachra url: https://3.chacra.ceph.com/binaries/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/x86_64/flavors/default/cephadm 2024-08-29T14:47:59.347 INFO:tasks.cephadm:Downloading cephadm from url: https://3.chacra.ceph.com/binaries/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/x86_64/flavors/default/cephadm 2024-08-29T14:47:59.347 DEBUG:teuthology.orchestra.run.smithi027:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-29T14:47:59.701 INFO:teuthology.orchestra.run.smithi027.stdout:-rw-r--r--. 1 ubuntu ubuntu 215529 Aug 29 14:47 /home/ubuntu/cephtest/cephadm 2024-08-29T14:47:59.701 DEBUG:teuthology.orchestra.run.smithi171:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/fe43f1c9f9e2945dda7244d8acbd33fb921449af/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-29T14:48:00.061 INFO:teuthology.orchestra.run.smithi171.stdout:-rw-r--r--. 1 ubuntu ubuntu 215529 Aug 29 14:48 /home/ubuntu/cephtest/cephadm 2024-08-29T14:48:00.062 DEBUG:teuthology.orchestra.run.smithi027:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-29T14:48:00.079 DEBUG:teuthology.orchestra.run.smithi171:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-29T14:48:00.536 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-08-29T14:48:00.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-08-29T14:48:00.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-08-29T14:48:02.045 INFO:teuthology.orchestra.run.smithi171.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-29T14:48:02.056 INFO:teuthology.orchestra.run.smithi027.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-29T14:48:33.966 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:48:33.966 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph_version": "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)", 2024-08-29T14:48:33.966 INFO:teuthology.orchestra.run.smithi027.stdout: "image_id": "96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892", 2024-08-29T14:48:33.966 INFO:teuthology.orchestra.run.smithi027.stdout: "repo_digests": [ 2024-08-29T14:48:33.967 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee" 2024-08-29T14:48:33.967 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T14:48:33.967 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:48:35.380 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T14:48:35.380 INFO:teuthology.orchestra.run.smithi171.stdout: "ceph_version": "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)", 2024-08-29T14:48:35.380 INFO:teuthology.orchestra.run.smithi171.stdout: "image_id": "96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892", 2024-08-29T14:48:35.380 INFO:teuthology.orchestra.run.smithi171.stdout: "repo_digests": [ 2024-08-29T14:48:35.380 INFO:teuthology.orchestra.run.smithi171.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee" 2024-08-29T14:48:35.380 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T14:48:35.381 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T14:48:35.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo mkdir -p /etc/ceph 2024-08-29T14:48:35.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo mkdir -p /etc/ceph 2024-08-29T14:48:35.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 777 /etc/ceph 2024-08-29T14:48:35.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 777 /etc/ceph 2024-08-29T14:48:35.531 INFO:tasks.cephadm:Writing seed config... 2024-08-29T14:48:35.532 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-08-29T14:48:35.532 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-08-29T14:48:35.532 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-08-29T14:48:35.533 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-08-29T14:48:35.534 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:48:35.534 DEBUG:teuthology.orchestra.run.smithi027:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-08-29T14:48:35.565 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = af03926c-6615-11ef-bcd4-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-08-29T14:48:35.565 DEBUG:teuthology.orchestra.run.smithi027:mon.a> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service 2024-08-29T14:48:35.608 DEBUG:teuthology.orchestra.run.smithi027:mgr.y> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service 2024-08-29T14:48:35.651 INFO:tasks.cephadm:Bootstrapping... 2024-08-29T14:48:35.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid af03926c-6615-11ef-bcd4-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.27 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-08-29T14:48:35.817 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-08-29T14:48:35.817 INFO:teuthology.orchestra.run.smithi027.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', 'af03926c-6615-11ef-bcd4-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.27', '--skip-admin-label'] 2024-08-29T14:48:35.854 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stdout 5.2.2 2024-08-29T14:48:35.854 INFO:teuthology.orchestra.run.smithi027.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-08-29T14:48:35.855 INFO:teuthology.orchestra.run.smithi027.stdout:Verifying podman|docker is present... 2024-08-29T14:48:35.877 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stdout 5.2.2 2024-08-29T14:48:35.877 INFO:teuthology.orchestra.run.smithi027.stdout:Verifying lvm2 is present... 2024-08-29T14:48:35.877 INFO:teuthology.orchestra.run.smithi027.stdout:Verifying time synchronization is in place... 2024-08-29T14:48:35.885 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-29T14:48:35.885 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-29T14:48:35.891 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-29T14:48:35.891 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout inactive 2024-08-29T14:48:35.898 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout enabled 2024-08-29T14:48:35.905 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout active 2024-08-29T14:48:35.905 INFO:teuthology.orchestra.run.smithi027.stdout:Unit chronyd.service is enabled and running 2024-08-29T14:48:35.905 INFO:teuthology.orchestra.run.smithi027.stdout:Repeating the final host check... 2024-08-29T14:48:35.935 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stdout 5.2.2 2024-08-29T14:48:35.935 INFO:teuthology.orchestra.run.smithi027.stdout:podman (/bin/podman) version 5.2.2 is present 2024-08-29T14:48:35.935 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl is present 2024-08-29T14:48:35.935 INFO:teuthology.orchestra.run.smithi027.stdout:lvcreate is present 2024-08-29T14:48:35.941 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-29T14:48:35.941 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-29T14:48:35.948 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-29T14:48:35.948 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout inactive 2024-08-29T14:48:35.955 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout enabled 2024-08-29T14:48:35.961 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout active 2024-08-29T14:48:35.962 INFO:teuthology.orchestra.run.smithi027.stdout:Unit chronyd.service is enabled and running 2024-08-29T14:48:35.962 INFO:teuthology.orchestra.run.smithi027.stdout:Host looks OK 2024-08-29T14:48:35.962 INFO:teuthology.orchestra.run.smithi027.stdout:Cluster fsid: af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:35.962 INFO:teuthology.orchestra.run.smithi027.stdout:Acquiring lock 140087473108496 on /run/cephadm/af03926c-6615-11ef-bcd4-c7b262605968.lock 2024-08-29T14:48:35.962 INFO:teuthology.orchestra.run.smithi027.stdout:Lock 140087473108496 acquired on /run/cephadm/af03926c-6615-11ef-bcd4-c7b262605968.lock 2024-08-29T14:48:35.962 INFO:teuthology.orchestra.run.smithi027.stdout:Verifying IP 172.21.15.27 port 3300 ... 2024-08-29T14:48:35.963 INFO:teuthology.orchestra.run.smithi027.stdout:Verifying IP 172.21.15.27 port 6789 ... 2024-08-29T14:48:35.963 INFO:teuthology.orchestra.run.smithi027.stdout:Base mon IP(s) is [172.21.15.27:3300, 172.21.15.27:6789], mon addrv is [v2:172.21.15.27:3300,v1:172.21.15.27:6789] 2024-08-29T14:48:35.966 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.27 metric 100 2024-08-29T14:48:35.966 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.27 metric 100 2024-08-29T14:48:35.968 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-08-29T14:48:35.968 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-08-29T14:48:35.968 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-08-29T14:48:35.971 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-08-29T14:48:35.971 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-08-29T14:48:35.971 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-29T14:48:35.971 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout 3: ens1f0: mtu 1500 state UP qlen 1000 2024-08-29T14:48:35.971 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout inet6 fe80::25ac:4e2a:115e:3df3/64 scope link noprefixroute 2024-08-29T14:48:35.971 INFO:teuthology.orchestra.run.smithi027.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-29T14:48:35.972 INFO:teuthology.orchestra.run.smithi027.stdout:Mon IP `172.21.15.27` is in CIDR network `172.21.0.0/20` 2024-08-29T14:48:35.972 INFO:teuthology.orchestra.run.smithi027.stdout:Mon IP `172.21.15.27` is in CIDR network `172.21.0.0/20` 2024-08-29T14:48:35.972 INFO:teuthology.orchestra.run.smithi027.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-08-29T14:48:35.972 INFO:teuthology.orchestra.run.smithi027.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-08-29T14:48:35.973 INFO:teuthology.orchestra.run.smithi027.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-29T14:48:36.467 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stdout 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 2024-08-29T14:48:36.468 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-08-29T14:48:36.468 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stderr Getting image source signatures 2024-08-29T14:48:36.468 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stderr Copying blob sha256:452a9a32cc731716b741f78571d7f7e1f52bf766794e9a5a40c0f18af70e0682 2024-08-29T14:48:36.468 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-08-29T14:48:36.468 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stderr Copying config sha256:96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 2024-08-29T14:48:36.468 INFO:teuthology.orchestra.run.smithi027.stdout:/bin/podman: stderr Writing manifest to image destination 2024-08-29T14:48:37.566 INFO:teuthology.orchestra.run.smithi027.stdout:ceph: stdout ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable) 2024-08-29T14:48:37.566 INFO:teuthology.orchestra.run.smithi027.stdout:Ceph version: ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable) 2024-08-29T14:48:37.566 INFO:teuthology.orchestra.run.smithi027.stdout:Extracting ceph user uid/gid from container image... 2024-08-29T14:48:38.537 INFO:teuthology.orchestra.run.smithi027.stdout:stat: stdout 167 167 2024-08-29T14:48:38.538 INFO:teuthology.orchestra.run.smithi027.stdout:Creating initial keys... 2024-08-29T14:48:39.417 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph-authtool: stdout AQBGitBmdmbsMhAAKmzTGWIV1pIHFvJy7aw+zA== 2024-08-29T14:48:40.458 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph-authtool: stdout AQBHitBmFORWLRAARnmLCrj95C/jM0sWnjvaZA== 2024-08-29T14:48:41.354 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph-authtool: stdout AQBIitBm2fMmLhAA7PjIYV5JaBdDR2EtSZUkwg== 2024-08-29T14:48:41.355 INFO:teuthology.orchestra.run.smithi027.stdout:Creating initial monmap... 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:monmaptool for a [v2:172.21.15.27:3300,v1:172.21.15.27:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:setting min_mon_release = pacific 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/monmaptool: set fsid to af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:48:42.286 INFO:teuthology.orchestra.run.smithi027.stdout:Creating mon... 2024-08-29T14:48:43.167 INFO:teuthology.orchestra.run.smithi027.stdout:create mon.a on 2024-08-29T14:48:43.382 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-08-29T14:48:43.530 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-08-29T14:48:43.714 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-af03926c-6615-11ef-bcd4-c7b262605968.target → /etc/systemd/system/ceph-af03926c-6615-11ef-bcd4-c7b262605968.target. 2024-08-29T14:48:43.714 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-af03926c-6615-11ef-bcd4-c7b262605968.target → /etc/systemd/system/ceph-af03926c-6615-11ef-bcd4-c7b262605968.target. 2024-08-29T14:48:43.935 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a 2024-08-29T14:48:43.935 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Failed to reset failed state of unit ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service: Unit ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service not loaded. 2024-08-29T14:48:44.110 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-af03926c-6615-11ef-bcd4-c7b262605968.target.wants/ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service → /etc/systemd/system/ceph-af03926c-6615-11ef-bcd4-c7b262605968@.service. 2024-08-29T14:48:44.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:44 smithi027 podman[18494]: 2024-08-29 14:48:44.588582306 +0000 UTC m=+0.358039126 container start cee5a73e2bc7ef260f54c5f3451c9c2661a7ec580b9dcf20f55fe4698de246b1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.1, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T14:48:44.701 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-29T14:48:44.701 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout disabled 2024-08-29T14:48:44.709 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-29T14:48:44.710 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout inactive 2024-08-29T14:48:44.710 INFO:teuthology.orchestra.run.smithi027.stdout:firewalld.service is not enabled 2024-08-29T14:48:44.710 INFO:teuthology.orchestra.run.smithi027.stdout:Not possible to enable service . firewalld.service is not available 2024-08-29T14:48:44.710 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for mon to start... 2024-08-29T14:48:44.710 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for mon... 2024-08-29T14:48:44.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:44 smithi027 ceph-mon[18520]: mkfs af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:44.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:44 smithi027 ceph-mon[18520]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-29T14:48:44.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:44 smithi027 bash[18494]: cee5a73e2bc7ef260f54c5f3451c9c2661a7ec580b9dcf20f55fe4698de246b1 2024-08-29T14:48:44.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:44 smithi027 systemd[1]: Started Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:48:45.712 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout cluster: 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout id: af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout services: 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.56412s) 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout data: 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-08-29T14:48:45.713 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-08-29T14:48:45.714 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout pgs: 2024-08-29T14:48:45.714 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:45.714 INFO:teuthology.orchestra.run.smithi027.stdout:mon is available 2024-08-29T14:48:45.714 INFO:teuthology.orchestra.run.smithi027.stdout:Assimilating anything we can from ceph.conf... 2024-08-29T14:48:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:45 smithi027 ceph-mon[18520]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-29T14:48:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:45 smithi027 ceph-mon[18520]: monmap e1: 1 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:48:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:45 smithi027 ceph-mon[18520]: fsmap 2024-08-29T14:48:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:45 smithi027 ceph-mon[18520]: osdmap e1: 0 total, 0 up, 0 in 2024-08-29T14:48:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:45 smithi027 ceph-mon[18520]: mgrmap e1: no daemons active 2024-08-29T14:48:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:45 smithi027 ceph-mon[18520]: from='client.? 172.21.15.27:0/2119575688' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-29T14:48:46.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:46 smithi027 ceph-mon[18520]: from='client.? 172.21.15.27:0/2064923232' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-29T14:48:46.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:46 smithi027 ceph-mon[18520]: from='client.? 172.21.15.27:0/2064923232' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout [global] 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout fsid = af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.27:3300,v1:172.21.15.27:6789] 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:46.736 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-29T14:48:46.737 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-29T14:48:46.737 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:46.737 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout [osd] 2024-08-29T14:48:46.737 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-29T14:48:46.737 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-29T14:48:46.737 INFO:teuthology.orchestra.run.smithi027.stdout:Generating new minimal ceph.conf... 2024-08-29T14:48:47.708 INFO:teuthology.orchestra.run.smithi027.stdout:Restarting the monitor... 2024-08-29T14:48:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:47 smithi027 ceph-mon[18520]: from='client.? 172.21.15.27:0/393975964' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:48:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:47 smithi027 systemd[1]: Stopping Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:48:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[18508]: 2024-08-29T14:48:47.991+0000 7f4d97e5a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T14:48:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[18508]: 2024-08-29T14:48:47.991+0000 7f4d97e5a640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-08-29T14:48:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:48 smithi027 podman[18768]: 2024-08-29 14:48:48.155248427 +0000 UTC m=+0.274601605 container died cee5a73e2bc7ef260f54c5f3451c9c2661a7ec580b9dcf20f55fe4698de246b1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-fe43f1c, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-29T14:48:48.667 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:48 smithi027 podman[18768]: 2024-08-29 14:48:48.482624773 +0000 UTC m=+0.601978016 container cleanup cee5a73e2bc7ef260f54c5f3451c9c2661a7ec580b9dcf20f55fe4698de246b1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-fe43f1c, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-29T14:48:48.667 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:48 smithi027 bash[18768]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a 2024-08-29T14:48:48.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:48 smithi027 podman[18781]: 2024-08-29 14:48:48.666627807 +0000 UTC m=+0.506692222 container remove cee5a73e2bc7ef260f54c5f3451c9c2661a7ec580b9dcf20f55fe4698de246b1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-08-29T14:48:48.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:48 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-29T14:48:48.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:48 smithi027 systemd[1]: Stopped Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:48:48.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:48 smithi027 systemd[1]: Starting Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:48:49.386 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 podman[18873]: 2024-08-29 14:48:49.036430475 +0000 UTC m=+0.015833534 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph:reef 2024-08-29T14:48:49.387 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 podman[18873]: 2024-08-29 14:48:49.142617785 +0000 UTC m=+0.122020870 container create 517d2f5042c9097eb28678c5173890bb128f85ae39c7122b89803201d0e12591 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, org.label-schema.vendor=CentOS) 2024-08-29T14:48:49.387 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 podman[18873]: 2024-08-29 14:48:49.361410171 +0000 UTC m=+0.340813215 container init 517d2f5042c9097eb28678c5173890bb128f85ae39c7122b89803201d0e12591 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c) 2024-08-29T14:48:49.387 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 podman[18873]: 2024-08-29 14:48:49.364643418 +0000 UTC m=+0.344046462 container start 517d2f5042c9097eb28678c5173890bb128f85ae39c7122b89803201d0e12591 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T14:48:49.449 INFO:teuthology.orchestra.run.smithi027.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: set uid:gid to 167:167 (ceph:ceph) 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable), process ceph-mon, pid 2 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: pidfile_write: ignore empty --pid-file 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: load: jerasure load: lrc 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: RocksDB version: 7.9.2 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Git sha 0 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Compile date 2024-08-26 17:15:55 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: DB SUMMARY 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: DB Session ID: VL0343H06AZ12BXN2PX8 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: CURRENT file: CURRENT 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: IDENTITY file: IDENTITY 2024-08-29T14:48:49.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84544 ; 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.error_if_exists: 0 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.create_if_missing: 0 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.paranoid_checks: 1 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.env: 0x5607f46f9bc0 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.fs: PosixFileSystem 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.info_log: 0x5607f6ca2be0 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_file_opening_threads: 16 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.statistics: (nil) 2024-08-29T14:48:49.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.use_fsync: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_log_file_size: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.keep_log_file_num: 1000 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.recycle_log_file_num: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.allow_fallocate: 1 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.allow_mmap_reads: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.allow_mmap_writes: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.use_direct_reads: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.create_missing_column_families: 0 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.db_log_dir: 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.wal_dir: 2024-08-29T14:48:49.641 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.advise_random_on_open: 1 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.db_write_buffer_size: 0 2024-08-29T14:48:49.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.write_buffer_manager: 0x5607f6cb0320 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.rate_limiter: (nil) 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.wal_recovery_mode: 2 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.enable_thread_tracking: 0 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.enable_pipelined_write: 0 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.unordered_write: 0 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-29T14:48:49.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.row_cache: None 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.wal_filter: None 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.allow_ingest_behind: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.two_write_queues: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.manual_wal_flush: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.wal_compression: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.atomic_flush: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.log_readahead_size: 0 2024-08-29T14:48:49.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.best_efforts_recovery: 0 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.allow_data_in_errors: 0 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.db_host_id: __hostname__ 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_background_jobs: 2 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_background_compactions: -1 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_subcompactions: 1 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_total_wal_size: 0 2024-08-29T14:48:49.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_open_files: -1 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bytes_per_sync: 0 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_readahead_size: 0 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_background_flushes: -1 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Compression algorithms supported: 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kZSTD supported: 0 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kXpressCompression supported: 0 2024-08-29T14:48:49.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kBZip2Compression supported: 0 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kLZ4Compression supported: 1 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kZlibCompression supported: 1 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kLZ4HCCompression supported: 1 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: kSnappyCompression supported: 1 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.merge_operator: 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_filter: None 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_filter_factory: None 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.sst_partitioner_factory: None 2024-08-29T14:48:49.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5607f6ca27e0) 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_type: 0 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_index_type: 0 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_shortening: 1 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: checksum: 4 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: no_block_cache: 0 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache: 0x5607f6c97350 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-08-29T14:48:49.649 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_options: 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: capacity : 536870912 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_shard_bits : 4 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: strict_capacity_limit : 0 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_compressed: (nil) 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: persistent_cache: (nil) 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size: 4096 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size_deviation: 10 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_restart_interval: 16 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_block_restart_interval: 1 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: metadata_block_size: 4096 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: partition_filters: 0 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: use_delta_encoding: 1 2024-08-29T14:48:49.650 INFO:journalctl@ceph.mon.a.smithi027.stdout: filter_policy: bloomfilter 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: whole_key_filtering: 1 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: verify_compression: 0 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: format_version: 5 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_index_compression: 1 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_align: 0 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: max_auto_readahead_size: 262144 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: prepopulate_block_cache: 0 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: initial_auto_readahead_size: 8192 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.write_buffer_size: 33554432 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_write_buffer_number: 2 2024-08-29T14:48:49.651 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression: NoCompression 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression: Disabled 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.prefix_extractor: nullptr 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.num_levels: 7 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-29T14:48:49.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.level: 32767 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.strategy: 0 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.enabled: false 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-29T14:48:49.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.target_file_size_base: 67108864 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-29T14:48:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.arena_block_size: 1048576 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.disable_auto_compactions: 0 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-29T14:48:49.655 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.inplace_update_support: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.bloom_locality: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.max_successive_merges: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.paranoid_file_checks: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.force_consistency_checks: 1 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.report_bg_io_stats: 0 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.ttl: 2592000 2024-08-29T14:48:49.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.enable_blob_files: false 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.min_blob_size: 0 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.blob_file_size: 268435456 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.blob_file_starting_level: 0 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-29T14:48:49.657 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 20dc9dcb-0469-47e4-947b-3d131c636d86 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724942929391531, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724942929392416, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81094, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79244, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73445, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1724942929, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "20dc9dcb-0469-47e4-947b-3d131c636d86", "db_session_id": "VL0343H06AZ12BXN2PX8", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724942929392513, "job": 1, "event": "recovery_finished"} 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5607f6cc2e00 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: DB pointer 0x5607f6db6000 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** DB Stats ** 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T14:48:49.658 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: L0 2/0 81.05 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Sum 2/0 81.05 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 140.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T14:48:49.659 INFO:journalctl@ceph.mon.a.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 140.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative compaction: 0.00 GB write, 13.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval compaction: 0.00 GB write, 13.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T14:48:49.660 INFO:journalctl@ceph.mon.a.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache BinnedLRUCache@0x5607f6c97350#2 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: starting mon.a rank 0 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???) e1 preinit fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???).mds e1 new map 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???).mds e1 print_map 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: e1 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: legacy client fscid: -1 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T14:48:49.661 INFO:journalctl@ceph.mon.a.smithi027.stdout: No filesystems configured 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 bash[18873]: 517d2f5042c9097eb28678c5173890bb128f85ae39c7122b89803201d0e12591 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 systemd[1]: Started Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: monmap e1: 1 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: fsmap 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: osdmap e1: 0 total, 0 up, 0 in 2024-08-29T14:48:49.662 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:49 smithi027 ceph-mon[18892]: mgrmap e1: no daemons active 2024-08-29T14:48:50.596 INFO:teuthology.orchestra.run.smithi027.stdout:Wrote config to /etc/ceph/ceph.conf 2024-08-29T14:48:50.609 INFO:teuthology.orchestra.run.smithi027.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-08-29T14:48:50.610 INFO:teuthology.orchestra.run.smithi027.stdout:Creating mgr... 2024-08-29T14:48:50.610 INFO:teuthology.orchestra.run.smithi027.stdout:Verifying port 0.0.0.0:9283 ... 2024-08-29T14:48:50.610 INFO:teuthology.orchestra.run.smithi027.stdout:Verifying port 0.0.0.0:8765 ... 2024-08-29T14:48:50.864 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y 2024-08-29T14:48:50.864 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Failed to reset failed state of unit ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service: Unit ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service not loaded. 2024-08-29T14:48:51.033 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-af03926c-6615-11ef-bcd4-c7b262605968.target.wants/ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-af03926c-6615-11ef-bcd4-c7b262605968@.service. 2024-08-29T14:48:51.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:50 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/755917914' entity='client.admin' 2024-08-29T14:48:51.055 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 systemd[1]: Starting Ceph mgr.y for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:48:51.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 podman[19102]: 2024-08-29 14:48:51.16909023 +0000 UTC m=+0.016209120 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph:reef 2024-08-29T14:48:51.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 podman[19102]: 2024-08-29 14:48:51.269899607 +0000 UTC m=+0.117018487 container create b6d2336620e1c9ceb45726b999f9eb7d9f0e18d2dd94316c043327cfe3371016 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, RELEASE=reef-fe43f1c, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-08-29T14:48:51.555 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-29T14:48:51.555 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout disabled 2024-08-29T14:48:51.563 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-29T14:48:51.563 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout inactive 2024-08-29T14:48:51.563 INFO:teuthology.orchestra.run.smithi027.stdout:firewalld.service is not enabled 2024-08-29T14:48:51.563 INFO:teuthology.orchestra.run.smithi027.stdout:Not possible to enable service . firewalld.service is not available 2024-08-29T14:48:51.572 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-29T14:48:51.573 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout disabled 2024-08-29T14:48:51.581 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-29T14:48:51.581 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout inactive 2024-08-29T14:48:51.581 INFO:teuthology.orchestra.run.smithi027.stdout:firewalld.service is not enabled 2024-08-29T14:48:51.581 INFO:teuthology.orchestra.run.smithi027.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-08-29T14:48:51.581 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for mgr to start... 2024-08-29T14:48:51.581 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for mgr... 2024-08-29T14:48:51.664 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 podman[19102]: 2024-08-29 14:48:51.465207118 +0000 UTC m=+0.312325999 container init b6d2336620e1c9ceb45726b999f9eb7d9f0e18d2dd94316c043327cfe3371016 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-fe43f1c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.1, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-08-29T14:48:51.665 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 podman[19102]: 2024-08-29 14:48:51.467944438 +0000 UTC m=+0.315063327 container start b6d2336620e1c9ceb45726b999f9eb7d9f0e18d2dd94316c043327cfe3371016 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, io.buildah.version=1.37.1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T14:48:51.665 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 bash[19102]: b6d2336620e1c9ceb45726b999f9eb7d9f0e18d2dd94316c043327cfe3371016 2024-08-29T14:48:51.665 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 systemd[1]: Started Ceph mgr.y for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:48:51.665 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:51.600+0000 7f5ef7726040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-29T14:48:51.665 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:51.662+0000 7f5ef7726040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-29T14:48:51.926 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:51.823+0000 7f5ef7726040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-29T14:48:51.927 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:51 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:51.892+0000 7f5ef7726040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-29T14:48:52.201 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:52 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1324719453' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-29T14:48:52.202 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:52 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:52.047+0000 7f5ef7726040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:48:52.202 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:52 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:52.199+0000 7f5ef7726040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-29T14:48:52.624 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout { 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "fsid": "af03926c-6615-11ef-bcd4-c7b262605968", 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "health": { 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-29T14:48:52.627 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 0 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "a" 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-29T14:48:52.628 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-29T14:48:52.632 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-29T14:48:52.633 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "restful" 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "modified": "2024-08-29T14:48:44.627279+0000", 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:52.634 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-29T14:48:52.635 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout } 2024-08-29T14:48:52.635 INFO:teuthology.orchestra.run.smithi027.stdout:mgr not available, waiting (1/15)... 2024-08-29T14:48:52.785 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:52 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:52.527+0000 7f5ef7726040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-29T14:48:52.785 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:52 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:52.590+0000 7f5ef7726040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-29T14:48:53.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:52 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:52.782+0000 7f5ef7726040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-29T14:48:54.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:53 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:53.689+0000 7f5ef7726040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-29T14:48:54.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:53 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:53.758+0000 7f5ef7726040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-29T14:48:54.781 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:54.509+0000 7f5ef7726040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-29T14:48:54.781 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:54.629+0000 7f5ef7726040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-29T14:48:54.782 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:54.779+0000 7f5ef7726040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:48:55.034 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:54.844+0000 7f5ef7726040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-29T14:48:55.034 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.002+0000 7f5ef7726040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-29T14:48:55.310 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2325499470' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-29T14:48:55.310 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.075+0000 7f5ef7726040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-29T14:48:55.311 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.144+0000 7f5ef7726040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-29T14:48:55.311 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.308+0000 7f5ef7726040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-29T14:48:55.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.376+0000 7f5ef7726040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-29T14:48:55.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.448+0000 7f5ef7726040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-29T14:48:55.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.536+0000 7f5ef7726040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout { 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "fsid": "af03926c-6615-11ef-bcd4-c7b262605968", 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "health": { 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-29T14:48:55.659 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 0 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "a" 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-29T14:48:55.660 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:55.661 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-29T14:48:55.661 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-29T14:48:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "restful" 2024-08-29T14:48:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "modified": "2024-08-29T14:48:44.627279+0000", 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout } 2024-08-29T14:48:55.667 INFO:teuthology.orchestra.run.smithi027.stdout:mgr not available, waiting (2/15)... 2024-08-29T14:48:56.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.598+0000 7f5ef7726040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-29T14:48:56.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:55 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:55.660+0000 7f5ef7726040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-29T14:48:56.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:48:56.473+0000 7f5ef7726040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-29T14:48:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: Activating manager daemon y 2024-08-29T14:48:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: mgrmap e2: y(active, starting, since 0.00305953s) 2024-08-29T14:48:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:48:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:48:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: Manager daemon y is now available 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' 2024-08-29T14:48:56.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:56 smithi027 ceph-mon[18892]: from='mgr.14100 172.21.15.27:0/294193201' entity='mgr.y' 2024-08-29T14:48:58.791 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:58 smithi027 ceph-mon[18892]: mgrmap e3: y(active, since 1.00719s) 2024-08-29T14:48:58.791 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:58 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2623843315' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-29T14:48:58.828 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:58.828 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout { 2024-08-29T14:48:58.828 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "fsid": "af03926c-6615-11ef-bcd4-c7b262605968", 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "health": { 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 0 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "a" 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:58.829 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-29T14:48:58.830 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-29T14:48:58.834 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "restful" 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ], 2024-08-29T14:48:58.835 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "modified": "2024-08-29T14:48:44.627279+0000", 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout }, 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout } 2024-08-29T14:48:58.836 INFO:teuthology.orchestra.run.smithi027.stdout:mgr is available 2024-08-29T14:48:59.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:59 smithi027 ceph-mon[18892]: mgrmap e4: y(active, since 2s) 2024-08-29T14:48:59.791 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:48:59 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2445168464' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-29T14:48:59.978 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout [global] 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout fsid = af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout [osd] 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-29T14:48:59.979 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-29T14:48:59.980 INFO:teuthology.orchestra.run.smithi027.stdout:Enabling cephadm module... 2024-08-29T14:49:00.964 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:00 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1443614755' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-08-29T14:49:00.965 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:00 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setuser ceph since I am not root 2024-08-29T14:49:00.965 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:00 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setgroup ceph since I am not root 2024-08-29T14:49:00.965 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:00 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:00.787+0000 7f94e8bde040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-29T14:49:00.965 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:00 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:00.845+0000 7f94e8bde040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-29T14:49:01.224 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:01.001+0000 7f94e8bde040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-29T14:49:01.224 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:01.070+0000 7f94e8bde040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-29T14:49:01.224 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:01.222+0000 7f94e8bde040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:49:01.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:01.372+0000 7f94e8bde040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-29T14:49:01.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1443614755' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-08-29T14:49:01.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-mon[18892]: mgrmap e5: y(active, since 4s) 2024-08-29T14:49:01.894 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:01.699+0000 7f94e8bde040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-29T14:49:01.894 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:01.763+0000 7f94e8bde040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-29T14:49:02.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:01 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:01.957+0000 7f94e8bde040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-29T14:49:02.939 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:02 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4270207276' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-29T14:49:02.939 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:02 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:02.869+0000 7f94e8bde040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-29T14:49:02.940 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:02 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:02.938+0000 7f94e8bde040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-29T14:49:03.969 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:03.693+0000 7f94e8bde040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-29T14:49:03.970 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:03.818+0000 7f94e8bde040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-29T14:49:04.262 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:03.968+0000 7f94e8bde040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:49:04.262 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.033+0000 7f94e8bde040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-29T14:49:04.262 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.188+0000 7f94e8bde040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-29T14:49:04.262 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.260+0000 7f94e8bde040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-29T14:49:04.561 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.324+0000 7f94e8bde040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-29T14:49:04.561 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.490+0000 7f94e8bde040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-29T14:49:04.561 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.559+0000 7f94e8bde040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-29T14:49:04.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.630+0000 7f94e8bde040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-29T14:49:04.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.716+0000 7f94e8bde040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-29T14:49:04.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.778+0000 7f94e8bde040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-29T14:49:05.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:04 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:04.839+0000 7f94e8bde040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-29T14:49:05.748 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout { 2024-08-29T14:49:05.748 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-08-29T14:49:05.748 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-29T14:49:05.748 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-29T14:49:05.748 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-29T14:49:05.749 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout } 2024-08-29T14:49:05.749 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for the mgr to restart... 2024-08-29T14:49:05.749 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for mgr epoch 5... 2024-08-29T14:49:05.922 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:05.652+0000 7f94e8bde040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-29T14:49:05.922 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: Active manager daemon y restarted 2024-08-29T14:49:05.922 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: Activating manager daemon y 2024-08-29T14:49:05.922 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: osdmap e2: 0 total, 0 up, 0 in 2024-08-29T14:49:05.922 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: mgrmap e6: y(active, starting, since 0.00375733s) 2024-08-29T14:49:05.922 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:49:05.922 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: Manager daemon y is now available 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: Found migration_current of "None". Setting to last migration. 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:49:05.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:05 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:49:07.548 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:07.548 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:07.548 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-mon[18892]: mgrmap e7: y(active, since 1.00542s) 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:07.678+0000 7f94508b8640 -1 log_channel(cephadm) log [ERR] : [29/Aug/2024:14:49:07] ENGINE Error in HTTPServer.serve 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: Traceback (most recent call last): 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._connections.run(self.expiration_interval) 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._run(expiration_interval) 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:49:07.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: s = self.context.wrap_socket( 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: return self.sslsocket_class._create( 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self.do_handshake() 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._sslobj.do_handshake() 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:49:07.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:08.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:08 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:07] ENGINE Bus STARTING 2024-08-29T14:49:08.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:08 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:07] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:49:08.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:08 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:07] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:49:08.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:08 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:07] ENGINE Bus STARTED 2024-08-29T14:49:08.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:08 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:08 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:07] ENGINE Error in HTTPServer.serve 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:49:08.497 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:49:08.498 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:49:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:09 smithi027 ceph-mon[18892]: mgrmap e8: y(active, since 2s) 2024-08-29T14:49:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:09 smithi027 ceph-mon[18892]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-29T14:49:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:09 smithi027 ceph-mon[18892]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-29T14:49:13.714 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout { 2024-08-29T14:49:13.714 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 8, 2024-08-29T14:49:13.714 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-29T14:49:13.715 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout } 2024-08-29T14:49:13.715 INFO:teuthology.orchestra.run.smithi027.stdout:mgr epoch 5 is available 2024-08-29T14:49:13.715 INFO:teuthology.orchestra.run.smithi027.stdout:Setting orchestrator backend to cephadm... 2024-08-29T14:49:15.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:15 smithi027 ceph-mon[18892]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:15.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:15 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:15.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:15 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:16.167 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout value unchanged 2024-08-29T14:49:16.168 INFO:teuthology.orchestra.run.smithi027.stdout:Generating ssh key... 2024-08-29T14:49:16.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-mon[18892]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: Generating public/private rsa key pair. 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: Your identification has been saved in /tmp/tmpcpjwwyc1/key 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: Your public key has been saved in /tmp/tmpcpjwwyc1/key.pub 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: The key fingerprint is: 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: SHA256:iUehPW4Z+Gq0JmbnMdwWt5F4YSK9VLsi0IK/AUTQFsw ceph-af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: The key's randomart image is: 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: +---[RSA 3072]----+ 2024-08-29T14:49:17.238 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: |.*+. . . | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | .E. . = o . | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | .o o = O + | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | o o B X + | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | o + S * | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | = B = o | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | = X o . | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | o * + | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: | . | 2024-08-29T14:49:17.239 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: +----[SHA256]-----+ 2024-08-29T14:49:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:17 smithi027 ceph-mon[18892]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:17 smithi027 ceph-mon[18892]: Generating ssh key... 2024-08-29T14:49:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:17 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:17 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:19.114 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0gpvOSvsNBVExTEBZ+j9uytFD1yiUZCNS0pdX2dWMldNO3RU+blPqIUNg3mFHL0eLVTd59dOlK+5qgZy07X7oXzRJh0G4ecGq1q8JR7gQpT7ucNTYE97cRlAlFLC+Hz22X9aaqrY4ScZf/eu7GlPyB4BXyoBNrqV42QhsWwIHKZQa8v/zEsXMm/qNpmNngMuI882M7HQluAAmXbe9pF9vHIxlHEFpjgPp3YjEVF2mDQvh86OkRBPbP0s+FPkwccnYf4FNimpcSJtZiNfcwZe6klxZdyHgqEGgb1gwF55Hd8fZr2ghYcb6+FJqDNUFGtz0E8kP9hSr6eik+TRyKrLAIYp0MwpQROwtwtKEfguChhu5keGLOlqCNKpeQhzp0V4Y+hR9ngBUk049S7keq6sziPj2xbv8dlxw0DSC0zSZW6YapZ1vEzy4i+Iq884/y9P5vfesEE/4sG4Xgq/JspvQ6Dq/2BpUUgMjj+nAMBMOcxe1OBE2ZdAUrJIcmfF72YE= ceph-af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:49:19.115 INFO:teuthology.orchestra.run.smithi027.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-08-29T14:49:19.115 INFO:teuthology.orchestra.run.smithi027.stdout:Adding key to root@localhost authorized_keys... 2024-08-29T14:49:19.115 INFO:teuthology.orchestra.run.smithi027.stdout:Adding host smithi027... 2024-08-29T14:49:19.240 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:18 smithi027 ceph-mon[18892]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:21.059 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:20 smithi027 ceph-mon[18892]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi027", "addr": "172.21.15.27", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:21.968 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:21 smithi027 ceph-mon[18892]: Deploying cephadm binary to smithi027 2024-08-29T14:49:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:24 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:24 smithi027 ceph-mon[18892]: Added host smithi027 2024-08-29T14:49:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:24 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:24.803 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout Added host 'smithi027' with addr '172.21.15.27' 2024-08-29T14:49:24.803 INFO:teuthology.orchestra.run.smithi027.stdout:Deploying unmanaged mon service... 2024-08-29T14:49:25.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:25 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:25.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:25 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:25.951 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-08-29T14:49:25.952 INFO:teuthology.orchestra.run.smithi027.stdout:Deploying unmanaged mgr service... 2024-08-29T14:49:26.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:26 smithi027 ceph-mon[18892]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:26.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:26 smithi027 ceph-mon[18892]: Saving service mon spec with placement count:5 2024-08-29T14:49:26.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:26 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:26.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:26 smithi027 ceph-mon[18892]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:49:27.177 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-08-29T14:49:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:27 smithi027 ceph-mon[18892]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:27 smithi027 ceph-mon[18892]: Saving service mgr spec with placement count:2 2024-08-29T14:49:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:27 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:28.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:28 smithi027 ceph-mon[18892]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:49:28.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:28 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:28.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:28 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4148625567' entity='client.admin' 2024-08-29T14:49:28.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:28 smithi027 ceph-mon[18892]: from='mgr.14118 172.21.15.27:0/334870021' entity='mgr.y' 2024-08-29T14:49:29.953 INFO:teuthology.orchestra.run.smithi027.stdout:Enabling the dashboard module... 2024-08-29T14:49:30.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:30 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1191812803' entity='client.admin' 2024-08-29T14:49:30.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:30 smithi027 ceph-mon[18892]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:49:31.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2440727288' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-08-29T14:49:31.742 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setuser ceph since I am not root 2024-08-29T14:49:31.742 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setgroup ceph since I am not root 2024-08-29T14:49:31.742 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:31.527+0000 7f89b5cb2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-29T14:49:31.742 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:31.586+0000 7f89b5cb2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-29T14:49:31.742 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:31.739+0000 7f89b5cb2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-29T14:49:32.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:31.813+0000 7f89b5cb2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-29T14:49:32.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:31.964+0000 7f89b5cb2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:49:32.378 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:32 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:32.112+0000 7f89b5cb2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-29T14:49:32.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:32 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2440727288' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-08-29T14:49:32.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:32 smithi027 ceph-mon[18892]: mgrmap e9: y(active, since 25s) 2024-08-29T14:49:32.699 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:32 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:32.439+0000 7f89b5cb2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-29T14:49:32.700 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:32 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:32.504+0000 7f89b5cb2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-29T14:49:32.700 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:32 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:32.696+0000 7f89b5cb2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-29T14:49:33.321 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout { 2024-08-29T14:49:33.322 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-08-29T14:49:33.322 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-29T14:49:33.322 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-29T14:49:33.322 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-29T14:49:33.322 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout } 2024-08-29T14:49:33.322 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for the mgr to restart... 2024-08-29T14:49:33.322 INFO:teuthology.orchestra.run.smithi027.stdout:Waiting for mgr epoch 9... 2024-08-29T14:49:33.633 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:33 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2647511713' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-29T14:49:34.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:33 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:33.631+0000 7f89b5cb2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-29T14:49:34.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:33 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:33.701+0000 7f89b5cb2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-29T14:49:34.719 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:34.447+0000 7f89b5cb2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-29T14:49:34.719 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:34.566+0000 7f89b5cb2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-29T14:49:35.015 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:34.717+0000 7f89b5cb2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:49:35.015 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:34.782+0000 7f89b5cb2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-29T14:49:35.015 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:34.939+0000 7f89b5cb2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-29T14:49:35.015 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.012+0000 7f89b5cb2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-29T14:49:35.317 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.079+0000 7f89b5cb2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-29T14:49:35.317 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.245+0000 7f89b5cb2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-29T14:49:35.317 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.314+0000 7f89b5cb2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-29T14:49:35.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.383+0000 7f89b5cb2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-29T14:49:35.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.470+0000 7f89b5cb2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-29T14:49:35.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.532+0000 7f89b5cb2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-29T14:49:36.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:35.594+0000 7f89b5cb2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-29T14:49:36.663 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:36.409+0000 7f89b5cb2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-29T14:49:36.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: Active manager daemon y restarted 2024-08-29T14:49:36.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: Activating manager daemon y 2024-08-29T14:49:36.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: osdmap e3: 0 total, 0 up, 0 in 2024-08-29T14:49:36.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: mgrmap e10: y(active, starting, since 0.00315846s) 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: Manager daemon y is now available 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:49:36.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:36 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:49:38.114 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout { 2024-08-29T14:49:38.114 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-08-29T14:49:38.114 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-29T14:49:38.114 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout } 2024-08-29T14:49:38.114 INFO:teuthology.orchestra.run.smithi027.stdout:mgr epoch 9 is available 2024-08-29T14:49:38.114 INFO:teuthology.orchestra.run.smithi027.stdout:Generating a dashboard self-signed certificate... 2024-08-29T14:49:38.363 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:38.216+0000 7f891c13b640 -1 log_channel(cephadm) log [ERR] : [29/Aug/2024:14:49:38] ENGINE Error in HTTPServer.serve 2024-08-29T14:49:38.363 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: Traceback (most recent call last): 2024-08-29T14:49:38.363 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:49:38.363 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._connections.run(self.expiration_interval) 2024-08-29T14:49:38.363 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:49:38.363 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._run(expiration_interval) 2024-08-29T14:49:38.363 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: s = self.context.wrap_socket( 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: return self.sslsocket_class._create( 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self.do_handshake() 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:49:38.364 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._sslobj.do_handshake() 2024-08-29T14:49:38.365 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:49:38.365 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:49:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-mon[18892]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-29T14:49:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-mon[18892]: mgrmap e11: y(active, since 1.00547s) 2024-08-29T14:49:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-mon[18892]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-29T14:49:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:39.445 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-08-29T14:49:39.445 INFO:teuthology.orchestra.run.smithi027.stdout:Creating initial admin user... 2024-08-29T14:49:39.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:37] ENGINE Bus STARTING 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:38] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:38] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:38] ENGINE Bus STARTED 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: [29/Aug/2024:14:49:38] ENGINE Error in HTTPServer.serve 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:49:39.536 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:39.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:39 smithi027 ceph-mon[18892]: mgrmap e12: y(active, since 2s) 2024-08-29T14:49:40.712 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:40 smithi027 ceph-mon[18892]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:40.712 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:40.990 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$KrXD5oNh76r7DfazSYzOLOgcOP8MQtyTUmfNhJQ8zkR6T/SUQb7We", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1724942980, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-08-29T14:49:40.990 INFO:teuthology.orchestra.run.smithi027.stdout:Fetching dashboard port number... 2024-08-29T14:49:41.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:41 smithi027 ceph-mon[18892]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:42.307 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stdout 8443 2024-08-29T14:49:42.318 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-29T14:49:42.318 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout disabled 2024-08-29T14:49:42.326 INFO:teuthology.orchestra.run.smithi027.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-29T14:49:42.326 INFO:teuthology.orchestra.run.smithi027.stdout:systemctl: stdout inactive 2024-08-29T14:49:42.326 INFO:teuthology.orchestra.run.smithi027.stdout:firewalld.service is not enabled 2024-08-29T14:49:42.326 INFO:teuthology.orchestra.run.smithi027.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-08-29T14:49:42.328 INFO:teuthology.orchestra.run.smithi027.stdout:Ceph Dashboard is now available at: 2024-08-29T14:49:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: URL: https://smithi027:8443/ 2024-08-29T14:49:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: User: admin 2024-08-29T14:49:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: Password: lqvdsgzp4l 2024-08-29T14:49:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:42.328 INFO:teuthology.orchestra.run.smithi027.stdout:Saving cluster configuration to /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config directory 2024-08-29T14:49:42.329 INFO:teuthology.orchestra.run.smithi027.stdout:Enabling autotune for osd_memory_target 2024-08-29T14:49:42.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:42 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/569212638' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-08-29T14:49:43.750 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:43 smithi027 ceph-mon[18892]: mgrmap e13: y(active, since 6s) 2024-08-29T14:49:44.846 INFO:teuthology.orchestra.run.smithi027.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout:Or, if you are only running a single cluster on this host: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout:Please consider enabling telemetry to help improve Ceph: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: ceph telemetry on 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout:For more information see: 2024-08-29T14:49:44.847 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.848 INFO:teuthology.orchestra.run.smithi027.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-08-29T14:49:44.848 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:44.848 INFO:teuthology.orchestra.run.smithi027.stdout:Bootstrap complete. 2024-08-29T14:49:44.888 INFO:tasks.cephadm:Fetching config... 2024-08-29T14:49:44.889 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:49:44.889 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-08-29T14:49:44.908 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-08-29T14:49:44.908 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:49:44.908 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-08-29T14:49:44.966 INFO:tasks.cephadm:Fetching mon keyring... 2024-08-29T14:49:44.966 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:49:44.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/keyring of=/dev/stdout 2024-08-29T14:49:45.034 INFO:tasks.cephadm:Fetching pub ssh key... 2024-08-29T14:49:45.034 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:49:45.034 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-08-29T14:49:45.091 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-08-29T14:49:45.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0gpvOSvsNBVExTEBZ+j9uytFD1yiUZCNS0pdX2dWMldNO3RU+blPqIUNg3mFHL0eLVTd59dOlK+5qgZy07X7oXzRJh0G4ecGq1q8JR7gQpT7ucNTYE97cRlAlFLC+Hz22X9aaqrY4ScZf/eu7GlPyB4BXyoBNrqV42QhsWwIHKZQa8v/zEsXMm/qNpmNngMuI882M7HQluAAmXbe9pF9vHIxlHEFpjgPp3YjEVF2mDQvh86OkRBPbP0s+FPkwccnYf4FNimpcSJtZiNfcwZe6klxZdyHgqEGgb1gwF55Hd8fZr2ghYcb6+FJqDNUFGtz0E8kP9hSr6eik+TRyKrLAIYp0MwpQROwtwtKEfguChhu5keGLOlqCNKpeQhzp0V4Y+hR9ngBUk049S7keq6sziPj2xbv8dlxw0DSC0zSZW6YapZ1vEzy4i+Iq884/y9P5vfesEE/4sG4Xgq/JspvQ6Dq/2BpUUgMjj+nAMBMOcxe1OBE2ZdAUrJIcmfF72YE= ceph-af03926c-6615-11ef-bcd4-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-29T14:49:45.170 INFO:teuthology.orchestra.run.smithi027.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0gpvOSvsNBVExTEBZ+j9uytFD1yiUZCNS0pdX2dWMldNO3RU+blPqIUNg3mFHL0eLVTd59dOlK+5qgZy07X7oXzRJh0G4ecGq1q8JR7gQpT7ucNTYE97cRlAlFLC+Hz22X9aaqrY4ScZf/eu7GlPyB4BXyoBNrqV42QhsWwIHKZQa8v/zEsXMm/qNpmNngMuI882M7HQluAAmXbe9pF9vHIxlHEFpjgPp3YjEVF2mDQvh86OkRBPbP0s+FPkwccnYf4FNimpcSJtZiNfcwZe6klxZdyHgqEGgb1gwF55Hd8fZr2ghYcb6+FJqDNUFGtz0E8kP9hSr6eik+TRyKrLAIYp0MwpQROwtwtKEfguChhu5keGLOlqCNKpeQhzp0V4Y+hR9ngBUk049S7keq6sziPj2xbv8dlxw0DSC0zSZW6YapZ1vEzy4i+Iq884/y9P5vfesEE/4sG4Xgq/JspvQ6Dq/2BpUUgMjj+nAMBMOcxe1OBE2ZdAUrJIcmfF72YE= ceph-af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:49:45.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0gpvOSvsNBVExTEBZ+j9uytFD1yiUZCNS0pdX2dWMldNO3RU+blPqIUNg3mFHL0eLVTd59dOlK+5qgZy07X7oXzRJh0G4ecGq1q8JR7gQpT7ucNTYE97cRlAlFLC+Hz22X9aaqrY4ScZf/eu7GlPyB4BXyoBNrqV42QhsWwIHKZQa8v/zEsXMm/qNpmNngMuI882M7HQluAAmXbe9pF9vHIxlHEFpjgPp3YjEVF2mDQvh86OkRBPbP0s+FPkwccnYf4FNimpcSJtZiNfcwZe6klxZdyHgqEGgb1gwF55Hd8fZr2ghYcb6+FJqDNUFGtz0E8kP9hSr6eik+TRyKrLAIYp0MwpQROwtwtKEfguChhu5keGLOlqCNKpeQhzp0V4Y+hR9ngBUk049S7keq6sziPj2xbv8dlxw0DSC0zSZW6YapZ1vEzy4i+Iq884/y9P5vfesEE/4sG4Xgq/JspvQ6Dq/2BpUUgMjj+nAMBMOcxe1OBE2ZdAUrJIcmfF72YE= ceph-af03926c-6615-11ef-bcd4-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-29T14:49:45.228 INFO:teuthology.orchestra.run.smithi171.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0gpvOSvsNBVExTEBZ+j9uytFD1yiUZCNS0pdX2dWMldNO3RU+blPqIUNg3mFHL0eLVTd59dOlK+5qgZy07X7oXzRJh0G4ecGq1q8JR7gQpT7ucNTYE97cRlAlFLC+Hz22X9aaqrY4ScZf/eu7GlPyB4BXyoBNrqV42QhsWwIHKZQa8v/zEsXMm/qNpmNngMuI882M7HQluAAmXbe9pF9vHIxlHEFpjgPp3YjEVF2mDQvh86OkRBPbP0s+FPkwccnYf4FNimpcSJtZiNfcwZe6klxZdyHgqEGgb1gwF55Hd8fZr2ghYcb6+FJqDNUFGtz0E8kP9hSr6eik+TRyKrLAIYp0MwpQROwtwtKEfguChhu5keGLOlqCNKpeQhzp0V4Y+hR9ngBUk049S7keq6sziPj2xbv8dlxw0DSC0zSZW6YapZ1vEzy4i+Iq884/y9P5vfesEE/4sG4Xgq/JspvQ6Dq/2BpUUgMjj+nAMBMOcxe1OBE2ZdAUrJIcmfF72YE= ceph-af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:49:45.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-08-29T14:49:45.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:45 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2435357418' entity='client.admin' 2024-08-29T14:49:45.497 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:49:48.647 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-08-29T14:49:48.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-08-29T14:49:48.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:48 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1368823089' entity='client.admin' 2024-08-29T14:49:48.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:48.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:48.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi027", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:49:48.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:48.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:48.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:48.933 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:49:52.069 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi171 2024-08-29T14:49:52.069 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:49:52.069 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/etc/ceph/ceph.conf 2024-08-29T14:49:52.088 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:49:52.088 DEBUG:teuthology.orchestra.run.smithi171:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:49:52.147 INFO:tasks.cephadm:Adding host smithi171 to orchestrator... 2024-08-29T14:49:52.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch host add smithi171 2024-08-29T14:49:52.351 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:52.472 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:53.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:53 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:49:53.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:53 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:49:53.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:53 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:49:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:54 smithi027 ceph-mon[18892]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi171", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:49:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:55 smithi027 ceph-mon[18892]: Deploying cephadm binary to smithi171 2024-08-29T14:49:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:56 smithi027 ceph-mon[18892]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:49:57.295 INFO:teuthology.orchestra.run.smithi027.stdout:Added host 'smithi171' with addr '172.21.15.171' 2024-08-29T14:49:57.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch host ls --format=json 2024-08-29T14:49:58.135 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:49:58.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:58.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:58 smithi027 ceph-mon[18892]: Added host smithi171 2024-08-29T14:49:58.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:49:58.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:58.563 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:59.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:59 smithi027 ceph-mon[18892]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:49:59.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:49:59 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:49:59.844 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:49:59.844 INFO:teuthology.orchestra.run.smithi027.stdout:[{"addr": "172.21.15.27", "hostname": "smithi027", "labels": [], "status": ""}, {"addr": "172.21.15.171", "hostname": "smithi171", "labels": [], "status": ""}] 2024-08-29T14:50:00.499 INFO:tasks.cephadm:Setting crush tunables to default 2024-08-29T14:50:00.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd crush tunables default 2024-08-29T14:50:00.701 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:50:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:00 smithi027 ceph-mon[18892]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:50:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:00 smithi027 ceph-mon[18892]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi171", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:50:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:50:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:03.348 INFO:teuthology.orchestra.run.smithi027.stderr:adjusted tunables profile to default 2024-08-29T14:50:03.696 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:03 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:03.696 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:03 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:50:03.696 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:03 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:50:03.696 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:03 smithi027 ceph-mon[18892]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:03.696 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:03 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3228789255' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-29T14:50:03.956 INFO:tasks.cephadm:Adding mon.a on smithi027 2024-08-29T14:50:03.956 INFO:tasks.cephadm:Adding mon.c on smithi027 2024-08-29T14:50:03.956 INFO:tasks.cephadm:Adding mon.b on smithi171 2024-08-29T14:50:03.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch apply mon '3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi171:172.21.15.171=b' 2024-08-29T14:50:04.155 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:04.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:04 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3228789255' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-29T14:50:04.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:04 smithi027 ceph-mon[18892]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:05.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:05 smithi027 ceph-mon[18892]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:05.878 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled mon update... 2024-08-29T14:50:06.416 DEBUG:teuthology.orchestra.run.smithi027:mon.c> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.c.service 2024-08-29T14:50:06.419 DEBUG:teuthology.orchestra.run.smithi171:mon.b> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.b.service 2024-08-29T14:50:06.422 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-29T14:50:06.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph mon dump -f json 2024-08-29T14:50:07.160 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:07.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi171:172.21.15.171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:07.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: Saving service mon spec with placement smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi171:172.21.15.171=b;count:3 2024-08-29T14:50:07.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:07.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:07.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:07.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:07.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:07.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:07.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:07.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: Deploying daemon mon.b on smithi171 2024-08-29T14:50:07.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:06 smithi027 ceph-mon[18892]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:08 smithi027 ceph-mon[18892]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:50:10.326 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":1,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","modified":"2024-08-29T14:48:41.684545Z","created":"2024-08-29T14:48:41.684545Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-08-29T14:50:10.327 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 1 2024-08-29T14:50:10.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:10 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/2332691896' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:11 smithi171 systemd[1]: Started Ceph mon.b for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:50:11.895 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-29T14:50:11.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph mon dump -f json 2024-08-29T14:50:12.100 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:50:16.686 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:50:16.686 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":2,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","modified":"2024-08-29T14:50:11.341096Z","created":"2024-08-29T14:48:41.684545Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:3300","nonce":0},{"type":"v1","addr":"172.21.15.171:6789","nonce":0}]},"addr":"172.21.15.171:6789/0","public_addr":"172.21.15.171:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-29T14:50:16.686 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 2 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: mon.a calling monitor election 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: mon.b calling monitor election 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: monmap e2: 2 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:50:16.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: fsmap 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: mgrmap e13: y(active, since 39s) 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: overall HEALTH_OK 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:16.688 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:16 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:16.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:50:16.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: mon.a calling monitor election 2024-08-29T14:50:16.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:16.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: mon.b calling monitor election 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: monmap e2: 2 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: fsmap 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: mgrmap e13: y(active, since 39s) 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: overall HEALTH_OK 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:16.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:16.718 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:16.718 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:16 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:17 smithi027 ceph-mon[18892]: Deploying daemon mon.c on smithi027 2024-08-29T14:50:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:17 smithi027 ceph-mon[18892]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:17 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/1359150615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:17 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:17 smithi171 ceph-mon[22959]: Deploying daemon mon.c on smithi027 2024-08-29T14:50:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:17 smithi171 ceph-mon[22959]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:17 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/1359150615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:17.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:17 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:18.293 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-29T14:50:18.293 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph mon dump -f json 2024-08-29T14:50:18.512 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:50:18.632 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:18 smithi171 ceph-mon[22959]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:18.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:18 smithi027 ceph-mon[18892]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:19.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 systemd[1]: Starting Ceph mon.c for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:50:19.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 podman[29324]: 2024-08-29 14:50:19.404960071 +0000 UTC m=+0.015939155 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T14:50:19.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 podman[29324]: 2024-08-29 14:50:19.518705437 +0000 UTC m=+0.129684511 container create f7785aec7256bbbfa602bd49c67d79e1ea78aba38fb3e94d5d95f2b55b33e795 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-29T14:50:19.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 podman[29324]: 2024-08-29 14:50:19.745798958 +0000 UTC m=+0.356778042 container init f7785aec7256bbbfa602bd49c67d79e1ea78aba38fb3e94d5d95f2b55b33e795 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-29T14:50:19.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 podman[29324]: 2024-08-29 14:50:19.749244989 +0000 UTC m=+0.360224064 container start f7785aec7256bbbfa602bd49c67d79e1ea78aba38fb3e94d5d95f2b55b33e795 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.1, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-fe43f1c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: set uid:gid to 167:167 (ceph:ceph) 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable), process ceph-mon, pid 2 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pidfile_write: ignore empty --pid-file 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: load: jerasure load: lrc 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: RocksDB version: 7.9.2 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Git sha 0 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Compile date 2024-08-26 17:15:55 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: DB SUMMARY 2024-08-29T14:50:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: DB Session ID: IBIJJ5GQ3LUNQX9754XG 2024-08-29T14:50:20.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: CURRENT file: CURRENT 2024-08-29T14:50:20.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: IDENTITY file: IDENTITY 2024-08-29T14:50:20.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-08-29T14:50:20.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-08-29T14:50:20.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-08-29T14:50:20.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.error_if_exists: 0 2024-08-29T14:50:20.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.create_if_missing: 0 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.paranoid_checks: 1 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.env: 0x55cea4a87bc0 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.fs: PosixFileSystem 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.info_log: 0x55cea5ab8be0 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_file_opening_threads: 16 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.statistics: (nil) 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.use_fsync: 0 2024-08-29T14:50:20.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_log_file_size: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.keep_log_file_num: 1000 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.recycle_log_file_num: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.allow_fallocate: 1 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.allow_mmap_reads: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.allow_mmap_writes: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.use_direct_reads: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.create_missing_column_families: 0 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.db_log_dir: 2024-08-29T14:50:20.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.wal_dir: 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.advise_random_on_open: 1 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.db_write_buffer_size: 0 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.write_buffer_manager: 0x55cea5ac6320 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-29T14:50:20.073 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.rate_limiter: (nil) 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.wal_recovery_mode: 2 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.enable_thread_tracking: 0 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.enable_pipelined_write: 0 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.unordered_write: 0 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.row_cache: None 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.wal_filter: None 2024-08-29T14:50:20.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.allow_ingest_behind: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.two_write_queues: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.manual_wal_flush: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.wal_compression: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.atomic_flush: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.log_readahead_size: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.best_efforts_recovery: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.allow_data_in_errors: 0 2024-08-29T14:50:20.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.db_host_id: __hostname__ 2024-08-29T14:50:20.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_background_jobs: 2 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_background_compactions: -1 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_subcompactions: 1 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_total_wal_size: 0 2024-08-29T14:50:20.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_open_files: -1 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bytes_per_sync: 0 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_readahead_size: 0 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_background_flushes: -1 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Compression algorithms supported: 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kZSTD supported: 0 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kXpressCompression supported: 0 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kBZip2Compression supported: 0 2024-08-29T14:50:20.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kLZ4Compression supported: 1 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kZlibCompression supported: 1 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kLZ4HCCompression supported: 1 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: kSnappyCompression supported: 1 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.merge_operator: 2024-08-29T14:50:20.079 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_filter: None 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_filter_factory: None 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.sst_partitioner_factory: None 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cea5ab87e0) 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_type: 0 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_index_type: 0 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_shortening: 1 2024-08-29T14:50:20.080 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: checksum: 4 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: no_block_cache: 0 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache: 0x55cea5aad350 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_options: 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: capacity : 536870912 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_shard_bits : 4 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: strict_capacity_limit : 0 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-08-29T14:50:20.081 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_compressed: (nil) 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: persistent_cache: (nil) 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size: 4096 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size_deviation: 10 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_restart_interval: 16 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_block_restart_interval: 1 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: metadata_block_size: 4096 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: partition_filters: 0 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: use_delta_encoding: 1 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: filter_policy: bloomfilter 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: whole_key_filtering: 1 2024-08-29T14:50:20.082 INFO:journalctl@ceph.mon.c.smithi027.stdout: verify_compression: 0 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: format_version: 5 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_index_compression: 1 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_align: 0 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: max_auto_readahead_size: 262144 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: prepopulate_block_cache: 0 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: initial_auto_readahead_size: 8192 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.write_buffer_size: 33554432 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_write_buffer_number: 2 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression: NoCompression 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression: Disabled 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.prefix_extractor: nullptr 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-29T14:50:20.083 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.num_levels: 7 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-29T14:50:20.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.level: 32767 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.strategy: 0 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.enabled: false 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-29T14:50:20.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.target_file_size_base: 67108864 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-29T14:50:20.086 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.arena_block_size: 1048576 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.disable_auto_compactions: 0 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-29T14:50:20.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.inplace_update_support: 0 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.bloom_locality: 0 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.max_successive_merges: 0 2024-08-29T14:50:20.088 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.paranoid_file_checks: 0 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.force_consistency_checks: 1 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.report_bg_io_stats: 0 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.ttl: 2592000 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.enable_blob_files: false 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.min_blob_size: 0 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.blob_file_size: 268435456 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-29T14:50:20.089 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.blob_file_starting_level: 0 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: db5d8f7d-b07b-4fd8-90bb-a6d33af7132d 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943019776385, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943019776874, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1773, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1724943019, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "db5d8f7d-b07b-4fd8-90bb-a6d33af7132d", "db_session_id": "IBIJJ5GQ3LUNQX9754XG", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-08-29T14:50:20.090 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943019776971, "job": 1, "event": "recovery_finished"} 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55cea5ad8e00 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: DB pointer 0x55cea5bcc000 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** DB Stats ** 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T14:50:20.091 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: L0 1/0 1.73 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: Sum 1/0 1.73 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-29T14:50:20.092 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache BinnedLRUCache@0x55cea5aad350#2 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-08-29T14:50:20.093 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-08-29T14:50:20.094 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T14:50:20.094 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-29T14:50:20.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-08-29T14:50:20.094 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: using public_addrv [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: starting mon.c rank -1 at public addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] at bind addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(???) e0 preinit fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).mds e1 new map 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).mds e1 print_map 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout: e1 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout: legacy client fscid: -1 2024-08-29T14:50:20.096 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout: No filesystems configured 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi171", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3228789255' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3228789255' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi171:172.21.15.171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: Saving service mon spec with placement smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi171:172.21.15.171=b;count:3 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:20.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: Deploying daemon mon.b on smithi171 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/2332691896' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.a calling monitor election 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:20.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.b calling monitor election 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: monmap e2: 2 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: fsmap 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mgrmap e13: y(active, since 39s) 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: overall HEALTH_OK 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: Deploying daemon mon.c on smithi027 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/1359150615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 ceph-mon[29342]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 bash[29324]: f7785aec7256bbbfa602bd49c67d79e1ea78aba38fb3e94d5d95f2b55b33e795 2024-08-29T14:50:20.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:19 smithi027 systemd[1]: Started Ceph mon.c for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:50:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:50:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: mon.a calling monitor election 2024-08-29T14:50:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: mon.b calling monitor election 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: mon.c calling monitor election 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T14:50:25.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: fsmap 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: mgrmap e13: y(active, since 48s) 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: overall HEALTH_OK 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:50:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: mon.a calling monitor election 2024-08-29T14:50:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: mon.b calling monitor election 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: mon.c calling monitor election 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:50:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: fsmap 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: mgrmap e13: y(active, since 48s) 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: overall HEALTH_OK 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: mon.a calling monitor election 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:50:25.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: mon.b calling monitor election 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: mon.c calling monitor election 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: fsmap 2024-08-29T14:50:25.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: osdmap e4: 0 total, 0 up, 0 in 2024-08-29T14:50:25.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: mgrmap e13: y(active, since 48s) 2024-08-29T14:50:25.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: overall HEALTH_OK 2024-08-29T14:50:25.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:25.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:26.138 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:50:26.138 INFO:teuthology.orchestra.run.smithi171.stdout:{"epoch":3,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","modified":"2024-08-29T14:50:19.801400Z","created":"2024-08-29T14:48:41.684545Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:3300","nonce":0},{"type":"v1","addr":"172.21.15.171:6789","nonce":0}]},"addr":"172.21.15.171:6789/0","public_addr":"172.21.15.171:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3301","nonce":0},{"type":"v1","addr":"172.21.15.27:6790","nonce":0}]},"addr":"172.21.15.27:6790/0","public_addr":"172.21.15.27:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-08-29T14:50:26.138 INFO:teuthology.orchestra.run.smithi171.stderr:dumped monmap epoch 3 2024-08-29T14:50:26.138 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:25 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:26.138 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:25 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:26.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:25 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:26.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:25 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:25 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:25 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:50:26.790 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-08-29T14:50:26.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph config generate-minimal-conf 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/1707269928' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:27.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:27.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/1707269928' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/1707269928' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:27.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:27.498 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:50:28.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[18892]: Reconfiguring mon.a (unknown last config time)... 2024-08-29T14:50:28.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[18892]: Reconfiguring daemon mon.a on smithi027 2024-08-29T14:50:28.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:28.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:28.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[29342]: Reconfiguring mon.a (unknown last config time)... 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[29342]: Reconfiguring daemon mon.a on smithi027 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:28.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:28 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:28 smithi171 ceph-mon[22959]: Reconfiguring mon.a (unknown last config time)... 2024-08-29T14:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:28 smithi171 ceph-mon[22959]: Reconfiguring daemon mon.a on smithi027 2024-08-29T14:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:28 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:28 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:28 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:28 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:28.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:28 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:29 smithi171 ceph-mon[22959]: Reconfiguring mon.c (monmap changed)... 2024-08-29T14:50:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:29 smithi171 ceph-mon[22959]: Reconfiguring daemon mon.c on smithi027 2024-08-29T14:50:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:29 smithi171 ceph-mon[22959]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:29 smithi027 ceph-mon[18892]: Reconfiguring mon.c (monmap changed)... 2024-08-29T14:50:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:29 smithi027 ceph-mon[18892]: Reconfiguring daemon mon.c on smithi027 2024-08-29T14:50:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:29 smithi027 ceph-mon[18892]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:29.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:29 smithi027 ceph-mon[29342]: Reconfiguring mon.c (monmap changed)... 2024-08-29T14:50:29.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:29 smithi027 ceph-mon[29342]: Reconfiguring daemon mon.c on smithi027 2024-08-29T14:50:29.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:29 smithi027 ceph-mon[29342]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:30.586 INFO:teuthology.orchestra.run.smithi027.stdout:# minimal ceph.conf for af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:50:30.586 INFO:teuthology.orchestra.run.smithi027.stdout:[global] 2024-08-29T14:50:30.586 INFO:teuthology.orchestra.run.smithi027.stdout: fsid = af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T14:50:30.586 INFO:teuthology.orchestra.run.smithi027.stdout: mon_host = [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] 2024-08-29T14:50:30.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:30.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:30.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: Reconfiguring mon.b (monmap changed)... 2024-08-29T14:50:30.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:30.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:30.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: Reconfiguring daemon mon.b on smithi171 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[18892]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: Reconfiguring mon.b (monmap changed)... 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: Reconfiguring daemon mon.b on smithi171 2024-08-29T14:50:30.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:30 smithi027 ceph-mon[29342]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: Reconfiguring mon.b (monmap changed)... 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: Reconfiguring daemon mon.b on smithi171 2024-08-29T14:50:30.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:30 smithi171 ceph-mon[22959]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:31.250 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-08-29T14:50:31.250 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:50:31.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.conf 2024-08-29T14:50:31.278 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:50:31.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:50:31.346 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:50:31.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.conf 2024-08-29T14:50:31.376 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:50:31.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:50:31.445 INFO:tasks.cephadm:Adding mgr.y on smithi027 2024-08-29T14:50:31.445 INFO:tasks.cephadm:Adding mgr.x on smithi171 2024-08-29T14:50:31.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch apply mgr '2;smithi027=y;smithi171=x' 2024-08-29T14:50:31.680 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:50:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:31 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3872902464' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:31.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:31.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3872902464' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3872902464' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:32.690 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:32 smithi171 ceph-mon[22959]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:32 smithi027 ceph-mon[29342]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:32 smithi027 ceph-mon[18892]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:33.362 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled mgr update... 2024-08-29T14:50:33.957 DEBUG:teuthology.orchestra.run.smithi171:mgr.x> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service 2024-08-29T14:50:33.959 DEBUG:tasks.cephadm:set 0 configs 2024-08-29T14:50:33.960 INFO:tasks.cephadm:Deploying OSDs... 2024-08-29T14:50:33.960 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:50:33.960 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/scratch_devs of=/dev/stdout 2024-08-29T14:50:33.981 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-29T14:50:33.981 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_1 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 965 Links: 1 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-08-29 14:50:19.229721126 +0000 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-08-29 14:45:50.442016896 +0000 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-08-29 14:45:50.442016896 +0000 2024-08-29T14:50:34.039 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-08-29 14:45:50.442016896 +0000 2024-08-29T14:50:34.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-29T14:50:34.105 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-08-29T14:50:34.105 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-08-29T14:50:34.105 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000215306 s, 2.4 MB/s 2024-08-29T14:50:34.106 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-29T14:50:34.163 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_2 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 980 Links: 1 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-08-29 14:50:19.229721126 +0000 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-08-29 14:45:50.443016871 +0000 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-08-29 14:45:50.443016871 +0000 2024-08-29T14:50:34.220 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-08-29 14:45:50.443016871 +0000 2024-08-29T14:50:34.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-29T14:50:34.284 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-08-29T14:50:34.285 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-08-29T14:50:34.285 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000194178 s, 2.6 MB/s 2024-08-29T14:50:34.286 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-29T14:50:34.342 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_3 2024-08-29T14:50:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-29T14:50:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:34.398 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-08-29T14:50:34.398 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:34.398 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:34.398 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-08-29 14:50:19.230721101 +0000 2024-08-29T14:50:34.399 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-08-29 14:45:50.443016871 +0000 2024-08-29T14:50:34.399 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-08-29 14:45:50.443016871 +0000 2024-08-29T14:50:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-08-29 14:45:50.443016871 +0000 2024-08-29T14:50:34.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-29T14:50:34.463 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-08-29T14:50:34.464 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-08-29T14:50:34.464 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000195757 s, 2.6 MB/s 2024-08-29T14:50:34.465 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-29T14:50:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi171=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: Saving service mgr spec with placement smithi027=y;smithi171=x;count:2 2024-08-29T14:50:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:34 smithi171 ceph-mon[22959]: Deploying daemon mgr.x on smithi171 2024-08-29T14:50:34.521 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_4 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi171=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: Saving service mgr spec with placement smithi027=y;smithi171=x;count:2 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[29342]: Deploying daemon mgr.x on smithi171 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi171=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: Saving service mgr spec with placement smithi027=y;smithi171=x;count:2 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:34.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:50:34.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-29T14:50:34.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:50:34.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:34.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:34 smithi027 ceph-mon[18892]: Deploying daemon mgr.x on smithi171 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 973 Links: 1 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-08-29 14:50:19.230721101 +0000 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-08-29 14:45:50.442016896 +0000 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-08-29 14:45:50.442016896 +0000 2024-08-29T14:50:34.578 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-08-29 14:45:50.442016896 +0000 2024-08-29T14:50:34.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-29T14:50:34.645 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-08-29T14:50:34.645 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-08-29T14:50:34.645 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000190426 s, 2.7 MB/s 2024-08-29T14:50:34.646 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-29T14:50:34.703 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:50:34.703 DEBUG:teuthology.orchestra.run.smithi171:> dd if=/scratch_devs of=/dev/stdout 2024-08-29T14:50:34.721 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-29T14:50:34.722 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_1 2024-08-29T14:50:34.779 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-08-29 14:50:10.464441448 +0000 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-08-29 14:45:50.434166716 +0000 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-08-29 14:45:50.434166716 +0000 2024-08-29T14:50:34.780 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-08-29 14:45:50.434166716 +0000 2024-08-29T14:50:34.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-29T14:50:34.845 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-08-29T14:50:34.845 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-08-29T14:50:34.846 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000195279 s, 2.6 MB/s 2024-08-29T14:50:34.847 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-29T14:50:34.904 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_2 2024-08-29T14:50:34.961 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-29T14:50:34.961 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:34.961 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-08-29T14:50:34.961 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:34.961 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:34.961 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-08-29 14:50:10.464441448 +0000 2024-08-29T14:50:34.962 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-08-29 14:45:50.431166344 +0000 2024-08-29T14:50:34.962 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-08-29 14:45:50.431166344 +0000 2024-08-29T14:50:34.962 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-08-29 14:45:50.431166344 +0000 2024-08-29T14:50:34.962 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-29T14:50:35.027 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-08-29T14:50:35.027 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-08-29T14:50:35.027 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000308455 s, 1.7 MB/s 2024-08-29T14:50:35.028 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-29T14:50:35.086 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_3 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 965 Links: 1 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-08-29 14:50:10.465441429 +0000 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-08-29 14:45:50.430166220 +0000 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-08-29 14:45:50.430166220 +0000 2024-08-29T14:50:35.142 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-08-29 14:45:50.430166220 +0000 2024-08-29T14:50:35.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-29T14:50:35.292 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-08-29T14:50:35.292 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-08-29T14:50:35.292 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000243355 s, 2.1 MB/s 2024-08-29T14:50:35.294 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-29T14:50:35.309 DEBUG:teuthology.orchestra.run.smithi171:> stat /dev/vg_nvme/lv_4 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout:Device: 5h/5d Inode: 977 Links: 1 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout:Context: system_u:object_r:device_t:s0 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout:Access: 2024-08-29 14:50:35.245985234 +0000 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout:Modify: 2024-08-29 14:45:50.433166592 +0000 2024-08-29T14:50:35.368 INFO:teuthology.orchestra.run.smithi171.stdout:Change: 2024-08-29 14:45:50.433166592 +0000 2024-08-29T14:50:35.369 INFO:teuthology.orchestra.run.smithi171.stdout: Birth: 2024-08-29 14:45:50.433166592 +0000 2024-08-29T14:50:35.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-29T14:50:35.430 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:35 smithi171 ceph-mon[22959]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:35.475 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records in 2024-08-29T14:50:35.475 INFO:teuthology.orchestra.run.smithi171.stderr:1+0 records out 2024-08-29T14:50:35.475 INFO:teuthology.orchestra.run.smithi171.stderr:512 bytes copied, 0.000207689 s, 2.5 MB/s 2024-08-29T14:50:35.476 DEBUG:teuthology.orchestra.run.smithi171:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-29T14:50:35.534 INFO:tasks.cephadm:Deploying osd.0 on smithi027 with /dev/vg_nvme/lv_4... 2024-08-29T14:50:35.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-29T14:50:35.557 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:35 smithi027 ceph-mon[18892]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:35.557 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:35 smithi027 ceph-mon[29342]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:35.707 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:35 smithi171 systemd[1]: Starting Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:50:35.732 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:50:35.976 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:35 smithi171 podman[24796]: 2024-08-29 14:50:35.609397882 +0000 UTC m=+0.016000990 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T14:50:35.976 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:35 smithi171 podman[24796]: 2024-08-29 14:50:35.737905941 +0000 UTC m=+0.144509091 container create fc352c65c32f4eaf7cc7d6c94a3f8f6c58418e76d1562cd561c21c44dbad73ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-08-29T14:50:35.977 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:35 smithi171 podman[24796]: 2024-08-29 14:50:35.948362084 +0000 UTC m=+0.354965188 container init fc352c65c32f4eaf7cc7d6c94a3f8f6c58418e76d1562cd561c21c44dbad73ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-fe43f1c, ceph=True) 2024-08-29T14:50:35.977 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:35 smithi171 podman[24796]: 2024-08-29 14:50:35.951070212 +0000 UTC m=+0.357673329 container start fc352c65c32f4eaf7cc7d6c94a3f8f6c58418e76d1562cd561c21c44dbad73ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-29T14:50:36.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:35 smithi171 ceph-mgr[24815]: -- 172.21.15.171:0/3837331469 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c2a2ebcd00 con 0x55c2a2e91000 2024-08-29T14:50:36.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 bash[24796]: fc352c65c32f4eaf7cc7d6c94a3f8f6c58418e76d1562cd561c21c44dbad73ab 2024-08-29T14:50:36.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 systemd[1]: Started Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:50:36.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:36.071+0000 7f666a2a1040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-29T14:50:36.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:36.130+0000 7f666a2a1040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-29T14:50:36.664 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:36.289+0000 7f666a2a1040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-29T14:50:36.664 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:36.359+0000 7f666a2a1040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-29T14:50:36.664 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:36.521+0000 7f666a2a1040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:50:36.664 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:36.660+0000 7f666a2a1040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-29T14:50:36.985 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:36.983+0000 7f666a2a1040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-29T14:50:37.155 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.155 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.155 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[29342]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:37.156 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[18892]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:37.157 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.244 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:37.045+0000 7f666a2a1040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-29T14:50:37.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:37.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-mon[22959]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:37.245 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:37.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:37.242+0000 7f666a2a1040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-29T14:50:38.262 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:50:38.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: Reconfiguring mgr.y (unknown last config time)... 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:38.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[18892]: Reconfiguring daemon mgr.y on smithi027 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: Reconfiguring mgr.y (unknown last config time)... 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:38.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:38 smithi027 ceph-mon[29342]: Reconfiguring daemon mgr.y on smithi027 2024-08-29T14:50:38.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_4 2024-08-29T14:50:38.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:38.157+0000 7f666a2a1040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-29T14:50:38.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:38.224+0000 7f666a2a1040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: Reconfiguring mgr.y (unknown last config time)... 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:38.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-mon[22959]: Reconfiguring daemon mgr.y on smithi027 2024-08-29T14:50:39.204 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-mon[22959]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:39.204 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:38.947+0000 7f666a2a1040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-29T14:50:39.204 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.060+0000 7f666a2a1040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-29T14:50:39.204 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.202+0000 7f666a2a1040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:50:39.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:39 smithi027 ceph-mon[18892]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:39.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:39 smithi027 ceph-mon[29342]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:39.395 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:50:39.476 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.262+0000 7f666a2a1040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-29T14:50:39.476 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.406+0000 7f666a2a1040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-29T14:50:39.476 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.473+0000 7f666a2a1040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-29T14:50:39.756 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.533+0000 7f666a2a1040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-29T14:50:39.756 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.689+0000 7f666a2a1040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-29T14:50:39.756 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.754+0000 7f666a2a1040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-29T14:50:40.017 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.820+0000 7f666a2a1040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-29T14:50:40.017 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.902+0000 7f666a2a1040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-29T14:50:40.017 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:39.961+0000 7f666a2a1040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-29T14:50:40.439 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:40.020+0000 7f666a2a1040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-29T14:50:40.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:40.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:40.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:40.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:40.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:40.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:40.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:50:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:41.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:50:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:50:40.801+0000 7f666a2a1040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-29T14:50:41.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:41.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: Standby manager daemon x started 2024-08-29T14:50:41.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:50:41.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:50:41.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: Standby manager daemon x started 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:50:41.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:50:41.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:50:41.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:41 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: Standby manager daemon x started 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3583732161' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:50:41.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:41 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:42.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:42 smithi027 ceph-mon[18892]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:42.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:42 smithi027 ceph-mon[18892]: mgrmap e14: y(active, since 65s), standbys: x 2024-08-29T14:50:42.736 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:42 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:50:42.737 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:42 smithi027 ceph-mon[29342]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:42.737 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:42 smithi027 ceph-mon[29342]: mgrmap e14: y(active, since 65s), standbys: x 2024-08-29T14:50:42.737 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:42 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:50:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:42 smithi171 ceph-mon[22959]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:50:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:42 smithi171 ceph-mon[22959]: mgrmap e14: y(active, since 65s), standbys: x 2024-08-29T14:50:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:42 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:50:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:43 smithi171 ceph-mon[22959]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:43.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:43 smithi027 ceph-mon[18892]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:43.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:43 smithi027 ceph-mon[29342]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:44.724 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/965534510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]: dispatch 2024-08-29T14:50:44.724 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]: dispatch 2024-08-29T14:50:44.724 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]': finished 2024-08-29T14:50:44.724 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[18892]: osdmap e5: 1 total, 0 up, 1 in 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1970202437' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/965534510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]: dispatch 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]: dispatch 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]': finished 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[29342]: osdmap e5: 1 total, 0 up, 1 in 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:50:44.725 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:44 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1970202437' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:44 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/965534510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]: dispatch 2024-08-29T14:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:44 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]: dispatch 2024-08-29T14:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:44 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "170eaad0-5c69-43ff-a4cc-cfdcc1763f52"}]': finished 2024-08-29T14:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:44 smithi171 ceph-mon[22959]: osdmap e5: 1 total, 0 up, 1 in 2024-08-29T14:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:44 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:44 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1970202437' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:50:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:45 smithi027 ceph-mon[18892]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:45.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:45 smithi027 ceph-mon[29342]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:45 smithi171 ceph-mon[22959]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:46.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:46 smithi027 ceph-mon[18892]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:46.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:46 smithi027 ceph-mon[29342]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:46 smithi171 ceph-mon[22959]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:48.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:48 smithi027 ceph-mon[18892]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:48.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:48 smithi027 ceph-mon[29342]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:48 smithi171 ceph-mon[22959]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:49.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:49 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-29T14:50:49.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:49 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:49.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:49 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-29T14:50:49.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:49 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:49 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-29T14:50:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:49 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:50:50.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:50 smithi027 ceph-mon[18892]: Deploying daemon osd.0 on smithi027 2024-08-29T14:50:50.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:50 smithi027 ceph-mon[18892]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:50.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:50 smithi027 ceph-mon[29342]: Deploying daemon osd.0 on smithi027 2024-08-29T14:50:50.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:50 smithi027 ceph-mon[29342]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:51.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:50 smithi171 ceph-mon[22959]: Deploying daemon osd.0 on smithi027 2024-08-29T14:50:51.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:50 smithi171 ceph-mon[22959]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:52.725 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:52 smithi027 ceph-mon[18892]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:52.725 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:52 smithi027 ceph-mon[29342]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:52 smithi171 ceph-mon[22959]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:54.563 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:54 smithi027 ceph-mon[18892]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:54.563 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:54 smithi027 ceph-mon[29342]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:54 smithi171 ceph-mon[22959]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:55.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:55.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:55.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:55.645 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:55.645 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:55.645 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:50:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:56.764 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:56 smithi171 ceph-mon[22959]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:56 smithi027 ceph-mon[18892]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:56.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:56 smithi027 ceph-mon[29342]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:57.664 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 0 on host 'smithi027' 2024-08-29T14:50:58.348 DEBUG:teuthology.orchestra.run.smithi027:osd.0> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.0.service 2024-08-29T14:50:58.350 INFO:tasks.cephadm:Deploying osd.1 on smithi027 with /dev/vg_nvme/lv_3... 2024-08-29T14:50:58.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-29T14:50:58.606 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:50:58.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:58.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[18892]: from='osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[18892]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[18892]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[29342]: from='osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[29342]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[29342]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:58.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:58.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:58 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:58 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:58 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:58 smithi171 ceph-mon[22959]: from='osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-29T14:50:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:58 smithi171 ceph-mon[22959]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-29T14:50:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:58 smithi171 ceph-mon[22959]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:50:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:58 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:58 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:50:59.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[18892]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-29T14:50:59.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[18892]: osdmap e6: 1 total, 0 up, 1 in 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[18892]: from='osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[18892]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[29342]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[29342]: osdmap e6: 1 total, 0 up, 1 in 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[29342]: from='osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:50:59.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-mon[29342]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:50:59.995 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[33347]: 2024-08-29T14:50:59.686+0000 7ff07699e640 -1 osd.0 0 waiting for initial osdmap 2024-08-29T14:50:59.995 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:50:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[33347]: 2024-08-29T14:50:59.688+0000 7ff0727c7640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:51:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:59 smithi171 ceph-mon[22959]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-29T14:51:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:59 smithi171 ceph-mon[22959]: osdmap e6: 1 total, 0 up, 1 in 2024-08-29T14:51:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:59 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:59 smithi171 ceph-mon[22959]: from='osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:50:59 smithi171 ceph-mon[22959]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[18892]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[18892]: osdmap e7: 1 total, 0 up, 1 in 2024-08-29T14:51:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:00.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:00.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[18892]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:51:00.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[29342]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:00.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[29342]: osdmap e7: 1 total, 0 up, 1 in 2024-08-29T14:51:00.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:00.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:00.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:00 smithi027 ceph-mon[29342]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:51:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:00 smithi171 ceph-mon[22959]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:00 smithi171 ceph-mon[22959]: osdmap e7: 1 total, 0 up, 1 in 2024-08-29T14:51:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:00 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:00 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:00 smithi171 ceph-mon[22959]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[18892]: osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338] boot 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[18892]: osdmap e8: 1 total, 1 up, 1 in 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:51:01.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:01.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[29342]: osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338] boot 2024-08-29T14:51:01.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[29342]: osdmap e8: 1 total, 1 up, 1 in 2024-08-29T14:51:01.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:01.957 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:51:01.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_3 2024-08-29T14:51:02.020 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:01 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:51:02.020 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:01 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:51:02.020 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:02.020 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:01 smithi171 ceph-mon[22959]: osd.0 [v2:172.21.15.27:6802/1938465338,v1:172.21.15.27:6803/1938465338] boot 2024-08-29T14:51:02.020 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:01 smithi171 ceph-mon[22959]: osdmap e8: 1 total, 1 up, 1 in 2024-08-29T14:51:02.020 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:51:02.562 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:51:02.809 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:02 smithi027 ceph-mon[18892]: osdmap e9: 1 total, 1 up, 1 in 2024-08-29T14:51:02.809 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:02 smithi027 ceph-mon[18892]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:02.809 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:02 smithi027 ceph-mon[29342]: osdmap e9: 1 total, 1 up, 1 in 2024-08-29T14:51:02.809 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:02 smithi027 ceph-mon[29342]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:02 smithi171 ceph-mon[22959]: osdmap e9: 1 total, 1 up, 1 in 2024-08-29T14:51:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:02 smithi171 ceph-mon[22959]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:05.363 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi027 2024-08-29T14:51:05.363 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.363 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi027 to 16064M 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi027 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi027 to 16064M 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:05.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:05.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:05 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi027 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi027 to 16064M 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:05.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:05 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:06.383 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:06 smithi027 ceph-mon[18892]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:06.384 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:06 smithi027 ceph-mon[29342]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:06 smithi171 ceph-mon[22959]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:07.358 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[18892]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:07.358 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3177153171' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73986270-de54-40cf-969c-ddb7ecba6f81"}]: dispatch 2024-08-29T14:51:07.358 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3177153171' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73986270-de54-40cf-969c-ddb7ecba6f81"}]': finished 2024-08-29T14:51:07.358 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[18892]: osdmap e10: 2 total, 1 up, 2 in 2024-08-29T14:51:07.358 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:07.358 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[29342]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:07.359 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3177153171' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73986270-de54-40cf-969c-ddb7ecba6f81"}]: dispatch 2024-08-29T14:51:07.359 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3177153171' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73986270-de54-40cf-969c-ddb7ecba6f81"}]': finished 2024-08-29T14:51:07.359 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[29342]: osdmap e10: 2 total, 1 up, 2 in 2024-08-29T14:51:07.359 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:07 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:07 smithi171 ceph-mon[22959]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:07 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3177153171' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73986270-de54-40cf-969c-ddb7ecba6f81"}]: dispatch 2024-08-29T14:51:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:07 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3177153171' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73986270-de54-40cf-969c-ddb7ecba6f81"}]': finished 2024-08-29T14:51:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:07 smithi171 ceph-mon[22959]: osdmap e10: 2 total, 1 up, 2 in 2024-08-29T14:51:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:07 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:08 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3244753448' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:08.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:08 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3244753448' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:08.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:08 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3244753448' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:09 smithi171 ceph-mon[22959]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:09.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:09 smithi027 ceph-mon[18892]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:09.594 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:09 smithi027 ceph-mon[29342]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:10.554 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:10 smithi027 ceph-mon[18892]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:10.554 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:10 smithi027 ceph-mon[29342]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:10 smithi171 ceph-mon[22959]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:12 smithi171 ceph-mon[22959]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:12.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:12 smithi027 ceph-mon[29342]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:12.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:12 smithi027 ceph-mon[18892]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:13 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-29T14:51:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:13 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:13 smithi171 ceph-mon[22959]: Deploying daemon osd.1 on smithi027 2024-08-29T14:51:13.787 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:13 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-29T14:51:13.787 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:13 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:13.787 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:13 smithi027 ceph-mon[18892]: Deploying daemon osd.1 on smithi027 2024-08-29T14:51:13.787 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:13 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-29T14:51:13.787 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:13 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:13.787 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:13 smithi027 ceph-mon[29342]: Deploying daemon osd.1 on smithi027 2024-08-29T14:51:14.703 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:14 smithi027 ceph-mon[18892]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:14.704 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:14 smithi027 ceph-mon[29342]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:14.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:14 smithi171 ceph-mon[22959]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:16.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:16 smithi027 ceph-mon[18892]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:16.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:16 smithi027 ceph-mon[29342]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:16 smithi171 ceph-mon[22959]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:18.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:18 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:18.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:18 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:18.365 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:18 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:18.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:18 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:18.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:18 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:18.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:18 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:18 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:18 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:18 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:19.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:19 smithi027 ceph-mon[18892]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:19.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:19 smithi027 ceph-mon[29342]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:19 smithi171 ceph-mon[22959]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:20 smithi171 ceph-mon[22959]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:20.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:20 smithi027 ceph-mon[18892]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:20 smithi027 ceph-mon[29342]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:21.249 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 1 on host 'smithi027' 2024-08-29T14:51:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:21 smithi171 ceph-mon[22959]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-29T14:51:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:21 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:21 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:21.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:21 smithi027 ceph-mon[29342]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-29T14:51:21.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:21 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:21.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:21 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:21 smithi027 ceph-mon[18892]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-29T14:51:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:21 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:21 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:21.847 DEBUG:teuthology.orchestra.run.smithi027:osd.1> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.1.service 2024-08-29T14:51:21.850 INFO:tasks.cephadm:Deploying osd.2 on smithi027 with /dev/vg_nvme/lv_2... 2024-08-29T14:51:21.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-29T14:51:22.113 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:51:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[18892]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-29T14:51:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[18892]: osdmap e11: 2 total, 1 up, 2 in 2024-08-29T14:51:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[18892]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[18892]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[29342]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[29342]: osdmap e11: 2 total, 1 up, 2 in 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[29342]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-mon[29342]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:22.820 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[38855]: 2024-08-29T14:51:22.522+0000 7fe57126a640 -1 osd.1 0 waiting for initial osdmap 2024-08-29T14:51:22.820 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:22 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[38855]: 2024-08-29T14:51:22.529+0000 7fe56d093640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:51:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:22 smithi171 ceph-mon[22959]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-29T14:51:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:22 smithi171 ceph-mon[22959]: osdmap e11: 2 total, 1 up, 2 in 2024-08-29T14:51:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:22 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:22 smithi171 ceph-mon[22959]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:22 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:22 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:22 smithi171 ceph-mon[22959]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[18892]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[18892]: osdmap e12: 2 total, 1 up, 2 in 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[29342]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[29342]: osdmap e12: 2 total, 1 up, 2 in 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:23.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:23 smithi171 ceph-mon[22959]: from='osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:23 smithi171 ceph-mon[22959]: osdmap e12: 2 total, 1 up, 2 in 2024-08-29T14:51:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:24.697 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[18892]: osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746] boot 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[18892]: osdmap e13: 2 total, 2 up, 2 in 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[18892]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:51:24.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:24.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[29342]: osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746] boot 2024-08-29T14:51:24.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[29342]: osdmap e13: 2 total, 2 up, 2 in 2024-08-29T14:51:24.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:24.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:24 smithi027 ceph-mon[29342]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:24 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:51:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:24 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:51:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:24 smithi171 ceph-mon[22959]: osd.1 [v2:172.21.15.27:6810/2967503746,v1:172.21.15.27:6811/2967503746] boot 2024-08-29T14:51:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:24 smithi171 ceph-mon[22959]: osdmap e13: 2 total, 2 up, 2 in 2024-08-29T14:51:25.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:24 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:51:25.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:24 smithi171 ceph-mon[22959]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-29T14:51:25.196 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:51:25.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_2 2024-08-29T14:51:25.746 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:51:25.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:25 smithi027 ceph-mon[18892]: osdmap e14: 2 total, 2 up, 2 in 2024-08-29T14:51:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:25 smithi027 ceph-mon[29342]: osdmap e14: 2 total, 2 up, 2 in 2024-08-29T14:51:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:25 smithi171 ceph-mon[22959]: osdmap e14: 2 total, 2 up, 2 in 2024-08-29T14:51:26.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:26 smithi027 ceph-mon[18892]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:26 smithi027 ceph-mon[29342]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:26 smithi171 ceph-mon[22959]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi027 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi027 to 8032M 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:28.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:28.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:28.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi027 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi027 to 8032M 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi027 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi027 to 8032M 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[29342]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[29342]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[18892]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:29 smithi027 ceph-mon[18892]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:29 smithi171 ceph-mon[22959]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:29 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:29 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:29 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:29 smithi171 ceph-mon[22959]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:30 smithi171 ceph-mon[22959]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:30.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:30 smithi027 ceph-mon[18892]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:30.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:30 smithi027 ceph-mon[29342]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:31 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/4012934370' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2b83d1d-ecf8-4576-a411-6d05ef2528c9"}]: dispatch 2024-08-29T14:51:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:31 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/4012934370' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2b83d1d-ecf8-4576-a411-6d05ef2528c9"}]': finished 2024-08-29T14:51:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:31 smithi171 ceph-mon[22959]: osdmap e15: 3 total, 2 up, 3 in 2024-08-29T14:51:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:31 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/4161320363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4012934370' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2b83d1d-ecf8-4576-a411-6d05ef2528c9"}]: dispatch 2024-08-29T14:51:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4012934370' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2b83d1d-ecf8-4576-a411-6d05ef2528c9"}]': finished 2024-08-29T14:51:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[18892]: osdmap e15: 3 total, 2 up, 3 in 2024-08-29T14:51:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4161320363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/4012934370' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2b83d1d-ecf8-4576-a411-6d05ef2528c9"}]: dispatch 2024-08-29T14:51:31.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/4012934370' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2b83d1d-ecf8-4576-a411-6d05ef2528c9"}]': finished 2024-08-29T14:51:31.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[29342]: osdmap e15: 3 total, 2 up, 3 in 2024-08-29T14:51:31.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:31.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:31 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/4161320363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:32 smithi171 ceph-mon[22959]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:32 smithi027 ceph-mon[29342]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:32 smithi027 ceph-mon[18892]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:34.588 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:34 smithi027 ceph-mon[18892]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:34.588 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:34 smithi027 ceph-mon[29342]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:34 smithi171 ceph-mon[22959]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:36 smithi171 ceph-mon[22959]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:36 smithi027 ceph-mon[29342]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:36.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:36 smithi027 ceph-mon[18892]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-29T14:51:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:37 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:37 smithi171 ceph-mon[22959]: Deploying daemon osd.2 on smithi027 2024-08-29T14:51:37.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-29T14:51:37.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:37 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:37.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:37 smithi027 ceph-mon[18892]: Deploying daemon osd.2 on smithi027 2024-08-29T14:51:37.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-29T14:51:37.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:37 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:37.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:37 smithi027 ceph-mon[29342]: Deploying daemon osd.2 on smithi027 2024-08-29T14:51:38.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:38 smithi171 ceph-mon[22959]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:38 smithi027 ceph-mon[18892]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:38.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:38 smithi027 ceph-mon[29342]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:40 smithi027 ceph-mon[18892]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:40.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:40 smithi027 ceph-mon[29342]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:40 smithi171 ceph-mon[22959]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:41 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:41 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:41 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:41 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:41 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:41 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:41.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:41 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:41.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:41 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:41.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:41 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:42 smithi171 ceph-mon[22959]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:42.813 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:42 smithi027 ceph-mon[18892]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:42.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:42 smithi027 ceph-mon[29342]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:44.900 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 2 on host 'smithi027' 2024-08-29T14:51:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-29T14:51:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:45.593 DEBUG:teuthology.orchestra.run.smithi027:osd.2> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service 2024-08-29T14:51:45.596 INFO:tasks.cephadm:Deploying osd.3 on smithi027 with /dev/vg_nvme/lv_1... 2024-08-29T14:51:45.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-29T14:51:46.249 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:51:46.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[18892]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T14:51:46.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[18892]: osdmap e16: 3 total, 2 up, 3 in 2024-08-29T14:51:46.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:46.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[18892]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:46.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[18892]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:46.943 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[29342]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T14:51:46.943 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[29342]: osdmap e16: 3 total, 2 up, 3 in 2024-08-29T14:51:46.943 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:46.943 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[29342]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:46.943 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:46 smithi027 ceph-mon[29342]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:46 smithi171 ceph-mon[22959]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T14:51:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:46 smithi171 ceph-mon[22959]: osdmap e16: 3 total, 2 up, 3 in 2024-08-29T14:51:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:46 smithi171 ceph-mon[22959]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:51:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:46 smithi171 ceph-mon[22959]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:47.239 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[44691]: 2024-08-29T14:51:47.037+0000 7f22a3f50640 -1 osd.2 0 waiting for initial osdmap 2024-08-29T14:51:47.239 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[44691]: 2024-08-29T14:51:47.042+0000 7f229bd71640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:51:47.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[18892]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:47.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[18892]: osdmap e17: 3 total, 2 up, 3 in 2024-08-29T14:51:47.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:47.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:47.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[18892]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' 2024-08-29T14:51:47.867 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[29342]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:47.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[29342]: osdmap e17: 3 total, 2 up, 3 in 2024-08-29T14:51:47.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:47.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:47.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:47 smithi027 ceph-mon[29342]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' 2024-08-29T14:51:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:47 smithi171 ceph-mon[22959]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:51:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:47 smithi171 ceph-mon[22959]: osdmap e17: 3 total, 2 up, 3 in 2024-08-29T14:51:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:47 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:47 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:47 smithi171 ceph-mon[22959]: from='osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028]' entity='osd.2' 2024-08-29T14:51:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:51:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:51:49.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:49.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028] boot 2024-08-29T14:51:49.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: osdmap e18: 3 total, 3 up, 3 in 2024-08-29T14:51:49.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:49.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:49.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:48 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-29T14:51:49.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:51:49.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:51:49.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:49.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028] boot 2024-08-29T14:51:49.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: osdmap e18: 3 total, 3 up, 3 in 2024-08-29T14:51:49.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:49.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: osd.2 [v2:172.21.15.27:6818/302366028,v1:172.21.15.27:6819/302366028] boot 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: osdmap e18: 3 total, 3 up, 3 in 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-29T14:51:49.274 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:48 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: osdmap e19: 3 total, 3 up, 3 in 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi027 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi027 to 5354M 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: osdmap e19: 3 total, 3 up, 3 in 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi027 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi027 to 5354M 2024-08-29T14:51:50.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:50.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:50.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:50 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-29T14:51:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: osdmap e19: 3 total, 3 up, 3 in 2024-08-29T14:51:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-29T14:51:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi027 2024-08-29T14:51:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:51:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi027 to 5354M 2024-08-29T14:51:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:51:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:50 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:50.648 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:50 smithi027 sudo[48856]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-29T14:51:50.648 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:50 smithi027 sudo[48856]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:50.648 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:50 smithi027 sudo[48856]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:50.649 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:50 smithi027 sudo[48856]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:51:50.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_1 2024-08-29T14:51:50.997 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:51:51.351 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48933]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-29T14:51:51.351 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48933]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:51.351 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48933]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:51.351 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48933]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:51.352 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:51 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-29T14:51:51.352 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:51 smithi027 ceph-mon[29342]: osdmap e20: 3 total, 3 up, 3 in 2024-08-29T14:51:51.352 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:51 smithi027 ceph-mon[29342]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:51.352 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:51 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-29T14:51:51.352 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:51 smithi027 ceph-mon[18892]: osdmap e20: 3 total, 3 up, 3 in 2024-08-29T14:51:51.352 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:51 smithi027 ceph-mon[18892]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:51 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-29T14:51:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:51 smithi171 ceph-mon[22959]: osdmap e20: 3 total, 3 up, 3 in 2024-08-29T14:51:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:51 smithi171 ceph-mon[22959]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:51.804 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48955]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-29T14:51:51.804 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48955]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:51.804 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48955]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:51.804 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:51 smithi027 sudo[48955]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:52.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48960]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-29T14:51:52.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48960]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:52.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48960]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:52.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48960]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:52.371 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:52 smithi027 ceph-mon[29342]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-29T14:51:52.371 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:52 smithi027 ceph-mon[29342]: osdmap e21: 3 total, 3 up, 3 in 2024-08-29T14:51:52.371 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:52 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:52.371 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:52 smithi027 ceph-mon[18892]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-29T14:51:52.372 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:52 smithi027 ceph-mon[18892]: osdmap e21: 3 total, 3 up, 3 in 2024-08-29T14:51:52.372 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:52 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:52 smithi171 ceph-mon[22959]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-29T14:51:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:52 smithi171 ceph-mon[22959]: osdmap e21: 3 total, 3 up, 3 in 2024-08-29T14:51:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:52 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:52.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48988]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-29T14:51:52.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48988]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:52.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48988]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:52.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[48988]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:53.258 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[49079]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-29T14:51:53.258 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[49079]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:53.258 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[49079]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:53.258 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 14:51:52 smithi027 sudo[49079]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:53.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49140]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-29T14:51:53.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49140]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:53.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49140]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:53.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49140]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:54.106 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[29342]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:54.106 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:54.106 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:54.106 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:54.106 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49160]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-29T14:51:54.107 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49160]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:54.107 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49160]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:54.107 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 sudo[49160]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:54.107 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[18892]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:54.107 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:54.107 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:54.107 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:53 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:54.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25449]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-29T14:51:54.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25449]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:54.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25449]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:54.335 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25449]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:54.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25452]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-29T14:51:54.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25452]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:54.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25452]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:54.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 sudo[25452]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:54.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 ceph-mon[22959]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:54.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:51:54.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:51:54.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:51:55.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:54 smithi027 sudo[49211]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-29T14:51:55.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:54 smithi027 sudo[49211]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:55.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:54 smithi027 sudo[49211]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:55.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:54 smithi027 sudo[49211]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:55 smithi171 ceph-mon[22959]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 85 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 sudo[49234]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 sudo[49234]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 sudo[49234]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 sudo[49234]: pam_unix(sudo:session): session closed for user root 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:55.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[29342]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 85 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:51:55.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:51:55.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:51:55.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:51:55.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:55 smithi027 ceph-mon[18892]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 85 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:56 smithi171 ceph-mon[22959]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:56 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3846745274' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "afc1c2c8-0579-4020-b460-b3cbef9573d5"}]: dispatch 2024-08-29T14:51:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:56 smithi171 ceph-mon[22959]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:56.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:56 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3846745274' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "afc1c2c8-0579-4020-b460-b3cbef9573d5"}]': finished 2024-08-29T14:51:56.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:56 smithi171 ceph-mon[22959]: osdmap e22: 4 total, 3 up, 4 in 2024-08-29T14:51:56.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:56 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:51:56.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:56 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/685494098' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[18892]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3846745274' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "afc1c2c8-0579-4020-b460-b3cbef9573d5"}]: dispatch 2024-08-29T14:51:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[18892]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3846745274' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "afc1c2c8-0579-4020-b460-b3cbef9573d5"}]': finished 2024-08-29T14:51:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[18892]: osdmap e22: 4 total, 3 up, 4 in 2024-08-29T14:51:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/685494098' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[29342]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3846745274' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "afc1c2c8-0579-4020-b460-b3cbef9573d5"}]: dispatch 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[29342]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3846745274' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "afc1c2c8-0579-4020-b460-b3cbef9573d5"}]': finished 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[29342]: osdmap e22: 4 total, 3 up, 4 in 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:51:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:56 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/685494098' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:51:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:57 smithi027 ceph-mon[18892]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:57 smithi027 ceph-mon[29342]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:57 smithi171 ceph-mon[22959]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:51:58 smithi171 ceph-mon[22959]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:51:58 smithi027 ceph-mon[18892]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:51:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:51:58 smithi027 ceph-mon[29342]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:52:00.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:00 smithi027 ceph-mon[18892]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 51 KiB/s wr, 4 op/s 2024-08-29T14:52:00.550 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:00 smithi027 ceph-mon[29342]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 51 KiB/s wr, 4 op/s 2024-08-29T14:52:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:00 smithi171 ceph-mon[22959]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 51 KiB/s wr, 4 op/s 2024-08-29T14:52:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-29T14:52:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:01 smithi171 ceph-mon[22959]: Deploying daemon osd.3 on smithi027 2024-08-29T14:52:01.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-29T14:52:01.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:01.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:01 smithi027 ceph-mon[18892]: Deploying daemon osd.3 on smithi027 2024-08-29T14:52:01.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-29T14:52:01.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:01.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:01 smithi027 ceph-mon[29342]: Deploying daemon osd.3 on smithi027 2024-08-29T14:52:02.702 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:02 smithi027 ceph-mon[18892]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.3 KiB/s rd, 37 KiB/s wr, 3 op/s 2024-08-29T14:52:02.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:02 smithi027 ceph-mon[29342]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.3 KiB/s rd, 37 KiB/s wr, 3 op/s 2024-08-29T14:52:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:02 smithi171 ceph-mon[22959]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.3 KiB/s rd, 37 KiB/s wr, 3 op/s 2024-08-29T14:52:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:04 smithi171 ceph-mon[22959]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1005 B/s rd, 29 KiB/s wr, 2 op/s 2024-08-29T14:52:04.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:04 smithi027 ceph-mon[18892]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1005 B/s rd, 29 KiB/s wr, 2 op/s 2024-08-29T14:52:04.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:04 smithi027 ceph-mon[29342]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1005 B/s rd, 29 KiB/s wr, 2 op/s 2024-08-29T14:52:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:06 smithi171 ceph-mon[22959]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 825 B/s rd, 23 KiB/s wr, 2 op/s 2024-08-29T14:52:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:06 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:06.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:06 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:06.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:06 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[18892]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 825 B/s rd, 23 KiB/s wr, 2 op/s 2024-08-29T14:52:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:06.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[29342]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 825 B/s rd, 23 KiB/s wr, 2 op/s 2024-08-29T14:52:06.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:06.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:06.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:06 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:08.711 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:08 smithi027 ceph-mon[18892]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-08-29T14:52:08.712 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:08 smithi027 ceph-mon[29342]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-08-29T14:52:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:08 smithi171 ceph-mon[22959]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2024-08-29T14:52:09.805 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 3 on host 'smithi027' 2024-08-29T14:52:10.375 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.376 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.395 DEBUG:teuthology.orchestra.run.smithi027:osd.3> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.3.service 2024-08-29T14:52:10.398 INFO:tasks.cephadm:Deploying osd.4 on smithi171 with /dev/vg_nvme/lv_4... 2024-08-29T14:52:10.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-29T14:52:10.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:10.428 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.428 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:10.601 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:52:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:11 smithi171 ceph-mon[22959]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-29T14:52:11.640 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:11 smithi171 ceph-mon[22959]: osdmap e23: 4 total, 3 up, 4 in 2024-08-29T14:52:11.641 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:11 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:11.641 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:11 smithi171 ceph-mon[22959]: from='osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:52:11.641 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:11 smithi171 ceph-mon[22959]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:52:11.641 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:11 smithi171 ceph-mon[22959]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[18892]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[18892]: osdmap e23: 4 total, 3 up, 4 in 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[18892]: from='osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[18892]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[18892]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[29342]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[29342]: osdmap e23: 4 total, 3 up, 4 in 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:11.642 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[29342]: from='osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:52:11.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[29342]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-08-29T14:52:11.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-mon[29342]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:52:11.643 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[50900]: 2024-08-29T14:52:11.136+0000 7f227dd27640 -1 osd.3 0 waiting for initial osdmap 2024-08-29T14:52:11.643 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 14:52:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[50900]: 2024-08-29T14:52:11.140+0000 7f2279b50640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:52:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:12 smithi171 ceph-mon[22959]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:52:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:12 smithi171 ceph-mon[22959]: osdmap e24: 4 total, 3 up, 4 in 2024-08-29T14:52:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:12 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:12 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[18892]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:52:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[18892]: osdmap e24: 4 total, 3 up, 4 in 2024-08-29T14:52:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[29342]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-08-29T14:52:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[29342]: osdmap e24: 4 total, 3 up, 4 in 2024-08-29T14:52:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:12 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:13.036 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:52:13.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_4 2024-08-29T14:52:13.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:52:13.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:52:13.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:13.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[29342]: osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291] boot 2024-08-29T14:52:13.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[29342]: osdmap e25: 4 total, 4 up, 4 in 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[29342]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[18892]: osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291] boot 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[18892]: osdmap e25: 4 total, 4 up, 4 in 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:13.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:13 smithi027 ceph-mon[18892]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:52:13.299 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:52:13.420 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:13 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:52:13.420 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:13 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:52:13.421 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:13 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:13.421 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:13 smithi171 ceph-mon[22959]: osd.3 [v2:172.21.15.27:6826/250460291,v1:172.21.15.27:6827/250460291] boot 2024-08-29T14:52:13.421 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:13 smithi171 ceph-mon[22959]: osdmap e25: 4 total, 4 up, 4 in 2024-08-29T14:52:13.421 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:13 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:52:13.421 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:13 smithi171 ceph-mon[22959]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-08-29T14:52:14.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: osdmap e26: 4 total, 4 up, 4 in 2024-08-29T14:52:14.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi027 2024-08-29T14:52:14.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi027 to 4016M 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:14.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:14 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: osdmap e26: 4 total, 4 up, 4 in 2024-08-29T14:52:14.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi027 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi027 to 4016M 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: osdmap e26: 4 total, 4 up, 4 in 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi027 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi027 to 4016M 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:14.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:14.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:14.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:14 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:15.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[18892]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:15.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:52:15.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:52:15.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[29342]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:52:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:52:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:15 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:15.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:15 smithi171 ceph-mon[22959]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:15.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:15 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:52:15.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:15 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:52:15.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:15 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:16 smithi171 ceph-mon[22959]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:52:16.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:16 smithi027 ceph-mon[18892]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:52:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:16 smithi027 ceph-mon[29342]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:52:17.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:17 smithi171 ceph-mon[22959]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:17 smithi027 ceph-mon[18892]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:17.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:17 smithi027 ceph-mon[29342]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:18 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/1338479127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]: dispatch 2024-08-29T14:52:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:18 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]: dispatch 2024-08-29T14:52:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:18 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]': finished 2024-08-29T14:52:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:18 smithi171 ceph-mon[22959]: osdmap e27: 5 total, 4 up, 5 in 2024-08-29T14:52:18.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:18 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:18.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:18 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/2852343246' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:52:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/1338479127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]: dispatch 2024-08-29T14:52:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]: dispatch 2024-08-29T14:52:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]': finished 2024-08-29T14:52:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[29342]: osdmap e27: 5 total, 4 up, 5 in 2024-08-29T14:52:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:18.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/2852343246' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:52:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/1338479127' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]: dispatch 2024-08-29T14:52:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]: dispatch 2024-08-29T14:52:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adb26953-31d6-491f-8777-02fbb879b3a6"}]': finished 2024-08-29T14:52:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[18892]: osdmap e27: 5 total, 4 up, 5 in 2024-08-29T14:52:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:18 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/2852343246' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:52:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:19 smithi171 ceph-mon[22959]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:19.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:19 smithi027 ceph-mon[29342]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:19 smithi027 ceph-mon[18892]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:20.649 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:20 smithi171 ceph-mon[22959]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:20.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:20 smithi027 ceph-mon[18892]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:20 smithi027 ceph-mon[29342]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:22 smithi171 ceph-mon[22959]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:22 smithi027 ceph-mon[18892]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:22.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:22 smithi027 ceph-mon[29342]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-29T14:52:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:23 smithi171 ceph-mon[22959]: Deploying daemon osd.4 on smithi171 2024-08-29T14:52:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-29T14:52:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:23 smithi027 ceph-mon[18892]: Deploying daemon osd.4 on smithi171 2024-08-29T14:52:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-29T14:52:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:23 smithi027 ceph-mon[29342]: Deploying daemon osd.4 on smithi171 2024-08-29T14:52:24.698 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:24 smithi171 ceph-mon[22959]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:24.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:24 smithi027 ceph-mon[18892]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:24.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:24 smithi027 ceph-mon[29342]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:26 smithi171 ceph-mon[22959]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:26.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:26 smithi027 ceph-mon[18892]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:26 smithi027 ceph-mon[29342]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:27 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:27 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:28.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:28.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:28.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:27 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:29.234 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:28 smithi171 ceph-mon[22959]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:28 smithi027 ceph-mon[18892]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:28 smithi027 ceph-mon[29342]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:30.652 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:30 smithi171 ceph-mon[22959]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:30.687 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 4 on host 'smithi171' 2024-08-29T14:52:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:30 smithi027 ceph-mon[18892]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:30.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:30 smithi027 ceph-mon[29342]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:31.689 DEBUG:teuthology.orchestra.run.smithi171:osd.4> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.4.service 2024-08-29T14:52:31.691 INFO:tasks.cephadm:Deploying osd.5 on smithi171 with /dev/vg_nvme/lv_3... 2024-08-29T14:52:31.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-29T14:52:31.928 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:52:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:31 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:31 smithi171 ceph-mon[22959]: from='osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-29T14:52:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:31 smithi171 ceph-mon[22959]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-29T14:52:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[18892]: from='osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-29T14:52:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[18892]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-29T14:52:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[29342]: from='osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-29T14:52:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:31 smithi027 ceph-mon[29342]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: osdmap e28: 5 total, 4 up, 5 in 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: from='osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:32.955 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:32 smithi171 ceph-mon[22959]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-29T14:52:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: osdmap e28: 5 total, 4 up, 5 in 2024-08-29T14:52:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: from='osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[18892]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: osdmap e28: 5 total, 4 up, 5 in 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: from='osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:33.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:32 smithi027 ceph-mon[29342]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-08-29T14:52:33.207 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T14:52:33.156+0000 7fe9d2656640 -1 osd.4 0 waiting for initial osdmap 2024-08-29T14:52:33.207 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T14:52:33.159+0000 7fe9ca477640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: osdmap e29: 5 total, 4 up, 5 in 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: from='osd.4 ' entity='osd.4' 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111] boot 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: osdmap e30: 5 total, 5 up, 5 in 2024-08-29T14:52:33.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: osdmap e29: 5 total, 4 up, 5 in 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: from='osd.4 ' entity='osd.4' 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111] boot 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: osdmap e30: 5 total, 5 up, 5 in 2024-08-29T14:52:33.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:33 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:52:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: osdmap e29: 5 total, 4 up, 5 in 2024-08-29T14:52:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: from='osd.4 ' entity='osd.4' 2024-08-29T14:52:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: osd.4 [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111] boot 2024-08-29T14:52:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: osdmap e30: 5 total, 5 up, 5 in 2024-08-29T14:52:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:33 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:52:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:34 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:52:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:34 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:52:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:34 smithi171 ceph-mon[22959]: osdmap e31: 5 total, 5 up, 5 in 2024-08-29T14:52:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:34 smithi171 ceph-mon[22959]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:52:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:52:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[29342]: osdmap e31: 5 total, 5 up, 5 in 2024-08-29T14:52:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[29342]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:52:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:52:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[18892]: osdmap e31: 5 total, 5 up, 5 in 2024-08-29T14:52:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:34 smithi027 ceph-mon[18892]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:35.380 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:52:35.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_3 2024-08-29T14:52:35.909 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:52:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:36 smithi171 ceph-mon[22959]: osdmap e32: 5 total, 5 up, 5 in 2024-08-29T14:52:36.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:36 smithi027 ceph-mon[18892]: osdmap e32: 5 total, 5 up, 5 in 2024-08-29T14:52:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:36 smithi027 ceph-mon[29342]: osdmap e32: 5 total, 5 up, 5 in 2024-08-29T14:52:37.657 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:37 smithi171 ceph-mon[22959]: osdmap e33: 5 total, 5 up, 5 in 2024-08-29T14:52:37.657 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:37 smithi171 ceph-mon[22959]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:37.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:37 smithi027 ceph-mon[29342]: osdmap e33: 5 total, 5 up, 5 in 2024-08-29T14:52:37.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:37 smithi027 ceph-mon[29342]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:37 smithi027 ceph-mon[18892]: osdmap e33: 5 total, 5 up, 5 in 2024-08-29T14:52:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:37 smithi027 ceph-mon[18892]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:38.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: osdmap e34: 5 total, 5 up, 5 in 2024-08-29T14:52:38.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:38.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:52:38.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:38 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: osdmap e34: 5 total, 5 up, 5 in 2024-08-29T14:52:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:38.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:38.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:38.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:38.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:52:38.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:52:38.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: osdmap e34: 5 total, 5 up, 5 in 2024-08-29T14:52:38.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:52:38.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:38.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:38.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:38.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:52:38.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:38.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:52:38.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:52:38.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:38 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:39.592 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:39 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi171 2024-08-29T14:52:39.592 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:39 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi171 to 17088M 2024-08-29T14:52:39.592 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:39 smithi171 ceph-mon[22959]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:52:39.592 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:39 smithi171 ceph-mon[22959]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:39.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi171 2024-08-29T14:52:39.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi171 to 17088M 2024-08-29T14:52:39.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[18892]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:52:39.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[18892]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:39.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi171 2024-08-29T14:52:39.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi171 to 17088M 2024-08-29T14:52:39.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[29342]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:52:39.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:39 smithi027 ceph-mon[29342]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 519 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:41 smithi171 ceph-mon[22959]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:52:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:41 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/3405277754' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]: dispatch 2024-08-29T14:52:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:41 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]: dispatch 2024-08-29T14:52:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:41 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]': finished 2024-08-29T14:52:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:41 smithi171 ceph-mon[22959]: osdmap e35: 6 total, 5 up, 6 in 2024-08-29T14:52:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:41 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:41 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/434722244' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:52:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[18892]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:52:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/3405277754' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]: dispatch 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]: dispatch 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]': finished 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[18892]: osdmap e35: 6 total, 5 up, 6 in 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/434722244' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[29342]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/3405277754' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]: dispatch 2024-08-29T14:52:41.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]: dispatch 2024-08-29T14:52:41.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b6e927d-bca5-4fe8-b17f-7b83a872afad"}]': finished 2024-08-29T14:52:41.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[29342]: osdmap e35: 6 total, 5 up, 6 in 2024-08-29T14:52:41.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:41.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:41 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/434722244' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:52:43.717 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:43 smithi171 ceph-mon[22959]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:52:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:43 smithi027 ceph-mon[18892]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:52:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:43 smithi027 ceph-mon[29342]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:52:45.592 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:45 smithi171 ceph-mon[22959]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:52:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:45 smithi027 ceph-mon[18892]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:52:45.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:45 smithi027 ceph-mon[29342]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:52:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:47 smithi171 ceph-mon[22959]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-29T14:52:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:47 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-29T14:52:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:47 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:47 smithi027 ceph-mon[18892]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-29T14:52:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:47 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-29T14:52:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:47 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:47 smithi027 ceph-mon[29342]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-29T14:52:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:47 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-29T14:52:47.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:47 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:52:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:48 smithi171 ceph-mon[22959]: Deploying daemon osd.5 on smithi171 2024-08-29T14:52:48.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:48 smithi027 ceph-mon[18892]: Deploying daemon osd.5 on smithi171 2024-08-29T14:52:48.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:48 smithi027 ceph-mon[29342]: Deploying daemon osd.5 on smithi171 2024-08-29T14:52:49.657 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:49 smithi171 ceph-mon[22959]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:52:49.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:49 smithi027 ceph-mon[18892]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:52:49.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:49 smithi027 ceph-mon[29342]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:52:50.735 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:50 smithi171 ceph-mon[22959]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:50.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:50 smithi027 ceph-mon[18892]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:50.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:50 smithi027 ceph-mon[29342]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:51 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:51 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:51 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:51 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:51 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:51 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:51 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:52:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:51 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:51 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:53.004 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:52 smithi171 ceph-mon[22959]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:53.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:52 smithi027 ceph-mon[29342]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:52 smithi027 ceph-mon[18892]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:54.597 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:54 smithi171 ceph-mon[22959]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:54.628 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 5 on host 'smithi171' 2024-08-29T14:52:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:54 smithi027 ceph-mon[18892]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:54 smithi027 ceph-mon[29342]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:55.234 DEBUG:teuthology.orchestra.run.smithi171:osd.5> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.5.service 2024-08-29T14:52:55.237 INFO:tasks.cephadm:Deploying osd.6 on smithi171 with /dev/vg_nvme/lv_2... 2024-08-29T14:52:55.237 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-29T14:52:55.485 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:52:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[18892]: from='osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-29T14:52:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[18892]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[29342]: from='osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[29342]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:55 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.961 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.961 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.961 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:55 smithi171 ceph-mon[22959]: from='osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-29T14:52:55.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:55 smithi171 ceph-mon[22959]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-29T14:52:55.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:55.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:55 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:52:56.961 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-mon[22959]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-29T14:52:56.961 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-mon[22959]: osdmap e36: 6 total, 5 up, 6 in 2024-08-29T14:52:56.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:56.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-mon[22959]: from='osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:56.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-mon[22959]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:56.962 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-mon[22959]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:56.962 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[33336]: 2024-08-29T14:52:56.638+0000 7fb2ae599640 -1 osd.5 0 waiting for initial osdmap 2024-08-29T14:52:56.962 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 14:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[33336]: 2024-08-29T14:52:56.643+0000 7fb2aa3c2640 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:52:57.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[29342]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-29T14:52:57.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[29342]: osdmap e36: 6 total, 5 up, 6 in 2024-08-29T14:52:57.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[29342]: from='osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[29342]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[29342]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[18892]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[18892]: osdmap e36: 6 total, 5 up, 6 in 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[18892]: from='osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[18892]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:52:57.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:56 smithi027 ceph-mon[18892]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-08-29T14:52:57.907 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:57 smithi171 ceph-mon[22959]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:52:57.908 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:57 smithi171 ceph-mon[22959]: osdmap e37: 6 total, 5 up, 6 in 2024-08-29T14:52:57.908 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:57 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:57.908 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:57 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:57.908 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:57 smithi171 ceph-mon[22959]: osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519] boot 2024-08-29T14:52:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[18892]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:52:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[18892]: osdmap e37: 6 total, 5 up, 6 in 2024-08-29T14:52:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:58.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[18892]: osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519] boot 2024-08-29T14:52:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[29342]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:52:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[29342]: osdmap e37: 6 total, 5 up, 6 in 2024-08-29T14:52:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:57 smithi027 ceph-mon[29342]: osd.5 [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519] boot 2024-08-29T14:52:58.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:58 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:52:58.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:58 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:52:58.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:58 smithi171 ceph-mon[22959]: osdmap e38: 6 total, 6 up, 6 in 2024-08-29T14:52:58.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:58 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:58.739 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:58 smithi171 ceph-mon[22959]: pgmap v129: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:52:59.025 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:52:59.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_2 2024-08-29T14:52:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:52:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:52:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[18892]: osdmap e38: 6 total, 6 up, 6 in 2024-08-29T14:52:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[18892]: pgmap v129: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:52:59.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:52:59.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:52:59.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[29342]: osdmap e38: 6 total, 6 up, 6 in 2024-08-29T14:52:59.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:52:59.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:58 smithi027 ceph-mon[29342]: pgmap v129: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:52:59.527 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:52:59.903 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:52:59 smithi171 ceph-mon[22959]: osdmap e39: 6 total, 6 up, 6 in 2024-08-29T14:53:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:52:59 smithi027 ceph-mon[18892]: osdmap e39: 6 total, 6 up, 6 in 2024-08-29T14:53:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:52:59 smithi027 ceph-mon[29342]: osdmap e39: 6 total, 6 up, 6 in 2024-08-29T14:53:00.978 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:00 smithi171 ceph-mon[22959]: osdmap e40: 6 total, 6 up, 6 in 2024-08-29T14:53:00.978 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:00 smithi171 ceph-mon[22959]: pgmap v132: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:01.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:00 smithi027 ceph-mon[18892]: osdmap e40: 6 total, 6 up, 6 in 2024-08-29T14:53:01.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:00 smithi027 ceph-mon[18892]: pgmap v132: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:00 smithi027 ceph-mon[29342]: osdmap e40: 6 total, 6 up, 6 in 2024-08-29T14:53:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:00 smithi027 ceph-mon[29342]: pgmap v132: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: osdmap e41: 6 total, 6 up, 6 in 2024-08-29T14:53:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi171 2024-08-29T14:53:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi171 to 8544M 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:01 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: osdmap e41: 6 total, 6 up, 6 in 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi171 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi171 to 8544M 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:02.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:02.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:02.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: osdmap e41: 6 total, 6 up, 6 in 2024-08-29T14:53:02.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi171 2024-08-29T14:53:02.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi171 to 8544M 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:02.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:01 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:02 smithi171 ceph-mon[22959]: osdmap e42: 6 total, 6 up, 6 in 2024-08-29T14:53:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:02 smithi171 ceph-mon[22959]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:02 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:53:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:02 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:53:02.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:02 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:02.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:02 smithi171 ceph-mon[22959]: pgmap v135: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[18892]: osdmap e42: 6 total, 6 up, 6 in 2024-08-29T14:53:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[18892]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[18892]: pgmap v135: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[29342]: osdmap e42: 6 total, 6 up, 6 in 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[29342]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:03.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:02 smithi027 ceph-mon[29342]: pgmap v135: 1 pgs: 1 unknown; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:04 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/3154029620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]: dispatch 2024-08-29T14:53:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:04 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]: dispatch 2024-08-29T14:53:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:04 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]': finished 2024-08-29T14:53:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:04 smithi171 ceph-mon[22959]: osdmap e43: 7 total, 6 up, 7 in 2024-08-29T14:53:04.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:04 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/3154029620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]: dispatch 2024-08-29T14:53:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]: dispatch 2024-08-29T14:53:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]': finished 2024-08-29T14:53:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[18892]: osdmap e43: 7 total, 6 up, 7 in 2024-08-29T14:53:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/3154029620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]: dispatch 2024-08-29T14:53:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]: dispatch 2024-08-29T14:53:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3c5992d-70c0-4893-9221-b8bc5268d92c"}]': finished 2024-08-29T14:53:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[29342]: osdmap e43: 7 total, 6 up, 7 in 2024-08-29T14:53:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:04 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:05 smithi171 ceph-mon[22959]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-08-29T14:53:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:05 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/3886319564' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:53:05.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:05 smithi027 ceph-mon[18892]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-08-29T14:53:05.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:05 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/3886319564' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:53:05.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:05 smithi027 ceph-mon[29342]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 120 KiB/s, 0 objects/s recovering 2024-08-29T14:53:05.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:05 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/3886319564' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:53:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:06 smithi027 ceph-mon[18892]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:06.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:06 smithi027 ceph-mon[29342]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:06.894 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:06 smithi171 ceph-mon[22959]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:08.667 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:08 smithi171 ceph-mon[22959]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-08-29T14:53:08.822 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:08 smithi027 ceph-mon[18892]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-08-29T14:53:08.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:08 smithi027 ceph-mon[29342]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-08-29T14:53:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-29T14:53:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-29T14:53:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-29T14:53:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:11.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:11 smithi027 ceph-mon[18892]: Deploying daemon osd.6 on smithi171 2024-08-29T14:53:11.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:11 smithi027 ceph-mon[18892]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-08-29T14:53:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:11 smithi027 ceph-mon[29342]: Deploying daemon osd.6 on smithi171 2024-08-29T14:53:11.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:11 smithi027 ceph-mon[29342]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-08-29T14:53:11.615 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:11 smithi171 ceph-mon[22959]: Deploying daemon osd.6 on smithi171 2024-08-29T14:53:11.615 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:11 smithi171 ceph-mon[22959]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-08-29T14:53:12.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:12 smithi027 ceph-mon[18892]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:53:12.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:12 smithi027 ceph-mon[29342]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:53:12.833 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:12 smithi171 ceph-mon[22959]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:53:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:14 smithi171 ceph-mon[22959]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:14 smithi027 ceph-mon[18892]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:14.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:14 smithi027 ceph-mon[29342]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:15.787 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:15 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:15.788 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:15 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:15.788 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:15 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:15 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:15 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:15 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:15.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:15 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:15.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:15 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:15.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:15 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:16 smithi171 ceph-mon[22959]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:16.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:16 smithi027 ceph-mon[18892]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:16.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:16 smithi027 ceph-mon[29342]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:18.828 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 6 on host 'smithi171' 2024-08-29T14:53:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:19 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:19 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:19.614 DEBUG:teuthology.orchestra.run.smithi171:osd.6> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.6.service 2024-08-29T14:53:19.616 INFO:tasks.cephadm:Deploying osd.7 on smithi171 with /dev/vg_nvme/lv_1... 2024-08-29T14:53:19.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-29T14:53:20.666 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:53:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-mon[22959]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-29T14:53:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-mon[22959]: osdmap e44: 7 total, 6 up, 7 in 2024-08-29T14:53:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-mon[22959]: from='osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-mon[22959]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-mon[22959]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:20.768 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T14:53:20.519+0000 7f4beb0d2640 -1 osd.6 0 waiting for initial osdmap 2024-08-29T14:53:20.768 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 14:53:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T14:53:20.527+0000 7f4be6efb640 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:53:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[29342]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-29T14:53:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[29342]: osdmap e44: 7 total, 6 up, 7 in 2024-08-29T14:53:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[29342]: from='osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[29342]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[29342]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[18892]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-29T14:53:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[18892]: osdmap e44: 7 total, 6 up, 7 in 2024-08-29T14:53:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[18892]: from='osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[18892]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:20 smithi027 ceph-mon[18892]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-08-29T14:53:21.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:21 smithi171 ceph-mon[22959]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:53:21.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:21 smithi171 ceph-mon[22959]: osdmap e45: 7 total, 6 up, 7 in 2024-08-29T14:53:21.663 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:21 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:21.664 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:21 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[18892]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:53:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[18892]: osdmap e45: 7 total, 6 up, 7 in 2024-08-29T14:53:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:21.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[29342]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:53:21.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[29342]: osdmap e45: 7 total, 6 up, 7 in 2024-08-29T14:53:21.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:21.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:21 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:53:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:53:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[18892]: osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066] boot 2024-08-29T14:53:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[18892]: osdmap e46: 7 total, 7 up, 7 in 2024-08-29T14:53:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[18892]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[18892]: osdmap e47: 7 total, 7 up, 7 in 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[29342]: osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066] boot 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[29342]: osdmap e46: 7 total, 7 up, 7 in 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[29342]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:22 smithi027 ceph-mon[29342]: osdmap e47: 7 total, 7 up, 7 in 2024-08-29T14:53:22.896 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:22 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:53:22.896 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:22 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:53:22.896 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:22 smithi171 ceph-mon[22959]: osd.6 [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066] boot 2024-08-29T14:53:22.896 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:22 smithi171 ceph-mon[22959]: osdmap e46: 7 total, 7 up, 7 in 2024-08-29T14:53:22.897 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:22 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:53:22.897 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:22 smithi171 ceph-mon[22959]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:22.897 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:22 smithi171 ceph-mon[22959]: osdmap e47: 7 total, 7 up, 7 in 2024-08-29T14:53:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi171 2024-08-29T14:53:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi171 to 5696M 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:23 smithi171 ceph-mon[22959]: osdmap e48: 7 total, 7 up, 7 in 2024-08-29T14:53:24.288 INFO:teuthology.orchestra.run.smithi171.stdout: 2024-08-29T14:53:24.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi171:vg_nvme/lv_1 2024-08-29T14:53:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi171 2024-08-29T14:53:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi171 to 5696M 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[29342]: osdmap e48: 7 total, 7 up, 7 in 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi171 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi171 to 5696M 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:24.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:23 smithi027 ceph-mon[18892]: osdmap e48: 7 total, 7 up, 7 in 2024-08-29T14:53:24.506 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:53:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:25 smithi171 ceph-mon[22959]: osdmap e49: 7 total, 7 up, 7 in 2024-08-29T14:53:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:25 smithi171 ceph-mon[22959]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:25.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:25 smithi027 ceph-mon[18892]: osdmap e49: 7 total, 7 up, 7 in 2024-08-29T14:53:25.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:25 smithi027 ceph-mon[18892]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:25 smithi027 ceph-mon[29342]: osdmap e49: 7 total, 7 up, 7 in 2024-08-29T14:53:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:25 smithi027 ceph-mon[29342]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:26.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:26 smithi171 ceph-mon[22959]: osdmap e50: 7 total, 7 up, 7 in 2024-08-29T14:53:26.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:53:26.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:53:26.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:26 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[29342]: osdmap e50: 7 total, 7 up, 7 in 2024-08-29T14:53:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:53:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:53:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[18892]: osdmap e50: 7 total, 7 up, 7 in 2024-08-29T14:53:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-29T14:53:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-29T14:53:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:26 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:27.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:27 smithi171 ceph-mon[22959]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:27.757 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:27 smithi171 ceph-mon[22959]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:27 smithi027 ceph-mon[18892]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:27 smithi027 ceph-mon[18892]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:27.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:27 smithi027 ceph-mon[29342]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi171:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:27.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:27 smithi027 ceph-mon[29342]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:28 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/3679164488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]: dispatch 2024-08-29T14:53:28.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:28 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]: dispatch 2024-08-29T14:53:28.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:28 smithi171 ceph-mon[22959]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]': finished 2024-08-29T14:53:28.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:28 smithi171 ceph-mon[22959]: osdmap e51: 8 total, 7 up, 8 in 2024-08-29T14:53:28.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:28 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:28.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/3679164488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]: dispatch 2024-08-29T14:53:28.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]: dispatch 2024-08-29T14:53:28.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[29342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]': finished 2024-08-29T14:53:28.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[29342]: osdmap e51: 8 total, 7 up, 8 in 2024-08-29T14:53:28.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/3679164488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]: dispatch 2024-08-29T14:53:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]: dispatch 2024-08-29T14:53:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[18892]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3c1227a6-0c5b-4753-b5d3-f9e98383c3c3"}]': finished 2024-08-29T14:53:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[18892]: osdmap e51: 8 total, 7 up, 8 in 2024-08-29T14:53:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:28 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:29 smithi171 ceph-mon[22959]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-08-29T14:53:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:29 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/76877900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:53:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:29 smithi027 ceph-mon[29342]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-08-29T14:53:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:29 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/76877900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:53:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:29 smithi027 ceph-mon[18892]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-08-29T14:53:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:29 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/76877900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-29T14:53:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:31 smithi171 ceph-mon[22959]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:31 smithi027 ceph-mon[18892]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:31 smithi027 ceph-mon[29342]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:33.754 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:33 smithi171 ceph-mon[22959]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:53:33.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:33 smithi027 ceph-mon[18892]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:53:33.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:33 smithi027 ceph-mon[29342]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:53:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:35 smithi171 ceph-mon[22959]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-29T14:53:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:35 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-29T14:53:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:35 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:35 smithi027 ceph-mon[29342]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-29T14:53:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:35 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-29T14:53:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:35 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:35.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:35 smithi027 ceph-mon[18892]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-08-29T14:53:35.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:35 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-29T14:53:35.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:35 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:36 smithi171 ceph-mon[22959]: Deploying daemon osd.7 on smithi171 2024-08-29T14:53:36.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:36 smithi027 ceph-mon[18892]: Deploying daemon osd.7 on smithi171 2024-08-29T14:53:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:36 smithi027 ceph-mon[29342]: Deploying daemon osd.7 on smithi171 2024-08-29T14:53:37.742 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:37 smithi171 ceph-mon[22959]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:53:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:37 smithi027 ceph-mon[18892]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:53:37.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:37 smithi027 ceph-mon[29342]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-29T14:53:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:39 smithi171 ceph-mon[22959]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:39.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:39 smithi027 ceph-mon[18892]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:39.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:39 smithi027 ceph-mon[29342]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:40 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:40 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:40.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:40.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:40.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:40 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:41.755 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:41 smithi171 ceph-mon[22959]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:41 smithi027 ceph-mon[18892]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:41.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:41 smithi027 ceph-mon[29342]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:42.844 INFO:teuthology.orchestra.run.smithi171.stdout:Created osd(s) 7 on host 'smithi171' 2024-08-29T14:53:43.462 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.462 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.462 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:43.462 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:43.463 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.463 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:43.463 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-29T14:53:43.463 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-29T14:53:43.463 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:43.463 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.463 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:43 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.482 DEBUG:teuthology.orchestra.run.smithi171:osd.7> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.7.service 2024-08-29T14:53:43.485 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-08-29T14:53:43.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd stat -f json 2024-08-29T14:53:43.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:43.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:43.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:43.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:43.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-29T14:53:43.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-29T14:53:43.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:43.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:43 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:43.703 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:53:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:44 smithi171 ceph-mon[22959]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-29T14:53:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:44 smithi171 ceph-mon[22959]: osdmap e52: 8 total, 7 up, 8 in 2024-08-29T14:53:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:44 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:44 smithi171 ceph-mon[22959]: from='osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:44 smithi171 ceph-mon[22959]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:44.518 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 14:53:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T14:53:44.210+0000 7f5375959640 -1 osd.7 0 waiting for initial osdmap 2024-08-29T14:53:44.518 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 14:53:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T14:53:44.219+0000 7f5371782640 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T14:53:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[18892]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-29T14:53:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[18892]: osdmap e52: 8 total, 7 up, 8 in 2024-08-29T14:53:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[18892]: from='osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[18892]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[29342]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-29T14:53:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[29342]: osdmap e52: 8 total, 7 up, 8 in 2024-08-29T14:53:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[29342]: from='osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:44 smithi027 ceph-mon[29342]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]: dispatch 2024-08-29T14:53:45.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:53:45.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: osdmap e53: 8 total, 7 up, 8 in 2024-08-29T14:53:45.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687] boot 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: osdmap e54: 8 total, 8 up, 8 in 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[18892]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: osdmap e53: 8 total, 7 up, 8 in 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687] boot 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: osdmap e54: 8 total, 8 up, 8 in 2024-08-29T14:53:45.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:45 smithi027 ceph-mon[29342]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi171", "root=default"]}]': finished 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: osdmap e53: 8 total, 7 up, 8 in 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: osd.7 [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687] boot 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: osdmap e54: 8 total, 8 up, 8 in 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:53:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:45 smithi171 ceph-mon[22959]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-08-29T14:53:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:53:46.483 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1724943224,"num_in_osds":8,"osd_in_since":1724943208,"num_remapped_pgs":1} 2024-08-29T14:53:46.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T14:53:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: purged_snaps scrub starts 2024-08-29T14:53:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: purged_snaps scrub ok 2024-08-29T14:53:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: osdmap e55: 8 total, 8 up, 8 in 2024-08-29T14:53:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2626960388' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-29T14:53:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:46.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:46.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:46 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.729 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:53:46.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: purged_snaps scrub starts 2024-08-29T14:53:46.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: purged_snaps scrub ok 2024-08-29T14:53:46.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: osdmap e55: 8 total, 8 up, 8 in 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2626960388' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:46.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: purged_snaps scrub starts 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: purged_snaps scrub ok 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: osdmap e55: 8 total, 8 up, 8 in 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2626960388' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:46.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:46.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:46 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:47 smithi171 ceph-mon[22959]: Detected new or changed devices on smithi171 2024-08-29T14:53:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:47 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi171 to 4272M 2024-08-29T14:53:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:47 smithi171 ceph-mon[22959]: osdmap e56: 8 total, 8 up, 8 in 2024-08-29T14:53:47.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:47 smithi171 ceph-mon[22959]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[29342]: Detected new or changed devices on smithi171 2024-08-29T14:53:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi171 to 4272M 2024-08-29T14:53:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[29342]: osdmap e56: 8 total, 8 up, 8 in 2024-08-29T14:53:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[29342]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[18892]: Detected new or changed devices on smithi171 2024-08-29T14:53:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi171 to 4272M 2024-08-29T14:53:47.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[18892]: osdmap e56: 8 total, 8 up, 8 in 2024-08-29T14:53:47.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:47 smithi027 ceph-mon[18892]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:48.524 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:53:48.524 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":58,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T14:53:48.458438+0000","last_up_change":"2024-08-29T14:53:44.442031+0000","last_in_change":"2024-08-29T14:53:28.429586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6803","nonce":1938465338}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6805","nonce":1938465338}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6809","nonce":1938465338}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6807","nonce":1938465338}]},"public_addr":"172.21.15.27:6803/1938465338","cluster_addr":"172.21.15.27:6805/1938465338","heartbeat_back_addr":"172.21.15.27:6809/1938465338","heartbeat_front_addr":"172.21.15.27:6807/1938465338","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6811","nonce":2967503746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6813","nonce":2967503746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6817","nonce":2967503746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6815","nonce":2967503746}]},"public_addr":"172.21.15.27:6811/2967503746","cluster_addr":"172.21.15.27:6813/2967503746","heartbeat_back_addr":"172.21.15.27:6817/2967503746","heartbeat_front_addr":"172.21.15.27:6815/2967503746","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6819","nonce":302366028}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6821","nonce":302366028}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6825","nonce":302366028}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6823","nonce":302366028}]},"public_addr":"172.21.15.27:6819/302366028","cluster_addr":"172.21.15.27:6821/302366028","heartbeat_back_addr":"172.21.15.27:6825/302366028","heartbeat_front_addr":"172.21.15.27:6823/302366028","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6827","nonce":250460291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6829","nonce":250460291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6833","nonce":250460291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6831","nonce":250460291}]},"public_addr":"172.21.15.27:6827/250460291","cluster_addr":"172.21.15.27:6829/250460291","heartbeat_back_addr":"172.21.15.27:6833/250460291","heartbeat_front_addr":"172.21.15.27:6831/250460291","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6809","nonce":223130519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6811","nonce":223130519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6815","nonce":223130519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6813","nonce":223130519}]},"public_addr":"172.21.15.171:6809/223130519","cluster_addr":"172.21.15.171:6811/223130519","heartbeat_back_addr":"172.21.15.171:6815/223130519","heartbeat_front_addr":"172.21.15.171:6813/223130519","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T14:53:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:48 smithi171 ceph-mon[22959]: osdmap e57: 8 total, 8 up, 8 in 2024-08-29T14:53:48.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:48 smithi027 ceph-mon[18892]: osdmap e57: 8 total, 8 up, 8 in 2024-08-29T14:53:48.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:48 smithi027 ceph-mon[29342]: osdmap e57: 8 total, 8 up, 8 in 2024-08-29T14:53:49.325 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-08-29T14:51:48.913604+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-08-29T14:53:49.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-08-29T14:53:49.533 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:53:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:49 smithi027 ceph-mon[18892]: pgmap v173: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-29T14:53:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:49 smithi027 ceph-mon[18892]: osdmap e58: 8 total, 8 up, 8 in 2024-08-29T14:53:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:49 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2772542328' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:53:49.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:49 smithi027 ceph-mon[29342]: pgmap v173: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-29T14:53:49.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:49 smithi027 ceph-mon[29342]: osdmap e58: 8 total, 8 up, 8 in 2024-08-29T14:53:49.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:49 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2772542328' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:53:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:49 smithi171 ceph-mon[22959]: pgmap v173: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-29T14:53:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:49 smithi171 ceph-mon[22959]: osdmap e58: 8 total, 8 up, 8 in 2024-08-29T14:53:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:49 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2772542328' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:53:51.321 INFO:teuthology.orchestra.run.smithi027.stdout:pg_num: 1 2024-08-29T14:53:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:51 smithi171 ceph-mon[22959]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-08-29T14:53:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:51 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3606196985' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-29T14:53:51.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:51 smithi027 ceph-mon[18892]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-08-29T14:53:51.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:51 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3606196985' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-29T14:53:51.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:51 smithi027 ceph-mon[29342]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-08-29T14:53:51.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:51 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3606196985' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-29T14:53:51.913 INFO:tasks.cephadm:Adding prometheus.a on smithi171 2024-08-29T14:53:51.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch apply prometheus '1;smithi171=a' 2024-08-29T14:53:52.126 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:53:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:53 smithi171 ceph-mon[22959]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:53.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:53 smithi027 ceph-mon[18892]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:53.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:53 smithi027 ceph-mon[29342]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-29T14:53:53.899 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled prometheus update... 2024-08-29T14:53:54.529 DEBUG:teuthology.orchestra.run.smithi171:prometheus.a> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service 2024-08-29T14:53:54.531 INFO:tasks.cephadm:Adding node-exporter.a on smithi027 2024-08-29T14:53:54.531 INFO:tasks.cephadm:Adding node-exporter.b on smithi171 2024-08-29T14:53:54.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch apply node-exporter '2;smithi027=a;smithi171=b' 2024-08-29T14:53:54.780 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:53:55.243 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:55.243 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: Saving service prometheus spec with placement smithi171=a;count:1 2024-08-29T14:53:55.243 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:55.243 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:55.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:55.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:55.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:55.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:54 smithi171 ceph-mon[22959]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:53:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: Saving service prometheus spec with placement smithi171=a;count:1 2024-08-29T14:53:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[18892]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: Saving service prometheus spec with placement smithi171=a;count:1 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:53:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:55.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:54 smithi027 ceph-mon[29342]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-29T14:53:56.427 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:56 smithi171 ceph-mon[22959]: Deploying daemon prometheus.a on smithi171 2024-08-29T14:53:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:56 smithi027 ceph-mon[18892]: Deploying daemon prometheus.a on smithi171 2024-08-29T14:53:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:56 smithi027 ceph-mon[29342]: Deploying daemon prometheus.a on smithi171 2024-08-29T14:53:57.261 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled node-exporter update... 2024-08-29T14:53:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:57 smithi027 ceph-mon[18892]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:57 smithi027 ceph-mon[29342]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:57 smithi171 ceph-mon[22959]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:58.228 DEBUG:teuthology.orchestra.run.smithi027:node-exporter.a> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.a.service 2024-08-29T14:53:58.231 DEBUG:teuthology.orchestra.run.smithi171:node-exporter.b> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.b.service 2024-08-29T14:53:58.233 INFO:tasks.cephadm:Adding alertmanager.a on smithi027 2024-08-29T14:53:58.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch apply alertmanager '1;smithi027=a' 2024-08-29T14:53:58.475 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:53:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:58 smithi171 ceph-mon[22959]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi027=a;smithi171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:58 smithi171 ceph-mon[22959]: Saving service node-exporter spec with placement smithi027=a;smithi171=b;count:2 2024-08-29T14:53:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:58 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:58 smithi027 ceph-mon[18892]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi027=a;smithi171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:58 smithi027 ceph-mon[18892]: Saving service node-exporter spec with placement smithi027=a;smithi171=b;count:2 2024-08-29T14:53:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:58 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:58 smithi027 ceph-mon[29342]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi027=a;smithi171=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:53:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:58 smithi027 ceph-mon[29342]: Saving service node-exporter spec with placement smithi027=a;smithi171=b;count:2 2024-08-29T14:53:58.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:58 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:53:59.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:53:59 smithi027 ceph-mon[18892]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:59.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:53:59 smithi027 ceph-mon[29342]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:53:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:53:59 smithi171 ceph-mon[22959]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:00 smithi027 ceph-mon[18892]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:00.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:00 smithi027 ceph-mon[29342]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:00 smithi171 ceph-mon[22959]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:02.653 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:02 smithi171 ceph-mon[22959]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:02 smithi027 ceph-mon[18892]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:02 smithi027 ceph-mon[29342]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:03.138 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled alertmanager update... 2024-08-29T14:54:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:04 smithi171 ceph-mon[22959]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:54:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:04 smithi171 ceph-mon[22959]: Saving service alertmanager spec with placement smithi027=a;count:1 2024-08-29T14:54:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:04 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:04 smithi027 ceph-mon[18892]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:54:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:04 smithi027 ceph-mon[18892]: Saving service alertmanager spec with placement smithi027=a;count:1 2024-08-29T14:54:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:04 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:04.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:04 smithi027 ceph-mon[29342]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:54:04.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:04 smithi027 ceph-mon[29342]: Saving service alertmanager spec with placement smithi027=a;count:1 2024-08-29T14:54:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:04 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:04.588 DEBUG:teuthology.orchestra.run.smithi027:alertmanager.a> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@alertmanager.a.service 2024-08-29T14:54:04.591 INFO:tasks.cephadm:Adding grafana.a on smithi171 2024-08-29T14:54:04.591 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph orch apply grafana '1;smithi171=a' 2024-08-29T14:54:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:05 smithi171 ceph-mon[22959]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:05.349 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:05 smithi027 ceph-mon[18892]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:05.350 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:05 smithi027 ceph-mon[29342]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:06.574 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:54:06.702 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:06 smithi171 ceph-mon[22959]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:06 smithi027 ceph-mon[18892]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:06.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:06 smithi027 ceph-mon[29342]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:08.524 INFO:teuthology.orchestra.run.smithi171.stdout:Scheduled grafana update... 2024-08-29T14:54:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:08 smithi171 ceph-mon[22959]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:08 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:08 smithi027 ceph-mon[18892]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:08 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:08.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:08 smithi027 ceph-mon[29342]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:08.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:08 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:09.169 DEBUG:teuthology.orchestra.run.smithi171:grafana.a> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@grafana.a.service 2024-08-29T14:54:09.172 INFO:tasks.cephadm:Setting up client nodes... 2024-08-29T14:54:09.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-29T14:54:09.389 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:09.407 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:54:09 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:54:09.377Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-29T14:54:09.407 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:54:09 smithi171 bash[49895]: 232a8598fe9ddd33c65319bcdf1a06474377a87038d2b5ee2d23cac1b77da90d 2024-08-29T14:54:09.767 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:54:09 smithi171 systemd[1]: Started Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:54:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-mon[22959]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:54:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-mon[22959]: Saving service grafana spec with placement smithi171=a;count:1 2024-08-29T14:54:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-29T14:54:10.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: ignoring --setuser ceph since I am not root 2024-08-29T14:54:10.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: ignoring --setgroup ceph since I am not root 2024-08-29T14:54:10.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-mgr[24815]: -- 172.21.15.171:0/3542509860 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561b77c68d00 con 0x561b77c3c800 2024-08-29T14:54:10.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:10.623+0000 7f5f3aaec040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-29T14:54:10.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:10.681+0000 7f5f3aaec040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-29T14:54:10.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[29342]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:54:10.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[29342]: Saving service grafana spec with placement smithi171=a;count:1 2024-08-29T14:54:10.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[18892]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi171=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[18892]: Saving service grafana spec with placement smithi171=a;count:1 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setuser ceph since I am not root 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setgroup ceph since I am not root 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:10.627+0000 7f97664a6040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-29T14:54:10.814 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:10.688+0000 7f97664a6040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-29T14:54:11.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:10.851+0000 7f97664a6040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-29T14:54:11.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:10 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:10.921+0000 7f97664a6040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-29T14:54:11.173 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:10.826+0000 7f5f3aaec040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-29T14:54:11.173 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:10.892+0000 7f5f3aaec040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-29T14:54:11.173 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:11.034+0000 7f5f3aaec040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:54:11.173 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:11.171+0000 7f5f3aaec040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-29T14:54:11.474 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:11.079+0000 7f97664a6040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:54:11.475 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:11.230+0000 7f97664a6040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-29T14:54:11.483 INFO:teuthology.orchestra.run.smithi027.stdout:[client.0] 2024-08-29T14:54:11.483 INFO:teuthology.orchestra.run.smithi027.stdout: key = AQCTi9BmyUSDHBAAjJGkFRt8XxCOEu7JtDRTaQ== 2024-08-29T14:54:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-mon[22959]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-mon[22959]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-29T14:54:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-mon[22959]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-29T14:54:11.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:11.474+0000 7f5f3aaec040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-29T14:54:11.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:11.534+0000 7f5f3aaec040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-29T14:54:11.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:11.713+0000 7f5f3aaec040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-29T14:54:11.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-mon[18892]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:11.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-mon[18892]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-29T14:54:11.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-mon[18892]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-29T14:54:11.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-mon[29342]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:11.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-mon[29342]: from='mgr.14150 172.21.15.27:0/2967370568' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-29T14:54:11.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-mon[29342]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-29T14:54:11.774 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:11.564+0000 7f97664a6040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-29T14:54:11.774 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:11.628+0000 7f97664a6040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-29T14:54:12.049 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:11.822+0000 7f97664a6040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-29T14:54:12.070 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:54:12.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-08-29T14:54:12.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-08-29T14:54:12.109 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-29T14:54:12.321 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T14:54:12.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:12 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1750970444' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:12.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:12 smithi171 ceph-mon[22959]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:12.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:12 smithi171 ceph-mon[22959]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-29T14:54:12.695 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:12 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:12.612+0000 7f5f3aaec040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-29T14:54:12.695 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:12 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:12.678+0000 7f5f3aaec040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-29T14:54:12.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1750970444' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:12.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-mon[18892]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:12.741 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-mon[18892]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-29T14:54:12.741 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1750970444' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:12.741 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-mon[29342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:12.741 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-mon[29342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-29T14:54:13.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:12.739+0000 7f97664a6040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-29T14:54:13.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:12 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:12.808+0000 7f97664a6040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-29T14:54:13.663 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:13 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:13.395+0000 7f5f3aaec040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-29T14:54:13.664 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:13 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:13.518+0000 7f5f3aaec040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-29T14:54:13.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:13 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:13.561+0000 7f97664a6040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-29T14:54:13.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:13 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:13.681+0000 7f97664a6040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-29T14:54:13.918 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:13 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:13.661+0000 7f5f3aaec040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:54:13.918 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:13 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:13.729+0000 7f5f3aaec040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-29T14:54:13.918 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:13 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:13.880+0000 7f5f3aaec040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-29T14:54:14.119 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:13 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:13.829+0000 7f97664a6040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:54:14.119 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:13 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:13.893+0000 7f97664a6040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-29T14:54:14.119 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.047+0000 7f97664a6040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-29T14:54:14.120 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.118+0000 7f97664a6040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-29T14:54:14.171 INFO:teuthology.orchestra.run.smithi171.stdout:[client.1] 2024-08-29T14:54:14.171 INFO:teuthology.orchestra.run.smithi171.stdout: key = AQCWi9BmEUT+CRAAEAVVn2v3nn5/7ia3hqY+QQ== 2024-08-29T14:54:14.172 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:13 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:13.952+0000 7f5f3aaec040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-29T14:54:14.172 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:14.014+0000 7f5f3aaec040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-29T14:54:14.417 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-mon[18892]: from='client.? 172.21.15.171:0/4063963705' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:14.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-mon[18892]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:14.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-mon[18892]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-29T14:54:14.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-mon[29342]: from='client.? 172.21.15.171:0/4063963705' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:14.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-mon[29342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:14.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-mon[29342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-29T14:54:14.419 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.182+0000 7f97664a6040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-29T14:54:14.419 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.348+0000 7f97664a6040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-29T14:54:14.419 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.416+0000 7f97664a6040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-29T14:54:14.457 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-mon[22959]: from='client.? 172.21.15.171:0/4063963705' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:14.458 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-mon[22959]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-29T14:54:14.458 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-mon[22959]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-29T14:54:14.458 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:14.176+0000 7f5f3aaec040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-29T14:54:14.458 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:14.244+0000 7f5f3aaec040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-29T14:54:14.458 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:14.311+0000 7f5f3aaec040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-29T14:54:14.458 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:14.395+0000 7f5f3aaec040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-29T14:54:14.458 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:14.455+0000 7f5f3aaec040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-29T14:54:14.696 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.485+0000 7f97664a6040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-29T14:54:14.696 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.571+0000 7f97664a6040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-29T14:54:14.696 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.633+0000 7f97664a6040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-29T14:54:14.696 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:14.694+0000 7f97664a6040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-29T14:54:14.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:14.516+0000 7f5f3aaec040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-29T14:54:14.800 DEBUG:teuthology.orchestra.run.smithi171:> set -ex 2024-08-29T14:54:14.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-08-29T14:54:14.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-08-29T14:54:14.839 INFO:tasks.cephadm:Creating RBD pool 2024-08-29T14:54:14.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool create rbd 8 2024-08-29T14:54:15.042 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:15.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[18892]: Standby manager daemon x restarted 2024-08-29T14:54:15.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[18892]: Standby manager daemon x started 2024-08-29T14:54:15.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:54:15.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:54:15.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:54:15.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:54:15.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[29342]: Standby manager daemon x restarted 2024-08-29T14:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[29342]: Standby manager daemon x started 2024-08-29T14:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:54:15.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:54:15.722 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:15.538+0000 7f97664a6040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-29T14:54:15.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:15] ENGINE Bus STARTING 2024-08-29T14:54:15.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: CherryPy Checker: 2024-08-29T14:54:15.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: The Application mounted at '' has an empty config. 2024-08-29T14:54:15.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:15.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:15] ENGINE Serving on http://:::9283 2024-08-29T14:54:15.723 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:15] ENGINE Bus STARTED 2024-08-29T14:54:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-mon[22959]: Standby manager daemon x restarted 2024-08-29T14:54:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-mon[22959]: Standby manager daemon x started 2024-08-29T14:54:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:54:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:54:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/1884558816' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:15.295+0000 7f5f3aaec040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: [29/Aug/2024:14:54:15] ENGINE Bus STARTING 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: CherryPy Checker: 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: The Application mounted at '' has an empty config. 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: [29/Aug/2024:14:54:15] ENGINE Serving on http://:::9283 2024-08-29T14:54:15.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:54:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[24811]: [29/Aug/2024:14:54:15] ENGINE Bus STARTED 2024-08-29T14:54:16.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-29T14:54:16.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: Active manager daemon y restarted 2024-08-29T14:54:16.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: Activating manager daemon y 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: osdmap e59: 8 total, 8 up, 8 in 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: mgrmap e18: y(active, starting, since 0.00603672s), standbys: x 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:54:16.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: Manager daemon y is now available 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:54:16.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: Active manager daemon y restarted 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: Activating manager daemon y 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: osdmap e59: 8 total, 8 up, 8 in 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: mgrmap e18: y(active, starting, since 0.00603672s), standbys: x 2024-08-29T14:54:16.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:54:16.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: Manager daemon y is now available 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:54:16.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: Active manager daemon y restarted 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: Activating manager daemon y 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: osdmap e59: 8 total, 8 up, 8 in 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: mgrmap e18: y(active, starting, since 0.00603672s), standbys: x 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:54:16.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: Manager daemon y is now available 2024-08-29T14:54:16.674 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:54:16.675 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:16.675 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:16.675 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:54:16.675 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:54:17.560 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'rbd' created 2024-08-29T14:54:17.591 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: mgrmap e19: y(active, since 1.01378s), standbys: x 2024-08-29T14:54:17.591 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3097548986' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:54:17] ENGINE Bus STARTING 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:54:17] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:54:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:54:17.592 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:54:17] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:54:17.593 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:54:17] ENGINE Bus STARTED 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: mgrmap e19: y(active, since 1.01378s), standbys: x 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3097548986' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:54:17] ENGINE Bus STARTING 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:54:17] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:54:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:54:17.594 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:54:17.595 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:54:17] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:54:17] ENGINE Bus STARTED 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:17.325+0000 7f96c421f640 -1 log_channel(cephadm) log [ERR] : [29/Aug/2024:14:54:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: Traceback (most recent call last): 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:54:17.596 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._connections.run(self.expiration_interval) 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._run(expiration_interval) 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: s = self.context.wrap_socket( 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: return self.sslsocket_class._create( 2024-08-29T14:54:17.597 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:54:17.598 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self.do_handshake() 2024-08-29T14:54:17.598 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:54:17.598 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: self._sslobj.do_handshake() 2024-08-29T14:54:17.598 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:54:17.598 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:54:17.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: mgrmap e19: y(active, since 1.01378s), standbys: x 2024-08-29T14:54:17.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3097548986' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-29T14:54:17.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-29T14:54:17.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:54:17] ENGINE Bus STARTING 2024-08-29T14:54:17.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:54:17] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:54:17.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:54:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:54:17.677 INFO:journalctl@ceph.mon.b.smithi171.stdout: Traceback (most recent call last): 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._run(expiration_interval) 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: s = self.context.wrap_socket( 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: return self.sslsocket_class._create( 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:54:17.678 INFO:journalctl@ceph.mon.b.smithi171.stdout: self.do_handshake() 2024-08-29T14:54:17.679 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:54:17.679 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._sslobj.do_handshake() 2024-08-29T14:54:17.679 INFO:journalctl@ceph.mon.b.smithi171.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:54:17.679 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:54:17] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:54:17.679 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:54:17] ENGINE Bus STARTED 2024-08-29T14:54:17.856 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:54:17] "GET /metrics HTTP/1.1" 200 23677 "" "Prometheus/2.43.0" 2024-08-29T14:54:18.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- rbd pool init rbd 2024-08-29T14:54:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-29T14:54:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: osdmap e60: 8 total, 8 up, 8 in 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi171 to 4016M 2024-08-29T14:54:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: osdmap e60: 8 total, 8 up, 8 in 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi171 to 4016M 2024-08-29T14:54:18.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:18 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.571 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-29T14:54:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: osdmap e60: 8 total, 8 up, 8 in 2024-08-29T14:54:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi171 to 4016M 2024-08-29T14:54:18.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:18 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:19.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: osdmap e61: 8 total, 8 up, 8 in 2024-08-29T14:54:19.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:19.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:19.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: osdmap e61: 8 total, 8 up, 8 in 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:54:19.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:54:19.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:54:19.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:54:19.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:54:19.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:19 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:54:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: osdmap e61: 8 total, 8 up, 8 in 2024-08-29T14:54:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:54:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:54:20.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:54:20.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:54:20.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:54:20.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:19 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:54:20.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:20.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: osdmap e62: 8 total, 8 up, 8 in 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: Deploying daemon alertmanager.a on smithi027 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-29T14:54:20.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/662893200' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: osdmap e62: 8 total, 8 up, 8 in 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:20.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T14:54:20.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: Deploying daemon alertmanager.a on smithi027 2024-08-29T14:54:20.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-29T14:54:20.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:20 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/662893200' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-29T14:54:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: osdmap e62: 8 total, 8 up, 8 in 2024-08-29T14:54:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:54:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:54:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: Deploying daemon alertmanager.a on smithi027 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-29T14:54:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:20 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/662893200' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-29T14:54:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:22 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/662893200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-29T14:54:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:22 smithi171 ceph-mon[22959]: osdmap e63: 8 total, 8 up, 8 in 2024-08-29T14:54:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:22 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/662893200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-29T14:54:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:22 smithi027 ceph-mon[18892]: osdmap e63: 8 total, 8 up, 8 in 2024-08-29T14:54:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:22 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/662893200' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-29T14:54:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:22 smithi027 ceph-mon[29342]: osdmap e63: 8 total, 8 up, 8 in 2024-08-29T14:54:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:23 smithi171 ceph-mon[22959]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:23 smithi171 ceph-mon[22959]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T14:54:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:23 smithi171 ceph-mon[22959]: Cluster is now healthy 2024-08-29T14:54:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:23 smithi171 ceph-mon[22959]: osdmap e64: 8 total, 8 up, 8 in 2024-08-29T14:54:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[18892]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[18892]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T14:54:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[18892]: Cluster is now healthy 2024-08-29T14:54:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[18892]: osdmap e64: 8 total, 8 up, 8 in 2024-08-29T14:54:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[29342]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[29342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T14:54:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[29342]: Cluster is now healthy 2024-08-29T14:54:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:23 smithi027 ceph-mon[29342]: osdmap e64: 8 total, 8 up, 8 in 2024-08-29T14:54:24.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph config log 1 --format=json 2024-08-29T14:54:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:24 smithi171 ceph-mon[22959]: osdmap e65: 8 total, 8 up, 8 in 2024-08-29T14:54:24.542 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:24 smithi027 ceph-mon[29342]: osdmap e65: 8 total, 8 up, 8 in 2024-08-29T14:54:24.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:24 smithi027 ceph-mon[18892]: osdmap e65: 8 total, 8 up, 8 in 2024-08-29T14:54:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:25 smithi171 ceph-mon[22959]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-08-29T14:54:25.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:25 smithi027 ceph-mon[18892]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-08-29T14:54:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:25 smithi027 ceph-mon[29342]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-08-29T14:54:27.074 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:27.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:27 smithi027 ceph-mon[18892]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-29T14:54:27.505 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:27 smithi027 ceph-mon[29342]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-29T14:54:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:27 smithi171 ceph-mon[22959]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-29T14:54:27.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:27 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:54:27] "GET /metrics HTTP/1.1" 200 23677 "" "Prometheus/2.43.0" 2024-08-29T14:54:27.818 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:27 smithi027 systemd[1]: Starting Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:54:28.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 podman[58225]: 2024-08-29 14:54:28.303608448 +0000 UTC m=+0.346026903 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-29T14:54:28.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 podman[58225]: 2024-08-29 14:54:28.442879368 +0000 UTC m=+0.485297808 volume create 6c8773304c2d3b76b41f0ec73148ad90c4bc75dcb5fe6ec45d1a79501964d313 2024-08-29T14:54:28.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 podman[58225]: 2024-08-29 14:54:28.518180907 +0000 UTC m=+0.560599347 container create 7493c9a457ffb4e132b7647f65c69f88683ec7b06102b3ec8fa05b1d191f2fb3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 podman[58225]: 2024-08-29 14:54:28.862688176 +0000 UTC m=+0.905106616 container init 7493c9a457ffb4e132b7647f65c69f88683ec7b06102b3ec8fa05b1d191f2fb3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 podman[58225]: 2024-08-29 14:54:28.865562541 +0000 UTC m=+0.907980989 container start 7493c9a457ffb4e132b7647f65c69f88683ec7b06102b3ec8fa05b1d191f2fb3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.877Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.878Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.879Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.880Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.903Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.903Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-29T14:54:28.970 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.906Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-29T14:54:28.971 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:28.906Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-29T14:54:29.233 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 bash[58225]: 7493c9a457ffb4e132b7647f65c69f88683ec7b06102b3ec8fa05b1d191f2fb3 2024-08-29T14:54:29.233 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:28 smithi027 systemd[1]: Started Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:54:29.233 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:29] ENGINE Bus STOPPING 2024-08-29T14:54:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 324 B/s wr, 18 op/s 2024-08-29T14:54:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 324 B/s wr, 18 op/s 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-29T14:54:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.637 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:54:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 324 B/s wr, 18 op/s 2024-08-29T14:54:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:29.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-29T14:54:29.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:30.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:54:30.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:29] ENGINE Bus STOPPED 2024-08-29T14:54:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:29] ENGINE Bus STARTING 2024-08-29T14:54:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:29] ENGINE Serving on http://:::9283 2024-08-29T14:54:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:54:29] ENGINE Bus STARTED 2024-08-29T14:54:30.407 INFO:teuthology.orchestra.run.smithi027.stdout:[{"version":21,"timestamp":"2024-08-29T14:54:29.111589+0000","name":"","changes":[{"name":"mgr/mgr/dashboard/GRAFANA_API_SSL_VERIFY","new_value":"false"}]}] 2024-08-29T14:54:30.407 INFO:tasks.ceph_manager:config epoch is 21 2024-08-29T14:54:30.407 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-29T14:54:30.407 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-29T14:54:30.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph mgr dump --format=json 2024-08-29T14:54:30.621 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:30.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[18892]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-29T14:54:30.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[18892]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-29T14:54:30.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[18892]: Deploying daemon grafana.a on smithi171 2024-08-29T14:54:30.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[18892]: osdmap e66: 8 total, 8 up, 8 in 2024-08-29T14:54:30.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2721708770' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-29T14:54:30.723 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[29342]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-29T14:54:30.724 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[29342]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-29T14:54:30.724 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[29342]: Deploying daemon grafana.a on smithi171 2024-08-29T14:54:30.724 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[29342]: osdmap e66: 8 total, 8 up, 8 in 2024-08-29T14:54:30.724 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2721708770' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-29T14:54:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:30 smithi171 ceph-mon[22959]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-29T14:54:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:30 smithi171 ceph-mon[22959]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-29T14:54:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:30 smithi171 ceph-mon[22959]: Deploying daemon grafana.a on smithi171 2024-08-29T14:54:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:30 smithi171 ceph-mon[22959]: osdmap e66: 8 total, 8 up, 8 in 2024-08-29T14:54:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:30 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2721708770' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-29T14:54:31.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:30 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:30.880Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000556643s 2024-08-29T14:54:31.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:31 smithi027 ceph-mon[29342]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 280 B/s wr, 0 op/s 2024-08-29T14:54:31.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:31 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:31.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:31 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:31.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:31 smithi027 ceph-mon[18892]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 280 B/s wr, 0 op/s 2024-08-29T14:54:31.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:31 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:31.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:31 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:31 smithi171 ceph-mon[22959]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 280 B/s wr, 0 op/s 2024-08-29T14:54:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:31 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:31 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:32.493 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:54:33.348 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":21,"flags":0,"active_gid":14403,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2650682884},{"type":"v1","addr":"172.21.15.27:6801","nonce":2650682884}]},"active_addr":"172.21.15.27:6801/2650682884","active_change":"2024-08-29T14:54:15.540026+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":2955701830}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":921286138}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":1523003109}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":2657305076}]}]} 2024-08-29T14:54:33.355 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-29T14:54:33.355 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-29T14:54:33.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T14:54:33.565 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:33.686 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:33 smithi027 ceph-mon[18892]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 246 B/s wr, 0 op/s 2024-08-29T14:54:33.686 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:33 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2224471942' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-29T14:54:33.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:33 smithi027 ceph-mon[29342]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 246 B/s wr, 0 op/s 2024-08-29T14:54:33.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:33 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2224471942' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-29T14:54:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:33 smithi171 ceph-mon[22959]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 246 B/s wr, 0 op/s 2024-08-29T14:54:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:33 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2224471942' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-29T14:54:35.322 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:54:35.322 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":66,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T14:54:29.448984+0000","last_up_change":"2024-08-29T14:53:44.442031+0000","last_in_change":"2024-08-29T14:53:28.429586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6803","nonce":1938465338}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6805","nonce":1938465338}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6809","nonce":1938465338}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6807","nonce":1938465338}]},"public_addr":"172.21.15.27:6803/1938465338","cluster_addr":"172.21.15.27:6805/1938465338","heartbeat_back_addr":"172.21.15.27:6809/1938465338","heartbeat_front_addr":"172.21.15.27:6807/1938465338","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6811","nonce":2967503746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6813","nonce":2967503746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6817","nonce":2967503746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6815","nonce":2967503746}]},"public_addr":"172.21.15.27:6811/2967503746","cluster_addr":"172.21.15.27:6813/2967503746","heartbeat_back_addr":"172.21.15.27:6817/2967503746","heartbeat_front_addr":"172.21.15.27:6815/2967503746","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6819","nonce":302366028}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6821","nonce":302366028}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6825","nonce":302366028}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6823","nonce":302366028}]},"public_addr":"172.21.15.27:6819/302366028","cluster_addr":"172.21.15.27:6821/302366028","heartbeat_back_addr":"172.21.15.27:6825/302366028","heartbeat_front_addr":"172.21.15.27:6823/302366028","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6827","nonce":250460291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6829","nonce":250460291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6833","nonce":250460291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6831","nonce":250460291}]},"public_addr":"172.21.15.27:6827/250460291","cluster_addr":"172.21.15.27:6829/250460291","heartbeat_back_addr":"172.21.15.27:6833/250460291","heartbeat_front_addr":"172.21.15.27:6831/250460291","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6809","nonce":223130519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6811","nonce":223130519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6815","nonce":223130519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6813","nonce":223130519}]},"public_addr":"172.21.15.171:6809/223130519","cluster_addr":"172.21.15.171:6811/223130519","heartbeat_back_addr":"172.21.15.171:6815/223130519","heartbeat_front_addr":"172.21.15.171:6813/223130519","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T14:54:35.593 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:35 smithi027 ceph-mon[29342]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-29T14:54:35.594 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:35 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/328374236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:54:35.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:35 smithi027 ceph-mon[18892]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-29T14:54:35.594 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:35 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/328374236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:54:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:35 smithi171 ceph-mon[22959]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-29T14:54:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:35 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/328374236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:54:35.893 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-29T14:54:35.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T14:54:36.101 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:37 smithi171 ceph-mon[22959]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-29T14:54:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:37 smithi027 ceph-mon[18892]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-29T14:54:37.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:37 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:54:37] "GET /metrics HTTP/1.1" 200 28509 "" "Prometheus/2.43.0" 2024-08-29T14:54:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:37 smithi027 ceph-mon[29342]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-29T14:54:37.876 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:54:37.876 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":66,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T14:54:29.448984+0000","last_up_change":"2024-08-29T14:53:44.442031+0000","last_in_change":"2024-08-29T14:53:28.429586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6803","nonce":1938465338}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6805","nonce":1938465338}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6809","nonce":1938465338}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6807","nonce":1938465338}]},"public_addr":"172.21.15.27:6803/1938465338","cluster_addr":"172.21.15.27:6805/1938465338","heartbeat_back_addr":"172.21.15.27:6809/1938465338","heartbeat_front_addr":"172.21.15.27:6807/1938465338","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6811","nonce":2967503746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6813","nonce":2967503746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6817","nonce":2967503746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6815","nonce":2967503746}]},"public_addr":"172.21.15.27:6811/2967503746","cluster_addr":"172.21.15.27:6813/2967503746","heartbeat_back_addr":"172.21.15.27:6817/2967503746","heartbeat_front_addr":"172.21.15.27:6815/2967503746","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6819","nonce":302366028}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6821","nonce":302366028}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6825","nonce":302366028}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6823","nonce":302366028}]},"public_addr":"172.21.15.27:6819/302366028","cluster_addr":"172.21.15.27:6821/302366028","heartbeat_back_addr":"172.21.15.27:6825/302366028","heartbeat_front_addr":"172.21.15.27:6823/302366028","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6827","nonce":250460291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6829","nonce":250460291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6833","nonce":250460291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6831","nonce":250460291}]},"public_addr":"172.21.15.27:6827/250460291","cluster_addr":"172.21.15.27:6829/250460291","heartbeat_back_addr":"172.21.15.27:6833/250460291","heartbeat_front_addr":"172.21.15.27:6831/250460291","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6809","nonce":223130519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6811","nonce":223130519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6815","nonce":223130519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6813","nonce":223130519}]},"public_addr":"172.21.15.171:6809/223130519","cluster_addr":"172.21.15.171:6811/223130519","heartbeat_back_addr":"172.21.15.171:6815/223130519","heartbeat_front_addr":"172.21.15.171:6813/223130519","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T14:54:38.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:38 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3829666657' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:54:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:38 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3829666657' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:54:38.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-29T14:54:38.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-29T14:54:38.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-29T14:54:38.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-29T14:54:38.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-29T14:54:38.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-29T14:54:38.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-29T14:54:38.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-29T14:54:38.731 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:38 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3829666657' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:54:39.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:54:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:54:38.882Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002611343s 2024-08-29T14:54:39.107 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.109 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.109 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.114 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.119 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.121 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.122 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.136 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:39.566 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:39 smithi027 ceph-mon[29342]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:39.567 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:39 smithi027 ceph-mon[18892]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:39.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:39 smithi171 ceph-mon[22959]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:41.268 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 systemd[1]: Starting Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:54:41.588 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-mon[22959]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:41.588 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 podman[51905]: 2024-08-29 14:54:41.235498 +0000 UTC m=+0.015118143 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-08-29T14:54:41.589 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 podman[51905]: 2024-08-29 14:54:41.360514209 +0000 UTC m=+0.140134395 container create f705829982dcfa5f65815e176b733adcca3b24d40eec444d50b3170ef8a46e65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, vcs-type=git, io.buildah.version=1.32.0, distribution-scope=public, version=8.5, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., release=239.1651231664) 2024-08-29T14:54:41.589 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 podman[51905]: 2024-08-29 14:54:41.58750741 +0000 UTC m=+0.367127554 container init f705829982dcfa5f65815e176b733adcca3b24d40eec444d50b3170ef8a46e65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, release=239.1651231664, vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container) 2024-08-29T14:54:41.809 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:41 smithi027 ceph-mon[18892]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:41.810 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:41 smithi027 ceph-mon[29342]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:41.838 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 podman[51905]: 2024-08-29 14:54:41.590578459 +0000 UTC m=+0.370198611 container start f705829982dcfa5f65815e176b733adcca3b24d40eec444d50b3170ef8a46e65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, version=8.5, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, description=Ceph Grafana Container, release=239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, vcs-type=git, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 bash[51905]: f705829982dcfa5f65815e176b733adcca3b24d40eec444d50b3170ef8a46e65 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 systemd[1]: Started Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748193976Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.74838194Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748399362Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.74840724Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748413676Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.74841995Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748426367Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748432492Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748438818Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-29T14:54:41.839 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748444735Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-29T14:54:41.840 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748450935Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-29T14:54:41.840 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748457489Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-29T14:54:41.840 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748463311Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-29T14:54:41.840 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=settings t=2024-08-29T14:54:41.748469231Z level=info msg="App mode production" 2024-08-29T14:54:41.840 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=sqlstore t=2024-08-29T14:54:41.748516296Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-29T14:54:41.840 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=sqlstore t=2024-08-29T14:54:41.748532145Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-29T14:54:41.840 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.749368674Z level=info msg="Starting DB migrations" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.749442458Z level=info msg="Executing migration" id="create migration_log table" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.749957812Z level=info msg="Executing migration" id="create user table" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.750400999Z level=info msg="Executing migration" id="add unique index user.login" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.753105974Z level=info msg="Executing migration" id="add unique index user.email" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.753545551Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.756379302Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.758881813Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.759636845Z level=info msg="Executing migration" id="create user table v2" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.762348493Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.765128926Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.767841514Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-08-29T14:54:41.841 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.768205493Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.77106328Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.771507415Z level=info msg="Executing migration" id="Update user table charset" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.773935274Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.77446029Z level=info msg="Executing migration" id="Add missing user data" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.776722923Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.777184175Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.77949519Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.779955584Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.78234119Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.785015096Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.787902962Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.790196073Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.792900605Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-08-29T14:54:41.842 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.793343416Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.796105382Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.796541845Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.798914956Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.799436902Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.802081978Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.804842023Z level=info msg="Executing migration" id="create temp_user v2" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.805529472Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-08-29T14:54:41.843 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.805960411Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.80691541Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.807552486Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.808371288Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.809125126Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.810012937Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.810708187Z level=info msg="Executing migration" id="create star table" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.81153046Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.812337994Z level=info msg="Executing migration" id="create org table v1" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.813096982Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.81383534Z level=info msg="Executing migration" id="create org_user table v1" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.814843601Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.815552625Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-08-29T14:54:41.844 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.816466052Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.817201151Z level=info msg="Executing migration" id="Update org table charset" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.818072371Z level=info msg="Executing migration" id="Update org_user table charset" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.818907894Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.819926865Z level=info msg="Executing migration" id="create dashboard table" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.820620617Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.821629407Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.8223404Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.823322793Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.824048851Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.824894379Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.825772692Z level=info msg="Executing migration" id="create dashboard v2" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.82661198Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-08-29T14:54:41.845 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.827572181Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.828334549Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.829205228Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.829762965Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.830068349Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.830898079Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.831441824Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.831961961Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.834897205Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.837723835Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-08-29T14:54:41.846 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.840248719Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.843053255Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.846719746Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.849525238Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.850052892Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.852350769Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.855070023Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.857920572Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-08-29T14:54:42.089 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.860474781Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.863419131Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.866024564Z level=info msg="Executing migration" id="Update dashboard title length" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.868581234Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.871528421Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.874287731Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.877350659Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.880084094Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.882810868Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.885663813Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-08-29T14:54:42.090 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.888294333Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-08-29T14:54:42.091 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.891041511Z level=info msg="Executing migration" id="Add check_sum column" 2024-08-29T14:54:42.091 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.893654814Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-08-29T14:54:42.091 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.896232699Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.898792003Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.901360547Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.904101781Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.906928856Z level=info msg="Executing migration" id="create data_source table" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.909491035Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.911962242Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.912464842Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-08-29T14:54:42.092 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.915052736Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.917747765Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.920523551Z level=info msg="Executing migration" id="create data_source table v2" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.923000386Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.925780404Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.928446278Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.929308652Z level=info msg="Executing migration" id="Add column with_credentials" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.930162865Z level=info msg="Executing migration" id="Add secure json data column" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.930938643Z level=info msg="Executing migration" id="Update data_source table charset" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.931918798Z level=info msg="Executing migration" id="Update initial version to 1" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.932783703Z level=info msg="Executing migration" id="Add read_only data column" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.933502096Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-08-29T14:54:42.093 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.934505775Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.935272173Z level=info msg="Executing migration" id="Add uid column" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.936021098Z level=info msg="Executing migration" id="Update uid value" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.936989912Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.93774674Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.938718458Z level=info msg="Executing migration" id="create api_key table" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.93932421Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.940338444Z level=info msg="Executing migration" id="add index api_key.key" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.941034664Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-08-29T14:54:42.094 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.941876241Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.942908664Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.9434599Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.9443393Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.945464823Z level=info msg="Executing migration" id="create api_key table v2" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.945918448Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.946364128Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.947238488Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.948118001Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.948932861Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-08-29T14:54:42.095 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.949316859Z level=info msg="Executing migration" id="Update api_key table charset" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.950238353Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.950987193Z level=info msg="Executing migration" id="Add service account foreign key" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.951572515Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.952458834Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.953078676Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.953956692Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.954929773Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.955753266Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.956198706Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-08-29T14:54:42.096 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.957212355Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-08-29T14:54:42.097 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.959939185Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-08-29T14:54:42.097 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.962752181Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-08-29T14:54:42.097 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.963100213Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-08-29T14:54:42.097 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.965917142Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-08-29T14:54:42.097 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.966687995Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-08-29T14:54:42.097 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.968811196Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-08-29T14:54:42.098 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.971627124Z level=info msg="Executing migration" id="create quota table v1" 2024-08-29T14:54:42.098 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.972089785Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-08-29T14:54:42.098 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.974685547Z level=info msg="Executing migration" id="Update quota table charset" 2024-08-29T14:54:42.098 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.975028538Z level=info msg="Executing migration" id="create plugin_setting table" 2024-08-29T14:54:42.098 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.975454445Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.975902022Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.976519956Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.976869Z level=info msg="Executing migration" id="create session table" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.977327838Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.977639519Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.978028963Z level=info msg="Executing migration" id="create playlist table v2" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.978456458Z level=info msg="Executing migration" id="create playlist item table v2" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.978880397Z level=info msg="Executing migration" id="Update playlist table charset" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.979217144Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.979549381Z level=info msg="Executing migration" id="drop preferences table v2" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.979882216Z level=info msg="Executing migration" id="drop preferences table v3" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.980231498Z level=info msg="Executing migration" id="create preferences table v3" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.980662454Z level=info msg="Executing migration" id="Update preferences table charset" 2024-08-29T14:54:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.981006772Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.981612701Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.981972499Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.982631199Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.983251735Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.983614144Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.984096712Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.984618837Z level=info msg="Executing migration" id="create alert table v1" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.987315488Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.990403343Z level=info msg="Executing migration" id="add index alert state" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.991022546Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-08-29T14:54:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.991500914Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-08-29T14:54:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.991989681Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-08-29T14:54:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.992441836Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-08-29T14:54:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.992905548Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-08-29T14:54:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.994291006Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-08-29T14:54:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.99473738Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-08-29T14:54:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.99520105Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.995545204Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.996221195Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.99661412Z level=info msg="Executing migration" id="Add column is_default" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.997230769Z level=info msg="Executing migration" id="Add column frequency" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.997861077Z level=info msg="Executing migration" id="Add column send_reminder" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.998474484Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.999102481Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.999520655Z level=info msg="Executing migration" id="Update alert table charset" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:41.999776934Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-08-29T14:54:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.000074703Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.000448317Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.000881899Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.001259072Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.001617057Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.002028548Z level=info msg="Executing migration" id="Add for to alert table" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.002639919Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.003277248Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.003558572Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-08-29T14:54:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.003983951Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.004375308Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.005035256Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.005992084Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.006739053Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.007678557Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.008139867Z level=info msg="Executing migration" id="create annotation table v5" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.010930914Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.013815842Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.016488576Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.016937956Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.019607741Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-08-29T14:54:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.022277108Z level=info msg="Executing migration" id="Update annotation table charset" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.024968689Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.025642864Z level=info msg="Executing migration" id="Drop category_id index" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.028437948Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.031178159Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.033685266Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.034134015Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.036993238Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.038560208Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.039797001Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-08-29T14:54:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.042643896Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.043144285Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.045975811Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.046312001Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.048548605Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.05132942Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.051875869Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.054427471Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.057064951Z level=info msg="Executing migration" id="Add epoch_end column" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.059799Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.060245519Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.062929231Z level=info msg="Executing migration" id="Move region to single row" 2024-08-29T14:54:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.063291453Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.065585708Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.066087276Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.068288502Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.06877799Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.069181251Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.071792061Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.072133104Z level=info msg="Executing migration" id="create test_data table" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.074795329Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.075442003Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.07587579Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-08-29T14:54:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.076677471Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.077655207Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.078393082Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.079182053Z level=info msg="Executing migration" id="create team table" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.079756108Z level=info msg="Executing migration" id="add index team.org_id" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.080198768Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.080965975Z level=info msg="Executing migration" id="create team member table" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.081858647Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.082494147Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.083281967Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.084114883Z level=info msg="Executing migration" id="Add column email to team table" 2024-08-29T14:54:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.08486975Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.085566948Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.086351887Z level=info msg="Executing migration" id="create dashboard acl table" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.087208739Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.088023514Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.088477925Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.089361419Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.090214626Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.091020959Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.091875431Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.09276782Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-08-29T14:54:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.09359865Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-08-29T14:54:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.094439728Z level=info msg="Executing migration" id="create tag table" 2024-08-29T14:54:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.095301032Z level=info msg="Executing migration" id="add index tag.key_value" 2024-08-29T14:54:42.340 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.096088505Z level=info msg="Executing migration" id="create login attempt table" 2024-08-29T14:54:42.340 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.096934997Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-08-29T14:54:42.340 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.097813426Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.098828315Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.100733843Z level=info msg="Executing migration" id="create login_attempt v2" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.101560655Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.102459Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.105071398Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.107983884Z level=info msg="Executing migration" id="create user auth table" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.11068451Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.113647517Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.116295939Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.118986574Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-08-29T14:54:42.341 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.121749023Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.124613313Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.127483382Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.129721237Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.132481127Z level=info msg="Executing migration" id="create server_lock table" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.135312554Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.138125812Z level=info msg="Executing migration" id="create user auth token table" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.140677995Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.143399858Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-08-29T14:54:42.342 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.146212889Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.149221492Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.15166019Z level=info msg="Executing migration" id="create cache_data table" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.154599781Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.157311213Z level=info msg="Executing migration" id="create short_url table v1" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.160243913Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.163008122Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.165595772Z level=info msg="Executing migration" id="delete alert_definition table" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.168306662Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.170909641Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.173695176Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.17623061Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.178769552Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.181473208Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-08-29T14:54:42.343 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.184240967Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.186943981Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.18954497Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.192196987Z level=info msg="Executing migration" id="drop alert_definition table" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.194813469Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.197434196Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.198194748Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.1991242Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.199789693Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.200671206Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.201406593Z level=info msg="Executing migration" id="create alert_instance table" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.202236409Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-08-29T14:54:42.344 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.203080517Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-08-29T14:54:42.345 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.20387724Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-08-29T14:54:42.345 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.204674863Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-08-29T14:54:42.345 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.205461378Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-08-29T14:54:42.345 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.206295796Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-08-29T14:54:42.345 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.209712333Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-08-29T14:54:42.345 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.213070652Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.213878829Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.214668443Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.21551903Z level=info msg="Executing migration" id="create alert_rule table" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.216431861Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.21734306Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.218118979Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.218882638Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-08-29T14:54:42.346 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.21990086Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.220887891Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.221773638Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.222620244Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.223445768Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.223880291Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.224710821Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.225597418Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.22604385Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.226864597Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.227686223Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.228011469Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.228452787Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-08-29T14:54:42.347 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.230989695Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-08-29T14:54:42.348 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.231434451Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-08-29T14:54:42.348 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.231700938Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-08-29T14:54:42.348 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.232585033Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-08-29T14:54:42.348 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.234615686Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-08-29T14:54:42.348 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.237099504Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-08-29T14:54:42.348 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.239740206Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-08-29T14:54:42.348 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.242157758Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.243022309Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.244022501Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.244948058Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.245936896Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.24695638Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.247807178Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.248654337Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.249495402Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.250284015Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-08-29T14:54:42.349 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.251227848Z level=info msg="Executing migration" id="create provenance_type table" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.252090744Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.252867805Z level=info msg="Executing migration" id="create alert_image table" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.253794662Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.254506383Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.255328298Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.255750531Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.256202378Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.256275149Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.256963443Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.257327342Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-08-29T14:54:42.350 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.257819883Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.257933032Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.258042069Z level=info msg="alerts found to migrate" alerts=0 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.258431737Z level=info msg="Executing migration" id="create library_element table v1" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.258882522Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.259742432Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.260711047Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.261609645Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.262337671Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-08-29T14:54:42.351 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.263132462Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.26352703Z level=info msg="Executing migration" id="create data_keys table" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.264086828Z level=info msg="Executing migration" id="create secrets table" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.264922123Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.269430637Z level=info msg="Executing migration" id="add name column into data_keys" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.270374303Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.271339683Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.275619852Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.279804288Z level=info msg="Executing migration" id="create kv_store table v1" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.280592607Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-08-29T14:54:42.352 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.28338346Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.286248873Z level=info msg="Executing migration" id="create permission table" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.288994303Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.291613541Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.294539443Z level=info msg="Executing migration" id="create role table" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.297257516Z level=info msg="Executing migration" id="add column display_name" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.300045924Z level=info msg="Executing migration" id="add column group_name" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.302946951Z level=info msg="Executing migration" id="add index role.org_id" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.305673671Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-08-29T14:54:42.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.308290244Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.311390259Z level=info msg="Executing migration" id="create team role table" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.314002065Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.316539924Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.319449891Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.322383557Z level=info msg="Executing migration" id="create user role table" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.325196377Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.328215118Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-08-29T14:54:42.354 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.330946674Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-08-29T14:54:42.355 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.333672001Z level=info msg="Executing migration" id="create builtin role table" 2024-08-29T14:54:42.355 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.336443176Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-08-29T14:54:42.355 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.339122213Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-08-29T14:54:42.355 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.341605925Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-08-29T14:54:42.726 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.344197278Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-08-29T14:54:42.726 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.344766156Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-08-29T14:54:42.726 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.345279948Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-08-29T14:54:42.726 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.345715282Z level=info msg="Executing migration" id="add unique index role.uid" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.346173955Z level=info msg="Executing migration" id="create seed assignment table" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.346579625Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.347055133Z level=info msg="Executing migration" id="add column hidden to role table" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.348060019Z level=info msg="Executing migration" id="create query_history table v1" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.350760796Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.351267987Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.351559734Z level=info msg="Executing migration" id="teams permissions migration" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.352006206Z level=info msg="Executing migration" id="dashboard permissions" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.352517765Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.352928194Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.353259844Z level=info msg="Executing migration" id="alerting notification permissions" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.353608753Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-08-29T14:54:42.727 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.354075175Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-08-29T14:54:42.728 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.354561024Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-08-29T14:54:42.728 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.355577442Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-08-29T14:54:42.728 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.355937281Z level=info msg="Executing migration" id="create correlation table v1" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.356378586Z level=info msg="Executing migration" id="add index correlations.uid" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.356840456Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.357379919Z level=info msg="Executing migration" id="add correlation config column" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.358450457Z level=info msg="Executing migration" id="create entity_events table" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.358927259Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.359399146Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.359501404Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.359798903Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.359904406Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.361615271Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-08-29T14:54:42.729 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.364504296Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.366965514Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.369772167Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.372479972Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.375120854Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.377933476Z level=info msg="Executing migration" id="Drop public config table" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.380671639Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.383387587Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.385853112Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.38683354Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.38753184Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.390537619Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.391631061Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.39268627Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-08-29T14:54:42.730 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.39351029Z level=info msg="Executing migration" id="add share column" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.394575915Z level=info msg="Executing migration" id="create default alerting folders" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.395435752Z level=info msg="Executing migration" id="create file table" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.396225012Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.397212559Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.39786737Z level=info msg="Executing migration" id="create file_meta table" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.398306249Z level=info msg="Executing migration" id="file table idx: path key" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.399279351Z level=info msg="Executing migration" id="set path collation in file table" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.399836166Z level=info msg="Executing migration" id="managed permissions migration" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.400714154Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.401595259Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.402390814Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.403443193Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.404285593Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-08-29T14:54:42.731 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.405146609Z level=info msg="Executing migration" id="update group index for alert rules" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.40610548Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.406848398Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.407810702Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.409032403Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.41005387Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.410897416Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.420661499Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.421498699Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.42247676Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.426068049Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.42692228Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.427786338Z level=info msg="Executing migration" id="create folder table" 2024-08-29T14:54:42.732 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.428565752Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.429495848Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.430266639Z level=info msg="Executing migration" id="Update folder title length" 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.431148952Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=migrator t=2024-08-29T14:54:42.43198293Z level=info msg="migrations completed" performed=485 skipped=0 duration=682.559179ms 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=sqlstore t=2024-08-29T14:54:42.432379684Z level=info msg="Created default organization" 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=plugin.loader t=2024-08-29T14:54:42.487742576Z level=info msg="Plugin registered" pluginID=input 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=plugin.loader t=2024-08-29T14:54:42.514492166Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=plugin.loader t=2024-08-29T14:54:42.514512597Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=secrets t=2024-08-29T14:54:42.514546937Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=query_data t=2024-08-29T14:54:42.516164861Z level=info msg="Query Service initialization" 2024-08-29T14:54:42.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=live.push_http t=2024-08-29T14:54:42.518380004Z level=info msg="Live Push Gateway initialization" 2024-08-29T14:54:42.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[18892]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:42.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[18892]: Deploying daemon node-exporter.a on smithi027 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[29342]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:42.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:42 smithi027 ceph-mon[29342]: Deploying daemon node-exporter.a on smithi027 2024-08-29T14:54:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-mon[22959]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:42 smithi171 ceph-mon[22959]: Deploying daemon node-exporter.a on smithi027 2024-08-29T14:54:43.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=infra.usagestats.collector t=2024-08-29T14:54:43.254452779Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-29T14:54:43.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=provisioning.datasources t=2024-08-29T14:54:43.310087532Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-29T14:54:43.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=provisioning.datasources t=2024-08-29T14:54:43.341861913Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=provisioning.alerting t=2024-08-29T14:54:43.363400244Z level=info msg="starting to provision alerting" 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=provisioning.alerting t=2024-08-29T14:54:43.363419318Z level=info msg="finished to provision alerting" 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=grafanaStorageLogger t=2024-08-29T14:54:43.363670593Z level=info msg="storage starting" 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=ngalert.state.manager t=2024-08-29T14:54:43.363589194Z level=info msg="Warming state cache for startup" 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=http.server t=2024-08-29T14:54:43.367900391Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=sqlstore.transactions t=2024-08-29T14:54:43.374775118Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=ngalert.state.manager t=2024-08-29T14:54:43.374881762Z level=info msg="State cache has been initialized" states=0 duration=11.292104ms 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=ticker t=2024-08-29T14:54:43.374928444Z level=info msg=starting first_tick=2024-08-29T14:54:50Z 2024-08-29T14:54:43.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=ngalert.multiorg.alertmanager t=2024-08-29T14:54:43.374943394Z level=info msg="starting MultiOrg Alertmanager" 2024-08-29T14:54:43.519 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=sqlstore.transactions t=2024-08-29T14:54:43.429627445Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-29T14:54:43.519 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 14:54:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[51920]: logger=sqlstore.transactions t=2024-08-29T14:54:43.440190149Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-08-29T14:54:45.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:44 smithi027 ceph-mon[18892]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:45.001 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:44 smithi027 ceph-mon[29342]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:45.002 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:54:44 smithi027 systemd[1]: Starting Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:54:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:44 smithi171 ceph-mon[22959]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:46.663 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:46 smithi027 ceph-mon[29342]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:46.663 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:46 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:46.663 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:46 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:46.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:46 smithi027 ceph-mon[18892]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:46.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:46 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:46.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:46 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:46 smithi171 ceph-mon[22959]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:46 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:54:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:46 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:54:47.861 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:54:47] "GET /metrics HTTP/1.1" 200 28505 "" "Prometheus/2.43.0" 2024-08-29T14:54:48.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:48 smithi027 ceph-mon[29342]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:48.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:48 smithi027 ceph-mon[18892]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:48 smithi171 ceph-mon[22959]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:49.886 INFO:teuthology.orchestra.run.smithi027.stdout:128849018908 2024-08-29T14:54:49.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-29T14:54:50.826 INFO:teuthology.orchestra.run.smithi027.stdout:34359738416 2024-08-29T14:54:50.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-29T14:54:50.853 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:50 smithi027 ceph-mon[18892]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:50.853 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:50 smithi027 ceph-mon[29342]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:51.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:50 smithi171 ceph-mon[22959]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:52.456 INFO:teuthology.orchestra.run.smithi027.stdout:77309411367 2024-08-29T14:54:52.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-29T14:54:52.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:52 smithi027 ceph-mon[18892]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:52 smithi027 ceph-mon[29342]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:53.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:52 smithi171 ceph-mon[22959]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:53.914 INFO:teuthology.orchestra.run.smithi027.stdout:231928234000 2024-08-29T14:54:53.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-29T14:54:54.314 INFO:teuthology.orchestra.run.smithi027.stdout:55834574893 2024-08-29T14:54:54.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-29T14:54:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:54 smithi027 ceph-mon[18892]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:54 smithi027 ceph-mon[29342]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:54 smithi171 ceph-mon[22959]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:55.238 INFO:teuthology.orchestra.run.smithi027.stdout:197568495636 2024-08-29T14:54:55.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-29T14:54:55.582 INFO:teuthology.orchestra.run.smithi027.stdout:107374182434 2024-08-29T14:54:55.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-29T14:54:56.000 INFO:teuthology.orchestra.run.smithi027.stdout:163208757273 2024-08-29T14:54:56.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-29T14:54:56.041 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:56.065 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:56.085 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:56.092 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:56.550 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:56.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:56 smithi027 ceph-mon[18892]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:56.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:56 smithi027 ceph-mon[29342]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:57.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:56 smithi171 ceph-mon[22959]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:57.779 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:54:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:54:57] "GET /metrics HTTP/1.1" 200 28505 "" "Prometheus/2.43.0" 2024-08-29T14:54:58.068 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:54:57 smithi027 bash[60784]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-29T14:54:58.217 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:54:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:54:58 smithi027 ceph-mon[18892]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:54:58 smithi027 ceph-mon[29342]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:54:58 smithi171 ceph-mon[22959]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:54:59.628 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:00.143 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:54:59 smithi027 bash[60784]: Getting image source signatures 2024-08-29T14:55:00.144 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:54:59 smithi027 bash[60784]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-29T14:55:00.144 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:54:59 smithi027 bash[60784]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-29T14:55:00.144 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:54:59 smithi027 bash[60784]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-29T14:55:00.776 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:00 smithi027 ceph-mon[29342]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:00.776 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:00 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:00.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:00 smithi027 ceph-mon[18892]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:00.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:00 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:00 smithi171 ceph-mon[22959]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:00 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:02.318 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:02 smithi027 bash[60784]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-29T14:55:02.681 INFO:teuthology.orchestra.run.smithi027.stdout:107374182436 2024-08-29T14:55:02.682 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:02 smithi027 bash[60784]: Writing manifest to image destination 2024-08-29T14:55:02.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:02 smithi027 ceph-mon[29342]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:02.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:02 smithi027 ceph-mon[18892]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:02 smithi171 ceph-mon[22959]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:03.296 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 podman[60784]: 2024-08-29 14:55:02.935873159 +0000 UTC m=+5.253072315 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-29T14:55:03.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 podman[60784]: 2024-08-29 14:55:03.480059537 +0000 UTC m=+5.797258735 container create 8caf48017aa973806a496e95a010e7d0bc30e350a28783ada13871f431bafa68 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:03.877 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3807545700' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-29T14:55:03.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3807545700' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-29T14:55:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:03 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3807545700' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 podman[60784]: 2024-08-29 14:55:03.877165963 +0000 UTC m=+6.194365104 container init 8caf48017aa973806a496e95a010e7d0bc30e350a28783ada13871f431bafa68 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 podman[60784]: 2024-08-29 14:55:03.880348952 +0000 UTC m=+6.197548089 container start 8caf48017aa973806a496e95a010e7d0bc30e350a28783ada13871f431bafa68 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.884Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.884Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.884Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.884Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-29T14:55:04.190 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=arp 2024-08-29T14:55:04.191 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-29T14:55:04.191 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-29T14:55:04.191 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-29T14:55:04.191 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-29T14:55:04.191 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-29T14:55:04.191 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=edac 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-29T14:55:04.192 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=os 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-29T14:55:04.193 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=stat 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=time 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=uname 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.885Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-29T14:55:04.194 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.886Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-29T14:55:04.195 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[61108]: ts=2024-08-29T14:55:03.886Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-29T14:55:04.195 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 bash[60784]: 8caf48017aa973806a496e95a010e7d0bc30e350a28783ada13871f431bafa68 2024-08-29T14:55:04.195 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:55:03 smithi027 systemd[1]: Started Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:55:04.580 INFO:teuthology.orchestra.run.smithi027.stdout:77309411370 2024-08-29T14:55:05.017 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:04 smithi171 bash[52182]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-29T14:55:05.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[18892]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:05.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[18892]: Deploying daemon node-exporter.b on smithi171 2024-08-29T14:55:05.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2313230223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-29T14:55:05.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[29342]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:05.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[29342]: Deploying daemon node-exporter.b on smithi171 2024-08-29T14:55:05.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:05 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2313230223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-29T14:55:05.314 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182434 got 107374182436 for osd.3 2024-08-29T14:55:05.314 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:05 smithi171 ceph-mon[22959]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:05 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:05 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:05 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:05 smithi171 ceph-mon[22959]: Deploying daemon node-exporter.b on smithi171 2024-08-29T14:55:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:05 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2313230223' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-29T14:55:05.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:05 smithi171 bash[52182]: Getting image source signatures 2024-08-29T14:55:05.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:05 smithi171 bash[52182]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-29T14:55:05.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:05 smithi171 bash[52182]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-29T14:55:05.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:05 smithi171 bash[52182]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-29T14:55:05.694 INFO:teuthology.orchestra.run.smithi027.stdout:231928234002 2024-08-29T14:55:06.085 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:06 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4288751297' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-29T14:55:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:06 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/4288751297' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-29T14:55:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:06 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/4288751297' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-29T14:55:06.973 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:06 smithi171 bash[52182]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-29T14:55:06.974 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:06 smithi171 bash[52182]: Writing manifest to image destination 2024-08-29T14:55:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-mon[22959]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:07.268 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:06 smithi171 podman[52182]: 2024-08-29 14:55:06.946642817 +0000 UTC m=+2.096282122 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-29T14:55:07.268 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 podman[52182]: 2024-08-29 14:55:07.053302558 +0000 UTC m=+2.202941849 container create 91bd04afdd9c9a56bbce21914292846f0527dfdcfa61d3b1d36a3fa2478200cc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:55:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:07 smithi027 ceph-mon[18892]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:07.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:07 smithi027 ceph-mon[29342]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:07.640 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:55:07] "GET /metrics HTTP/1.1" 200 28506 "" "Prometheus/2.43.0" 2024-08-29T14:55:07.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 podman[52182]: 2024-08-29 14:55:07.280578977 +0000 UTC m=+2.430218269 container init 91bd04afdd9c9a56bbce21914292846f0527dfdcfa61d3b1d36a3fa2478200cc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:55:07.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 podman[52182]: 2024-08-29 14:55:07.283427785 +0000 UTC m=+2.433067071 container start 91bd04afdd9c9a56bbce21914292846f0527dfdcfa61d3b1d36a3fa2478200cc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:55:07.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.287Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-29T14:55:07.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.287Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-29T14:55:07.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.287Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-29T14:55:07.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.287Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-29T14:55:07.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.287Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.287Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=arp 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-29T14:55:07.769 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-29T14:55:07.770 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-29T14:55:07.770 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-29T14:55:07.770 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=edac 2024-08-29T14:55:07.770 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-29T14:55:07.770 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-29T14:55:07.770 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-29T14:55:07.770 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-29T14:55:07.771 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=os 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=stat 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=time 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-29T14:55:07.772 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=uname 2024-08-29T14:55:07.773 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-29T14:55:07.773 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-29T14:55:07.773 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.288Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-29T14:55:07.773 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.289Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-29T14:55:07.773 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[52257]: ts=2024-08-29T14:55:07.289Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-29T14:55:07.773 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 bash[52182]: 91bd04afdd9c9a56bbce21914292846f0527dfdcfa61d3b1d36a3fa2478200cc 2024-08-29T14:55:07.773 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:55:07 smithi171 systemd[1]: Started Ceph node-exporter.b for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:55:08.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:08 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:08 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:08 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:08 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:08 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:08 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:08 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:09.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:09 smithi027 ceph-mon[29342]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:09 smithi027 ceph-mon[18892]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:09.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:09 smithi171 ceph-mon[22959]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:09.989 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234000 got 231928234002 for osd.7 2024-08-29T14:55:09.989 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:10.110 INFO:teuthology.orchestra.run.smithi027.stdout:128849018913 2024-08-29T14:55:10.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:10 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4128062375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-29T14:55:10.648 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:10 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/4128062375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-29T14:55:10.673 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018913 for osd.4 2024-08-29T14:55:10.673 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:10 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/4128062375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-29T14:55:11.035 INFO:teuthology.orchestra.run.smithi027.stdout:34359738420 2024-08-29T14:55:11.260 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411370 for osd.2 2024-08-29T14:55:11.260 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:11.547 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[29342]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:11.547 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.547 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.547 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.547 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1868038858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-29T14:55:11.547 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[18892]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:11.548 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.548 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.548 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.548 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:11 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1868038858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-29T14:55:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:11 smithi171 ceph-mon[22959]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:11 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:11 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:11 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:11 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1868038858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-29T14:55:12.313 INFO:teuthology.orchestra.run.smithi027.stdout:55834574896 2024-08-29T14:55:12.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:12 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1110759442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-29T14:55:12.559 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:12 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1110759442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-29T14:55:12.733 INFO:teuthology.orchestra.run.smithi027.stdout:197568495639 2024-08-29T14:55:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:12 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1110759442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-29T14:55:13.367 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495636 got 197568495639 for osd.6 2024-08-29T14:55:13.367 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:13.645 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:13 smithi027 ceph-mon[29342]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:13.645 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:13 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/491125721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-29T14:55:13.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:13 smithi027 ceph-mon[18892]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:13.645 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:13 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/491125721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-29T14:55:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:13 smithi171 ceph-mon[22959]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:13 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/491125721' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-29T14:55:13.849 INFO:teuthology.orchestra.run.smithi027.stdout:163208757277 2024-08-29T14:55:14.077 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738420 for osd.0 2024-08-29T14:55:14.077 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:14.660 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:14 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/652400149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-29T14:55:14.661 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:14 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/652400149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-29T14:55:14.684 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757273 got 163208757277 for osd.5 2024-08-29T14:55:14.685 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:14 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/652400149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-29T14:55:15.198 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574893 got 55834574896 for osd.1 2024-08-29T14:55:15.198 DEBUG:teuthology.parallel:result is None 2024-08-29T14:55:15.198 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-29T14:55:15.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T14:55:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:15 smithi171 ceph-mon[22959]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:15 smithi027 ceph-mon[18892]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:15.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:15 smithi027 ceph-mon[29342]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:15.947 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:16.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:16.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:17.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:17 smithi027 ceph-mon[18892]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:17.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:17 smithi027 ceph-mon[29342]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:17.612 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:55:17] "GET /metrics HTTP/1.1" 200 28503 "" "Prometheus/2.43.0" 2024-08-29T14:55:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:17 smithi171 ceph-mon[22959]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:17.769 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:55:17.770 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T14:55:19.029 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2024-08-29T14:55:17.560274+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227232,"kb_used_data":6976,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733984,"statfs":{"total":767960285184,"available":767727599616,"internally_reserved":0,"allocated":7143424,"data_stored":4033813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001746"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:30.326244+0000","last_change":"2024-08-29T14:54:23.263882+0000","last_active":"2024-08-29T14:54:30.326244+0000","last_peered":"2024-08-29T14:54:30.326244+0000","last_clean":"2024-08-29T14:54:30.326244+0000","last_became_active":"2024-08-29T14:54:18.559285+0000","last_became_peered":"2024-08-29T14:54:18.559285+0000","last_unstale":"2024-08-29T14:54:30.326244+0000","last_undegraded":"2024-08-29T14:54:30.326244+0000","last_fullsized":"2024-08-29T14:54:30.326244+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:26:21.042994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027796,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.451774+0000","last_change":"2024-08-29T14:54:23.263862+0000","last_active":"2024-08-29T14:54:29.451774+0000","last_peered":"2024-08-29T14:54:29.451774+0000","last_clean":"2024-08-29T14:54:29.451774+0000","last_became_active":"2024-08-29T14:54:18.771613+0000","last_became_peered":"2024-08-29T14:54:18.771613+0000","last_unstale":"2024-08-29T14:54:29.451774+0000","last_undegraded":"2024-08-29T14:54:29.451774+0000","last_fullsized":"2024-08-29T14:54:29.451774+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:22:56.189804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029192399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.534229+0000","last_change":"2024-08-29T14:54:23.447440+0000","last_active":"2024-08-29T14:54:29.534229+0000","last_peered":"2024-08-29T14:54:29.534229+0000","last_clean":"2024-08-29T14:54:29.534229+0000","last_became_active":"2024-08-29T14:54:18.771432+0000","last_became_peered":"2024-08-29T14:54:18.771432+0000","last_unstale":"2024-08-29T14:54:29.534229+0000","last_undegraded":"2024-08-29T14:54:29.534229+0000","last_fullsized":"2024-08-29T14:54:29.534229+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:18:29.200943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033832300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.451776+0000","last_change":"2024-08-29T14:54:23.263824+0000","last_active":"2024-08-29T14:54:29.451776+0000","last_peered":"2024-08-29T14:54:29.451776+0000","last_clean":"2024-08-29T14:54:29.451776+0000","last_became_active":"2024-08-29T14:54:18.562017+0000","last_became_peered":"2024-08-29T14:54:18.562017+0000","last_unstale":"2024-08-29T14:54:29.451776+0000","last_undegraded":"2024-08-29T14:54:29.451776+0000","last_fullsized":"2024-08-29T14:54:29.451776+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:07:43.010038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000266333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.978186+0000","last_change":"2024-08-29T14:54:23.262948+0000","last_active":"2024-08-29T14:54:29.978186+0000","last_peered":"2024-08-29T14:54:29.978186+0000","last_clean":"2024-08-29T14:54:29.978186+0000","last_became_active":"2024-08-29T14:54:18.561354+0000","last_became_peered":"2024-08-29T14:54:18.561354+0000","last_unstale":"2024-08-29T14:54:29.978186+0000","last_undegraded":"2024-08-29T14:54:29.978186+0000","last_fullsized":"2024-08-29T14:54:29.978186+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:14.431009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093147100000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:30.326032+0000","last_change":"2024-08-29T14:54:23.256858+0000","last_active":"2024-08-29T14:54:30.326032+0000","last_peered":"2024-08-29T14:54:30.326032+0000","last_clean":"2024-08-29T14:54:30.326032+0000","last_became_active":"2024-08-29T14:54:18.877989+0000","last_became_peered":"2024-08-29T14:54:18.877989+0000","last_unstale":"2024-08-29T14:54:30.326032+0000","last_undegraded":"2024-08-29T14:54:30.326032+0000","last_fullsized":"2024-08-29T14:54:30.326032+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:25:02.401747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033926300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.534187+0000","last_change":"2024-08-29T14:54:23.447441+0000","last_active":"2024-08-29T14:54:29.534187+0000","last_peered":"2024-08-29T14:54:29.534187+0000","last_clean":"2024-08-29T14:54:29.534187+0000","last_became_active":"2024-08-29T14:54:18.562058+0000","last_became_peered":"2024-08-29T14:54:18.562058+0000","last_unstale":"2024-08-29T14:54:29.534187+0000","last_undegraded":"2024-08-29T14:54:29.534187+0000","last_fullsized":"2024-08-29T14:54:29.534187+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:27:17.193516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032779599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.978195+0000","last_change":"2024-08-29T14:54:23.263036+0000","last_active":"2024-08-29T14:54:29.978195+0000","last_peered":"2024-08-29T14:54:29.978195+0000","last_clean":"2024-08-29T14:54:29.978195+0000","last_became_active":"2024-08-29T14:54:18.558518+0000","last_became_peered":"2024-08-29T14:54:18.558518+0000","last_unstale":"2024-08-29T14:54:29.978195+0000","last_undegraded":"2024-08-29T14:54:29.978195+0000","last_fullsized":"2024-08-29T14:54:29.978195+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:16.144414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001015642,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":262,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-08-29T14:54:23.447102+0000","last_change":"2024-08-29T14:53:48.474059+0000","last_active":"2024-08-29T14:54:23.447102+0000","last_peered":"2024-08-29T14:54:23.447102+0000","last_clean":"2024-08-29T14:54:23.447102+0000","last_became_active":"2024-08-29T14:53:48.473622+0000","last_became_peered":"2024-08-29T14:53:48.473622+0000","last_unstale":"2024-08-29T14:54:23.447102+0000","last_undegraded":"2024-08-29T14:54:23.447102+0000","last_fullsized":"2024-08-29T14:54:23.447102+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:51:04.923918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.55400000000000005}]},{"osd":1,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.442}]},{"osd":2,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.502},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.51200000000000001}]},{"osd":3,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.47299999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.379},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.54000000000000004}]},{"osd":5,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.48799999999999999}]},{"osd":6,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.41099999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.55000000000000004}]},{"osd":1,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.42599999999999999}]},{"osd":2,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.5}]},{"osd":3,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.47499999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.44700000000000001}]},{"osd":5,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.39100000000000001}]},{"osd":7,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.58799999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":282781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.60599999999999998,"5min":0.747,"15min":0.747},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.53300000000000003,"15min":0.52500000000000002},"min":{"1min":0.40600000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73799999999999999,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.50800000000000001,"15min":0.50600000000000001},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71199999999999997,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.46200000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.503,"15min":0.50600000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.54200000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69399999999999995,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.41999999999999998}]},{"osd":3,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51700000000000002,"15min":0.51900000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68999999999999995,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.63400000000000001,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.47199999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.376,"15min":0.36599999999999999},"min":{"1min":0.26600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.374,"15min":0.36799999999999999},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.58999999999999997,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.38500000000000001}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.432},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.53400000000000003}]},{"osd":7,"last update":"Thu Aug 29 14:54:49 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.60699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018915,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27672,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717480,"statfs":{"total":95995035648,"available":95966699520,"internally_reserved":0,"allocated":667648,"data_stored":282820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.72999999999999998,"5min":0.871,"15min":0.871},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71499999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.749}]},{"osd":1,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.38,"5min":0.375,"15min":0.375},"max":{"1min":0.68899999999999995,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.34799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.69599999999999995,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.61099999999999999}]},{"osd":2,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70399999999999996,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38500000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73599999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.63}]},{"osd":3,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.41299999999999998,"5min":0.375,"15min":0.375},"max":{"1min":0.72199999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40699999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72199999999999998,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.78300000000000003}]},{"osd":5,"last update":"Thu Aug 29 14:55:03 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.42199999999999999,"15min":0.41199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.442,"15min":0.434},"min":{"1min":0.32900000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.59399999999999997}]},{"osd":6,"last update":"Thu Aug 29 14:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.45700000000000002}]},{"osd":7,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.54700000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.375,"15min":0.37},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41599999999999998,"15min":0.41699999999999998},"min":{"1min":0.253,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.39200000000000002}]},{"osd":1,"last update":"Thu Aug 29 14:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.42099999999999999,"15min":0.42699999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.501,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42299999999999999,"15min":0.42599999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.56000000000000005,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.372}]},{"osd":2,"last update":"Thu Aug 29 14:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.53000000000000003,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.45900000000000002,"15min":0.46600000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65100000000000002,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.35799999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.54700000000000004,"15min":0.54600000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.58199999999999996,"15min":0.58499999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.73099999999999998,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.63700000000000001}]},{"osd":5,"last update":"Thu Aug 29 14:55:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.45100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73599999999999999,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.55400000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.61899999999999999}]},{"osd":6,"last update":"Thu Aug 29 14:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Aug 29 14:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.56299999999999994}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31764,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713388,"statfs":{"total":95995035648,"available":95962509312,"internally_reserved":0,"allocated":663552,"data_stored":282151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.34599999999999997},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.52400000000000002,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.313},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39000000000000001,"15min":0.38},"min":{"1min":0.23499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.56599999999999995,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.41499999999999998}]},{"osd":1,"last update":"Thu Aug 29 14:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.39300000000000002,"15min":0.374},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.60599999999999998,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.378,"15min":0.371},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.58399999999999996,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.29099999999999998}]},{"osd":3,"last update":"Thu Aug 29 14:54:19 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47199999999999998,"15min":0.47099999999999997},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.64900000000000002,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.42699999999999999,"15min":0.43099999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.32800000000000001}]},{"osd":4,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.58099999999999996,"15min":0.59099999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.94199999999999995,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.63100000000000001,"15min":0.63300000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.90300000000000002,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.57699999999999996}]},{"osd":5,"last update":"Thu Aug 29 14:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.60399999999999998,"15min":0.60599999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.82599999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57399999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81499999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.55300000000000005}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.51300000000000001}]},{"osd":7,"last update":"Thu Aug 29 14:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.60199999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":283112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.45400000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38700000000000001,"15min":0.38400000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.50600000000000001,"5min":0.623,"15min":0.623},"last":0.33900000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:54:55 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.37,"15min":0.36599999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.40000000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.66000000000000003,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.379}]},{"osd":3,"last update":"Thu Aug 29 14:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.45700000000000002,"15min":0.46000000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.434,"15min":0.441},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.35499999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.57899999999999996,"15min":0.58199999999999996},"min":{"1min":0.433,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.751,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.56399999999999995,"15min":0.56299999999999994},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.54000000000000004}]},{"osd":5,"last update":"Thu Aug 29 14:55:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.627,"15min":0.629},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.52800000000000002}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.504},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.54900000000000004}]},{"osd":7,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.55800000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1261568,"data_stored":873188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Aug 29 14:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.34999999999999998,"15min":0.32700000000000001},"min":{"1min":0.25900000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.63600000000000001,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.32900000000000001},"min":{"1min":0.23100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.64300000000000002,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.39000000000000001}]},{"osd":2,"last update":"Thu Aug 29 14:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.40100000000000002,"15min":0.40500000000000003},"min":{"1min":0.251,"5min":0.218,"15min":0.218},"max":{"1min":0.51500000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42899999999999999,"15min":0.42799999999999999},"min":{"1min":0.28100000000000003,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57199999999999995,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.34899999999999998}]},{"osd":3,"last update":"Thu Aug 29 14:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.49399999999999999,"15min":0.501},"min":{"1min":0.33900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60499999999999998,"5min":0.745,"15min":0.745},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.46600000000000003,"15min":0.47299999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.58099999999999996,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.33500000000000002}]},{"osd":4,"last update":"Thu Aug 29 14:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57099999999999995,"15min":0.57299999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72799999999999998,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.68999999999999995,"15min":0.65600000000000003},"min":{"1min":0.46100000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.5700000000000003,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":0.60899999999999999}]},{"osd":5,"last update":"Thu Aug 29 14:55:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.92700000000000005,"15min":0.98899999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78300000000000003,"5min":8.5920000000000005,"15min":8.5920000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81499999999999995,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.59199999999999997}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.63600000000000001}]},{"osd":7,"last update":"Thu Aug 29 14:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":8.5280000000000005,"5min":8.5280000000000005,"15min":8.5280000000000005},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.57599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T14:55:19.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T14:55:19.556 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:19 smithi027 ceph-mon[18892]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:19.556 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:19 smithi027 ceph-mon[18892]: from='client.14577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:55:19.556 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:19 smithi027 ceph-mon[29342]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:19.556 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:19 smithi027 ceph-mon[29342]: from='client.14577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:55:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:19 smithi171 ceph-mon[22959]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:19 smithi171 ceph-mon[22959]: from='client.14577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:55:19.771 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:20.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:20.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:20.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[29342]: Reconfiguring daemon alertmanager.a on smithi027 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:55:20.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:20.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-29T14:55:20.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:20 smithi027 ceph-mon[18892]: Reconfiguring daemon alertmanager.a on smithi027 2024-08-29T14:55:21.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:55:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-29T14:55:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:20 smithi171 ceph-mon[22959]: Reconfiguring daemon alertmanager.a on smithi027 2024-08-29T14:55:22.380 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:55:22.380 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T14:55:22.859 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:22 smithi027 ceph-mon[29342]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:22.859 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:22 smithi027 ceph-mon[29342]: from='client.14583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:55:22.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:22 smithi027 ceph-mon[18892]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:22.860 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:22 smithi027 ceph-mon[18892]: from='client.14583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:55:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:22 smithi171 ceph-mon[22959]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:22 smithi171 ceph-mon[22959]: from='client.14583 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:55:23.717 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-08-29T14:55:21.560791+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227232,"kb_used_data":6976,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733984,"statfs":{"total":767960285184,"available":767727599616,"internally_reserved":0,"allocated":7143424,"data_stored":4033813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001666"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:30.326244+0000","last_change":"2024-08-29T14:54:23.263882+0000","last_active":"2024-08-29T14:54:30.326244+0000","last_peered":"2024-08-29T14:54:30.326244+0000","last_clean":"2024-08-29T14:54:30.326244+0000","last_became_active":"2024-08-29T14:54:18.559285+0000","last_became_peered":"2024-08-29T14:54:18.559285+0000","last_unstale":"2024-08-29T14:54:30.326244+0000","last_undegraded":"2024-08-29T14:54:30.326244+0000","last_fullsized":"2024-08-29T14:54:30.326244+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:26:21.042994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027796,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.451774+0000","last_change":"2024-08-29T14:54:23.263862+0000","last_active":"2024-08-29T14:54:29.451774+0000","last_peered":"2024-08-29T14:54:29.451774+0000","last_clean":"2024-08-29T14:54:29.451774+0000","last_became_active":"2024-08-29T14:54:18.771613+0000","last_became_peered":"2024-08-29T14:54:18.771613+0000","last_unstale":"2024-08-29T14:54:29.451774+0000","last_undegraded":"2024-08-29T14:54:29.451774+0000","last_fullsized":"2024-08-29T14:54:29.451774+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:22:56.189804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029192399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.534229+0000","last_change":"2024-08-29T14:54:23.447440+0000","last_active":"2024-08-29T14:54:29.534229+0000","last_peered":"2024-08-29T14:54:29.534229+0000","last_clean":"2024-08-29T14:54:29.534229+0000","last_became_active":"2024-08-29T14:54:18.771432+0000","last_became_peered":"2024-08-29T14:54:18.771432+0000","last_unstale":"2024-08-29T14:54:29.534229+0000","last_undegraded":"2024-08-29T14:54:29.534229+0000","last_fullsized":"2024-08-29T14:54:29.534229+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:18:29.200943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033832300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.451776+0000","last_change":"2024-08-29T14:54:23.263824+0000","last_active":"2024-08-29T14:54:29.451776+0000","last_peered":"2024-08-29T14:54:29.451776+0000","last_clean":"2024-08-29T14:54:29.451776+0000","last_became_active":"2024-08-29T14:54:18.562017+0000","last_became_peered":"2024-08-29T14:54:18.562017+0000","last_unstale":"2024-08-29T14:54:29.451776+0000","last_undegraded":"2024-08-29T14:54:29.451776+0000","last_fullsized":"2024-08-29T14:54:29.451776+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:07:43.010038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000266333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.978186+0000","last_change":"2024-08-29T14:54:23.262948+0000","last_active":"2024-08-29T14:54:29.978186+0000","last_peered":"2024-08-29T14:54:29.978186+0000","last_clean":"2024-08-29T14:54:29.978186+0000","last_became_active":"2024-08-29T14:54:18.561354+0000","last_became_peered":"2024-08-29T14:54:18.561354+0000","last_unstale":"2024-08-29T14:54:29.978186+0000","last_undegraded":"2024-08-29T14:54:29.978186+0000","last_fullsized":"2024-08-29T14:54:29.978186+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:14.431009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093147100000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:30.326032+0000","last_change":"2024-08-29T14:54:23.256858+0000","last_active":"2024-08-29T14:54:30.326032+0000","last_peered":"2024-08-29T14:54:30.326032+0000","last_clean":"2024-08-29T14:54:30.326032+0000","last_became_active":"2024-08-29T14:54:18.877989+0000","last_became_peered":"2024-08-29T14:54:18.877989+0000","last_unstale":"2024-08-29T14:54:30.326032+0000","last_undegraded":"2024-08-29T14:54:30.326032+0000","last_fullsized":"2024-08-29T14:54:30.326032+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:25:02.401747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033926300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.534187+0000","last_change":"2024-08-29T14:54:23.447441+0000","last_active":"2024-08-29T14:54:29.534187+0000","last_peered":"2024-08-29T14:54:29.534187+0000","last_clean":"2024-08-29T14:54:29.534187+0000","last_became_active":"2024-08-29T14:54:18.562058+0000","last_became_peered":"2024-08-29T14:54:18.562058+0000","last_unstale":"2024-08-29T14:54:29.534187+0000","last_undegraded":"2024-08-29T14:54:29.534187+0000","last_fullsized":"2024-08-29T14:54:29.534187+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:27:17.193516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032779599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-29T14:54:29.978195+0000","last_change":"2024-08-29T14:54:23.263036+0000","last_active":"2024-08-29T14:54:29.978195+0000","last_peered":"2024-08-29T14:54:29.978195+0000","last_clean":"2024-08-29T14:54:29.978195+0000","last_became_active":"2024-08-29T14:54:18.558518+0000","last_became_peered":"2024-08-29T14:54:18.558518+0000","last_unstale":"2024-08-29T14:54:29.978195+0000","last_undegraded":"2024-08-29T14:54:29.978195+0000","last_fullsized":"2024-08-29T14:54:29.978195+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:16.144414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001015642,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":262,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-08-29T14:54:23.447102+0000","last_change":"2024-08-29T14:53:48.474059+0000","last_active":"2024-08-29T14:54:23.447102+0000","last_peered":"2024-08-29T14:54:23.447102+0000","last_clean":"2024-08-29T14:54:23.447102+0000","last_became_active":"2024-08-29T14:53:48.473622+0000","last_became_peered":"2024-08-29T14:53:48.473622+0000","last_unstale":"2024-08-29T14:54:23.447102+0000","last_undegraded":"2024-08-29T14:54:23.447102+0000","last_fullsized":"2024-08-29T14:54:23.447102+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:51:04.923918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.55400000000000005}]},{"osd":1,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.442}]},{"osd":2,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.502},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.51200000000000001}]},{"osd":3,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.47299999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.379},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.54000000000000004}]},{"osd":5,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.48799999999999999}]},{"osd":6,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.41099999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495642,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56399999999999995}]},{"osd":1,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.55100000000000005}]},{"osd":2,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.504},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.58799999999999997}]},{"osd":3,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.46899999999999997}]},{"osd":4,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.51300000000000001}]},{"osd":5,"last update":"Thu Aug 29 14:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.48099999999999998}]},{"osd":7,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.496},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.63600000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":282781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.60599999999999998,"5min":0.747,"15min":0.747},"last":0.441},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.53300000000000003,"15min":0.52500000000000002},"min":{"1min":0.40600000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.4390000000000001,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.60399999999999998}]},{"osd":1,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73799999999999999,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.50800000000000001,"15min":0.50600000000000001},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71199999999999997,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.55900000000000005}]},{"osd":2,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.503,"15min":0.50600000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.54200000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69399999999999995,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54000000000000004}]},{"osd":3,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51700000000000002,"15min":0.51900000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.68999999999999995,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.53500000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.63400000000000001,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.47999999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:55:02 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.376,"15min":0.36599999999999999},"min":{"1min":0.26600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.374,"15min":0.36799999999999999},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.58999999999999997,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.46600000000000003}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Aug 29 14:54:49 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.62}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27672,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717480,"statfs":{"total":95995035648,"available":95966699520,"internally_reserved":0,"allocated":667648,"data_stored":282820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.57099999999999995,"15min":0.57599999999999996},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.72999999999999998,"5min":0.871,"15min":0.871},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.71499999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.71399999999999997}]},{"osd":1,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.38,"5min":0.375,"15min":0.375},"max":{"1min":0.68899999999999995,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.34799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.69599999999999995,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.57899999999999996}]},{"osd":2,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70399999999999996,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38500000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73599999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65300000000000002}]},{"osd":3,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.41299999999999998,"5min":0.375,"15min":0.375},"max":{"1min":0.72199999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40699999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.72199999999999998,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.73299999999999998}]},{"osd":5,"last update":"Thu Aug 29 14:55:03 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.42199999999999999,"15min":0.41199999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.442,"15min":0.434},"min":{"1min":0.32900000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.628}]},{"osd":6,"last update":"Thu Aug 29 14:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.495},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51600000000000001}]},{"osd":7,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60999999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182441,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.379,"15min":0.371},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41599999999999998,"15min":0.41699999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.51200000000000001}]},{"osd":1,"last update":"Thu Aug 29 14:55:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.41199999999999998,"15min":0.42399999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41899999999999998,"15min":0.42499999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.497}]},{"osd":2,"last update":"Thu Aug 29 14:55:18 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44600000000000001,"15min":0.45900000000000002},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.59499999999999997,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.45100000000000001,"15min":0.46300000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.52800000000000002}]},{"osd":4,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.54700000000000004,"15min":0.54600000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.878},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.58199999999999996,"15min":0.58499999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.73099999999999998,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.88600000000000001}]},{"osd":5,"last update":"Thu Aug 29 14:55:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.45100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73599999999999999,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.55400000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.79900000000000004}]},{"osd":6,"last update":"Thu Aug 29 14:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.95499999999999996}]},{"osd":7,"last update":"Thu Aug 29 14:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.73699999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31764,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713388,"statfs":{"total":95995035648,"available":95962509312,"internally_reserved":0,"allocated":663552,"data_stored":282151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.34599999999999997},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.52400000000000002,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39000000000000001,"15min":0.38},"min":{"1min":0.23499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.56599999999999995,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.35499999999999998}]},{"osd":1,"last update":"Thu Aug 29 14:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.39300000000000002,"15min":0.374},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.60599999999999998,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.378,"15min":0.371},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.58399999999999996,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.318}]},{"osd":3,"last update":"Thu Aug 29 14:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.46400000000000002,"15min":0.46800000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63500000000000001,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41899999999999998,"15min":0.42799999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.38}]},{"osd":4,"last update":"Thu Aug 29 14:54:37 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.58099999999999996,"15min":0.59099999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.94199999999999995,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.63100000000000001,"15min":0.63300000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.90300000000000002,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.47099999999999997}]},{"osd":5,"last update":"Thu Aug 29 14:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.60399999999999998,"15min":0.60599999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.82599999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57399999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81499999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.501}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.40899999999999997}]},{"osd":7,"last update":"Thu Aug 29 14:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0369999999999999,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.439}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":283112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.45400000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.309},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38700000000000001,"15min":0.38400000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.50600000000000001,"5min":0.623,"15min":0.623},"last":0.40300000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:54:55 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.37,"15min":0.36599999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.40000000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.66000000000000003,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.36799999999999999}]},{"osd":3,"last update":"Thu Aug 29 14:55:13 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.45700000000000002,"15min":0.46000000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.434,"15min":0.441},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.5}]},{"osd":4,"last update":"Thu Aug 29 14:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.57899999999999996,"15min":0.58199999999999996},"min":{"1min":0.433,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.751,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.56399999999999995,"15min":0.56299999999999994},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.622}]},{"osd":5,"last update":"Thu Aug 29 14:55:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.627,"15min":0.629},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.56799999999999995}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.51600000000000001}]},{"osd":7,"last update":"Thu Aug 29 14:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.66000000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1261568,"data_stored":873188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Aug 29 14:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.34999999999999998,"15min":0.32700000000000001},"min":{"1min":0.25900000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.63600000000000001,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.251},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.32900000000000001},"min":{"1min":0.23100000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.64300000000000002,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.33500000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":0.40100000000000002,"15min":0.40500000000000003},"min":{"1min":0.251,"5min":0.218,"15min":0.218},"max":{"1min":0.51500000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42899999999999999,"15min":0.42799999999999999},"min":{"1min":0.28100000000000003,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.57199999999999995,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.31900000000000001}]},{"osd":3,"last update":"Thu Aug 29 14:55:16 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.48099999999999998,"15min":0.497},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":0.745,"15min":0.745},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.45900000000000002,"15min":0.47099999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.68000000000000005,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.377}]},{"osd":4,"last update":"Thu Aug 29 14:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.57099999999999995,"15min":0.57299999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72799999999999998,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.68999999999999995,"15min":0.65600000000000003},"min":{"1min":0.46100000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.5700000000000003,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":0.47099999999999997}]},{"osd":5,"last update":"Thu Aug 29 14:55:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.92700000000000005,"15min":0.98899999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78300000000000003,"5min":8.5920000000000005,"15min":8.5920000000000005},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81499999999999995,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.54100000000000004}]},{"osd":6,"last update":"Thu Aug 29 14:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.55000000000000004}]},{"osd":7,"last update":"Thu Aug 29 14:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":8.5280000000000005,"5min":8.5280000000000005,"15min":8.5280000000000005},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T14:55:23.718 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-29T14:55:23.719 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-29T14:55:23.719 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-29T14:55:23.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph health --format=json 2024-08-29T14:55:24.800 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:24.925 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:24 smithi027 ceph-mon[18892]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:24.925 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:24 smithi027 ceph-mon[29342]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:24.925 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:24 smithi027 systemd[1]: Stopping Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:55:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:24 smithi171 ceph-mon[22959]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:25.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:25 smithi027 podman[62711]: 2024-08-29 14:55:25.242011183 +0000 UTC m=+0.433699023 container died b8e10dc8eb0fc63e1dcadb110305d3c4241474cdd42c2e9da452b13a62e97a10 (image=quay.ceph.io/ceph-ci/ceph:reef, name=inspiring_lichterman, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=reef-fe43f1c, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-08-29T14:55:25.988 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:25 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[58258]: ts=2024-08-29T14:55:25.587Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-29T14:55:26.318 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:26 smithi027 podman[62770]: 2024-08-29 14:55:26.057805538 +0000 UTC m=+0.610005406 container died 7493c9a457ffb4e132b7647f65c69f88683ec7b06102b3ec8fa05b1d191f2fb3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:26.642 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:26 smithi027 podman[62770]: 2024-08-29 14:55:26.448871968 +0000 UTC m=+1.001071825 container cleanup 7493c9a457ffb4e132b7647f65c69f88683ec7b06102b3ec8fa05b1d191f2fb3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:26.642 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:26 smithi027 bash[62770]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-29T14:55:26.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:26 smithi027 ceph-mon[18892]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:26.902 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:26 smithi027 ceph-mon[29342]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:26.902 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:26 smithi027 podman[62782]: 2024-08-29 14:55:26.641578837 +0000 UTC m=+1.015299969 container remove 7493c9a457ffb4e132b7647f65c69f88683ec7b06102b3ec8fa05b1d191f2fb3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:26.902 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:26 smithi027 podman[62782]: 2024-08-29 14:55:26.708903855 +0000 UTC m=+1.082624942 volume remove 6c8773304c2d3b76b41f0ec73148ad90c4bc75dcb5fe6ec45d1a79501964d313 2024-08-29T14:55:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:26 smithi171 ceph-mon[22959]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:27.475 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:55:27.475 INFO:teuthology.orchestra.run.smithi027.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-29T14:55:27.476 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:27 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-29T14:55:27.476 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:27 smithi027 systemd[1]: Stopped Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:55:27.476 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:27 smithi027 systemd[1]: Starting Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:55:27.815 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:27 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:55:27] "GET /metrics HTTP/1.1" 200 28503 "" "Prometheus/2.43.0" 2024-08-29T14:55:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:27 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/101621057' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-29T14:55:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:27 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/101621057' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-29T14:55:28.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:27 smithi027 podman[62923]: 2024-08-29 14:55:27.822548534 +0000 UTC m=+0.330563698 volume create bba05364115ee27b29647b71454078ad22fc5f1065998066cda9ef39b3b72790 2024-08-29T14:55:28.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:27 smithi027 podman[62923]: 2024-08-29 14:55:27.738605741 +0000 UTC m=+0.246620905 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-29T14:55:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:27 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/101621057' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-29T14:55:28.283 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-29T14:55:28.283 INFO:tasks.cephadm:Setup complete, yielding 2024-08-29T14:55:28.283 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-08-29T14:55:28.293 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-08-29T14:55:28.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-08-29T14:55:28.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 podman[62923]: 2024-08-29 14:55:28.451042134 +0000 UTC m=+0.959057340 container create 2e3fd226c647821b371b4fa73e6ccf8f963ca285fd883615c8b59fcab5e71022 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:28.964 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-mon[29342]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:29.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-mon[18892]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:29.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 podman[62923]: 2024-08-29 14:55:28.779056432 +0000 UTC m=+1.287071597 container init 2e3fd226c647821b371b4fa73e6ccf8f963ca285fd883615c8b59fcab5e71022 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:29.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 podman[62923]: 2024-08-29 14:55:28.782175831 +0000 UTC m=+1.290190998 container start 2e3fd226c647821b371b4fa73e6ccf8f963ca285fd883615c8b59fcab5e71022 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.795Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.795Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.796Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.797Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.820Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.821Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.824Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:28.824Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 bash[62923]: 2e3fd226c647821b371b4fa73e6ccf8f963ca285fd883615c8b59fcab5e71022 2024-08-29T14:55:29.069 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:28 smithi027 systemd[1]: Started Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:55:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:28 smithi171 ceph-mon[22959]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:29 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:30.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:29 smithi171 ceph-mon[22959]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:55:30.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:29 smithi171 ceph-mon[22959]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:55:30.293 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:30.293 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:30.293 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[29342]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:55:30.293 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[29342]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:55:30.294 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:30.294 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:30.294 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[18892]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:55:30.294 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:29 smithi027 ceph-mon[18892]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:55:31.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:30 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:30.798Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000558131s 2024-08-29T14:55:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:31 smithi171 ceph-mon[22959]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:31 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:31 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-29T14:55:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:31 smithi027 ceph-mon[18892]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:31 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:31 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-29T14:55:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:31 smithi027 ceph-mon[29342]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:31 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:31.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:31 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-29T14:55:32.395 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 systemd[1]: Stopping Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:55:32.746 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[29342]: from='client.14595 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:55:32.746 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-29T14:55:32.747 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[29342]: osdmap e67: 8 total, 8 up, 8 in 2024-08-29T14:55:32.747 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-29T14:55:32.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[18892]: from='client.14595 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:55:32.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-29T14:55:32.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[18892]: osdmap e67: 8 total, 8 up, 8 in 2024-08-29T14:55:32.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-29T14:55:32.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:32 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[18888]: 2024-08-29T14:55:32.404+0000 7f2a9dc84640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-29T14:55:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-mon[22959]: from='client.14595 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:55:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-29T14:55:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-mon[22959]: osdmap e67: 8 total, 8 up, 8 in 2024-08-29T14:55:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.620Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-29T14:55:32.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[49944]: ts=2024-08-29T14:55:32.621Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-29T14:55:32.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:32 smithi171 podman[53020]: 2024-08-29 14:55:32.630931076 +0000 UTC m=+0.161935600 container died 232a8598fe9ddd33c65319bcdf1a06474377a87038d2b5ee2d23cac1b77da90d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:33.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-08-29T14:55:33.267 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 podman[53020]: 2024-08-29 14:55:33.062044549 +0000 UTC m=+0.593049071 container cleanup 232a8598fe9ddd33c65319bcdf1a06474377a87038d2b5ee2d23cac1b77da90d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:33.267 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 bash[53020]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a 2024-08-29T14:55:33.402 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:33.529 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: pgmap v49: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:33.529 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T14:55:33.529 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: osdmap e68: 8 total, 8 up, 8 in 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: osdmap e69: 8 total, 8 up, 8 in 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: fsmap foo:0 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: Saving service mds.foo spec with placement count:2 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: pgmap v49: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:33.530 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: osdmap e68: 8 total, 8 up, 8 in 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: osdmap e69: 8 total, 8 up, 8 in 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: fsmap foo:0 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: Saving service mds.foo spec with placement count:2 2024-08-29T14:55:33.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:33 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:33.619 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 podman[53032]: 2024-08-29 14:55:33.363640196 +0000 UTC m=+0.734411064 container remove 232a8598fe9ddd33c65319bcdf1a06474377a87038d2b5ee2d23cac1b77da90d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: pgmap v49: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: osdmap e68: 8 total, 8 up, 8 in 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: osdmap e69: 8 total, 8 up, 8 in 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-29T14:55:33.620 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: fsmap foo:0 2024-08-29T14:55:33.621 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: Saving service mds.foo spec with placement count:2 2024-08-29T14:55:33.621 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:33 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:33.954 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-29T14:55:33.954 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 systemd[1]: Stopped Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:55:33.954 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.129s CPU time. 2024-08-29T14:55:33.954 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 systemd[1]: Starting Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:55:33.954 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 podman[53123]: 2024-08-29 14:55:33.745860395 +0000 UTC m=+0.017090586 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-29T14:55:33.954 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:33 smithi171 podman[53123]: 2024-08-29 14:55:33.847894705 +0000 UTC m=+0.119124876 container create b057ad21b4045311f746be3b041c3ddeef92c6d39dc6369e3bd40be5a36a3edf (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 podman[53123]: 2024-08-29 14:55:34.050171639 +0000 UTC m=+0.321401818 container init b057ad21b4045311f746be3b041c3ddeef92c6d39dc6369e3bd40be5a36a3edf (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 podman[53123]: 2024-08-29 14:55:34.052905892 +0000 UTC m=+0.324136065 container start b057ad21b4045311f746be3b041c3ddeef92c6d39dc6369e3bd40be5a36a3edf (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.077Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.077Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.077Z caller=main.go:570 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi171 (none))" 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.077Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.077Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-29T14:55:34.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.079Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.079Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.080Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.080Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.082Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.083Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.565µs 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.083Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.084Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.084Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.084Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.545µs wal_replay_duration=1.466133ms wbl_replay_duration=95ns total_replay_duration=1.516678ms 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.085Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-29T14:55:34.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.085Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-29T14:55:34.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.085Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-29T14:55:34.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.105Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.987222ms db_storage=1.07µs remote_storage=996ns web_handler=272ns query_engine=552ns scrape=382.353µs scrape_sd=88.583µs notify=10.156µs notify_sd=8.052µs rules=19.196606ms tracing=6.284µs 2024-08-29T14:55:34.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.105Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-29T14:55:34.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:55:34.105Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-29T14:55:34.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 bash[53123]: b057ad21b4045311f746be3b041c3ddeef92c6d39dc6369e3bd40be5a36a3edf 2024-08-29T14:55:34.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:55:34 smithi171 systemd[1]: Started Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:55:34.536 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:34] ENGINE Bus STOPPING 2024-08-29T14:55:34.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: osdmap e70: 8 total, 8 up, 8 in 2024-08-29T14:55:34.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:34.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: osdmap e70: 8 total, 8 up, 8 in 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: osdmap e70: 8 total, 8 up, 8 in 2024-08-29T14:55:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:34.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:34 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:34.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:34] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:55:34.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:34] ENGINE Bus STOPPED 2024-08-29T14:55:34.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:34] ENGINE Bus STARTING 2024-08-29T14:55:34.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:34] ENGINE Serving on http://:::9283 2024-08-29T14:55:34.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:34] ENGINE Bus STARTED 2024-08-29T14:55:34.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:34 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:34] ENGINE Bus STOPPING 2024-08-29T14:55:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: pgmap v53: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T14:55:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-08-29T14:55:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:55:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-08-29T14:55:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[18892]: osdmap e71: 8 total, 8 up, 8 in 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: pgmap v53: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-08-29T14:55:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T14:55:35.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-mon[29342]: osdmap e71: 8 total, 8 up, 8 in 2024-08-29T14:55:35.570 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:55:35.570 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Bus STOPPED 2024-08-29T14:55:35.570 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Bus STARTING 2024-08-29T14:55:35.570 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Serving on http://:::9283 2024-08-29T14:55:35.570 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Bus STARTED 2024-08-29T14:55:35.570 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Bus STOPPING 2024-08-29T14:55:35.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: pgmap v53: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:35.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T14:55:35.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-08-29T14:55:35.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:55:35.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171.front.sepia.ceph.com:9095"}]: dispatch 2024-08-29T14:55:35.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T14:55:35.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171.front.sepia.ceph.com:3000"}]: dispatch 2024-08-29T14:55:35.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T14:55:35.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:35 smithi171 ceph-mon[22959]: osdmap e71: 8 total, 8 up, 8 in 2024-08-29T14:55:35.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:55:35.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Bus STOPPED 2024-08-29T14:55:35.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Bus STARTING 2024-08-29T14:55:35.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Serving on http://:::9283 2024-08-29T14:55:35.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:35 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:55:35] ENGINE Bus STARTED 2024-08-29T14:55:36.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-08-29T14:55:36.584 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:37 smithi171 ceph-mon[22959]: pgmap v55: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:37.749 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:37 smithi027 ceph-mon[29342]: pgmap v55: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:37.749 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:37 smithi027 ceph-mon[18892]: pgmap v55: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:38.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:38 smithi027 ceph-mon[29342]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:38.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:38 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:38.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:38 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:38.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:38 smithi027 ceph-mon[18892]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:38.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:38 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:38.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:38 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:38 smithi171 ceph-mon[22959]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:38 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:38 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:39.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 14:55:38 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T14:55:38.799Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001973591s 2024-08-29T14:55:40.021 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-29T14:55:40.030 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-29T14:55:40.030 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-29T14:55:40.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph mgr dump --format=json 2024-08-29T14:55:40.428 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:41.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:41.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.642 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[18892]: Deploying daemon mds.foo.smithi027.afjjno on smithi027 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:55:41.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.644 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-29T14:55:41.644 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-29T14:55:41.644 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:41.644 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:41 smithi027 ceph-mon[29342]: Deploying daemon mds.foo.smithi027.afjjno on smithi027 2024-08-29T14:55:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:55:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-29T14:55:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-29T14:55:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:41 smithi171 ceph-mon[22959]: Deploying daemon mds.foo.smithi027.afjjno on smithi027 2024-08-29T14:55:43.476 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:55:43.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:43 smithi027 ceph-mon[29342]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:43.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:43 smithi027 ceph-mon[18892]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:43 smithi171 ceph-mon[22959]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:44.120 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":21,"flags":0,"active_gid":14403,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2650682884},{"type":"v1","addr":"172.21.15.27:6801","nonce":2650682884}]},"active_addr":"172.21.15.27:6801/2650682884","active_change":"2024-08-29T14:54:15.540026+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":2955701830}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":921286138}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":1523003109}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":2657305076}]}]} 2024-08-29T14:55:44.125 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-29T14:55:44.125 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-29T14:55:44.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T14:55:44.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:44 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1155872386' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-29T14:55:44.610 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:44 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1155872386' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-29T14:55:44.724 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:44 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1155872386' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-29T14:55:45.479 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:45 smithi171 ceph-mon[22959]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:45.479 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:45 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.479 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:45 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.479 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:45 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.479 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:45 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-29T14:55:45.479 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:45 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-29T14:55:45.479 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:45 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[18892]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[29342]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-29T14:55:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-29T14:55:45.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:45 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:55:46.428 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:55:46.428 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":71,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T14:55:34.410713+0000","last_up_change":"2024-08-29T14:53:44.442031+0000","last_in_change":"2024-08-29T14:53:28.429586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-29T14:55:31.336677+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-29T14:55:31.828806+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6803","nonce":1938465338}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6805","nonce":1938465338}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6809","nonce":1938465338}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6807","nonce":1938465338}]},"public_addr":"172.21.15.27:6803/1938465338","cluster_addr":"172.21.15.27:6805/1938465338","heartbeat_back_addr":"172.21.15.27:6809/1938465338","heartbeat_front_addr":"172.21.15.27:6807/1938465338","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6811","nonce":2967503746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6813","nonce":2967503746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6817","nonce":2967503746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6815","nonce":2967503746}]},"public_addr":"172.21.15.27:6811/2967503746","cluster_addr":"172.21.15.27:6813/2967503746","heartbeat_back_addr":"172.21.15.27:6817/2967503746","heartbeat_front_addr":"172.21.15.27:6815/2967503746","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6819","nonce":302366028}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6821","nonce":302366028}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6825","nonce":302366028}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6823","nonce":302366028}]},"public_addr":"172.21.15.27:6819/302366028","cluster_addr":"172.21.15.27:6821/302366028","heartbeat_back_addr":"172.21.15.27:6825/302366028","heartbeat_front_addr":"172.21.15.27:6823/302366028","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6827","nonce":250460291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6829","nonce":250460291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6833","nonce":250460291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6831","nonce":250460291}]},"public_addr":"172.21.15.27:6827/250460291","cluster_addr":"172.21.15.27:6829/250460291","heartbeat_back_addr":"172.21.15.27:6833/250460291","heartbeat_front_addr":"172.21.15.27:6831/250460291","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6809","nonce":223130519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6811","nonce":223130519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6815","nonce":223130519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6813","nonce":223130519}]},"public_addr":"172.21.15.171:6809/223130519","cluster_addr":"172.21.15.171:6811/223130519","heartbeat_back_addr":"172.21.15.171:6815/223130519","heartbeat_front_addr":"172.21.15.171:6813/223130519","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: Deploying daemon mds.foo.smithi171.grbaeh on smithi171 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:boot 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: daemon mds.foo.smithi027.afjjno assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: Cluster is now healthy 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: fsmap foo:0 1 up:standby 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:creating} 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: daemon mds.foo.smithi027.afjjno is now active in filesystem foo as rank 0 2024-08-29T14:55:46.503 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:46 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: Deploying daemon mds.foo.smithi171.grbaeh on smithi171 2024-08-29T14:55:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:boot 2024-08-29T14:55:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: daemon mds.foo.smithi027.afjjno assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-29T14:55:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-29T14:55:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: Cluster is now healthy 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: fsmap foo:0 1 up:standby 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: fsmap foo:1 {0=foo.smithi027.afjjno=up:creating} 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: daemon mds.foo.smithi027.afjjno is now active in filesystem foo as rank 0 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: Deploying daemon mds.foo.smithi171.grbaeh on smithi171 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:boot 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: daemon mds.foo.smithi027.afjjno assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-29T14:55:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: Cluster is now healthy 2024-08-29T14:55:46.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: fsmap foo:0 1 up:standby 2024-08-29T14:55:46.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:55:46.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: fsmap foo:1 {0=foo.smithi027.afjjno=up:creating} 2024-08-29T14:55:46.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: daemon mds.foo.smithi027.afjjno is now active in filesystem foo as rank 0 2024-08-29T14:55:46.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:46 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:55:47.191 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-29T14:55:47.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T14:55:47.405 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:active 2024-08-29T14:55:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 2024-08-29T14:55:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1093640354' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:55:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:47 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:active 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1093640354' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:55:47.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:active 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1093640354' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:47.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:55:47.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:55:47] "GET /metrics HTTP/1.1" 200 33416 "" "Prometheus/2.43.0" 2024-08-29T14:55:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:48 smithi171 ceph-mon[22959]: mds.? [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] up:boot 2024-08-29T14:55:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:48 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:48 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:55:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:48 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[29342]: mds.? [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] up:boot 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[29342]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[29342]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[18892]: mds.? [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] up:boot 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[18892]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:55:48.713 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:48 smithi027 ceph-mon[18892]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:49.125 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:55:49.125 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":71,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T14:55:34.410713+0000","last_up_change":"2024-08-29T14:53:44.442031+0000","last_in_change":"2024-08-29T14:53:28.429586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-29T14:55:31.336677+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-29T14:55:31.828806+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6803","nonce":1938465338}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6805","nonce":1938465338}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6809","nonce":1938465338}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1938465338},{"type":"v1","addr":"172.21.15.27:6807","nonce":1938465338}]},"public_addr":"172.21.15.27:6803/1938465338","cluster_addr":"172.21.15.27:6805/1938465338","heartbeat_back_addr":"172.21.15.27:6809/1938465338","heartbeat_front_addr":"172.21.15.27:6807/1938465338","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6811","nonce":2967503746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6813","nonce":2967503746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6817","nonce":2967503746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2967503746},{"type":"v1","addr":"172.21.15.27:6815","nonce":2967503746}]},"public_addr":"172.21.15.27:6811/2967503746","cluster_addr":"172.21.15.27:6813/2967503746","heartbeat_back_addr":"172.21.15.27:6817/2967503746","heartbeat_front_addr":"172.21.15.27:6815/2967503746","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6819","nonce":302366028}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6821","nonce":302366028}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6825","nonce":302366028}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":302366028},{"type":"v1","addr":"172.21.15.27:6823","nonce":302366028}]},"public_addr":"172.21.15.27:6819/302366028","cluster_addr":"172.21.15.27:6821/302366028","heartbeat_back_addr":"172.21.15.27:6825/302366028","heartbeat_front_addr":"172.21.15.27:6823/302366028","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6827","nonce":250460291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6829","nonce":250460291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6833","nonce":250460291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":250460291},{"type":"v1","addr":"172.21.15.27:6831","nonce":250460291}]},"public_addr":"172.21.15.27:6827/250460291","cluster_addr":"172.21.15.27:6829/250460291","heartbeat_back_addr":"172.21.15.27:6833/250460291","heartbeat_front_addr":"172.21.15.27:6831/250460291","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6809","nonce":223130519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6811","nonce":223130519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6815","nonce":223130519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6813","nonce":223130519}]},"public_addr":"172.21.15.171:6809/223130519","cluster_addr":"172.21.15.171:6811/223130519","heartbeat_back_addr":"172.21.15.171:6815/223130519","heartbeat_front_addr":"172.21.15.171:6813/223130519","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T14:55:49.654 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:49 smithi171 ceph-mon[22959]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:49.654 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:49 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1389854269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:55:49.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:49 smithi027 ceph-mon[29342]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:49.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:49 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1389854269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:55:49.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:49 smithi027 ceph-mon[18892]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:49.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:49 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1389854269' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-29T14:55:50.277 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-29T14:55:50.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-29T14:55:50.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-29T14:55:50.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-29T14:55:50.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-29T14:55:50.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-29T14:55:50.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-29T14:55:50.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-29T14:55:50.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:50 smithi027 ceph-mon[18892]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:active 2024-08-29T14:55:50.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:50 smithi027 ceph-mon[18892]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:50.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:50 smithi027 ceph-mon[29342]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:active 2024-08-29T14:55:50.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:50 smithi027 ceph-mon[29342]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:50.624 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:50 smithi171 ceph-mon[22959]: mds.? [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] up:active 2024-08-29T14:55:50.625 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:50 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:50.807 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:50.809 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:50.812 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:50.813 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:50.813 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:50.816 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:50.822 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:50.823 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:55:51.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[29342]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:51.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[18892]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:51.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:51 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:51 smithi171 ceph-mon[22959]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:51 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:51 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:51 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:55:53.201 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:52 smithi027 ceph-mon[29342]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:53.201 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:52 smithi027 ceph-mon[29342]: mds.? [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] up:standby 2024-08-29T14:55:53.201 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:52 smithi027 ceph-mon[29342]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:53.201 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:52 smithi027 ceph-mon[18892]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:53.202 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:52 smithi027 ceph-mon[18892]: mds.? [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] up:standby 2024-08-29T14:55:53.202 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:52 smithi027 ceph-mon[18892]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:52 smithi171 ceph-mon[22959]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:52 smithi171 ceph-mon[22959]: mds.? [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] up:standby 2024-08-29T14:55:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:52 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T14:55:55.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:54 smithi027 ceph-mon[18892]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:55.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:54 smithi027 ceph-mon[29342]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:54 smithi171 ceph-mon[22959]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:57.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:56 smithi027 ceph-mon[29342]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:57.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:56 smithi027 ceph-mon[18892]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:56 smithi171 ceph-mon[22959]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:57.905 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:55:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:55:57] "GET /metrics HTTP/1.1" 200 33416 "" "Prometheus/2.43.0" 2024-08-29T14:55:59.183 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:55:58 smithi027 ceph-mon[29342]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:59.183 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:55:58 smithi027 ceph-mon[18892]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:55:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:55:58 smithi171 ceph-mon[22959]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-29T14:56:01.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:00 smithi027 ceph-mon[18892]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T14:56:01.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:00 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:00 smithi027 ceph-mon[29342]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T14:56:01.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:00 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:00 smithi171 ceph-mon[22959]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T14:56:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:00 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:01.899 INFO:teuthology.orchestra.run.smithi027.stdout:77309411382 2024-08-29T14:56:01.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-29T14:56:03.183 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:02 smithi027 ceph-mon[29342]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T14:56:03.184 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:02 smithi027 ceph-mon[18892]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T14:56:03.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:02 smithi171 ceph-mon[22959]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T14:56:03.730 INFO:teuthology.orchestra.run.smithi027.stdout:231928234013 2024-08-29T14:56:03.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-29T14:56:03.913 INFO:teuthology.orchestra.run.smithi027.stdout:128849018925 2024-08-29T14:56:03.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-29T14:56:04.886 INFO:teuthology.orchestra.run.smithi027.stdout:197568495651 2024-08-29T14:56:04.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-29T14:56:05.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:04 smithi027 ceph-mon[29342]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:05.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:04 smithi027 ceph-mon[18892]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:04 smithi171 ceph-mon[22959]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:05.901 INFO:teuthology.orchestra.run.smithi027.stdout:107374182449 2024-08-29T14:56:05.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-29T14:56:06.298 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:06.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.300 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:06 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:56:06.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:06 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:06 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:06 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:06.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:06 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:06.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:06 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:06.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:06 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:56:07.254 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:07 smithi027 ceph-mon[29342]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:07.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:07 smithi027 ceph-mon[18892]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:07.276 INFO:teuthology.orchestra.run.smithi027.stdout:34359738433 2024-08-29T14:56:07.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-29T14:56:07.387 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:07 smithi171 ceph-mon[22959]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:07.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:56:07] "GET /metrics HTTP/1.1" 200 33705 "" "Prometheus/2.43.0" 2024-08-29T14:56:07.912 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:09.427 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:09 smithi027 ceph-mon[29342]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:09.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:09 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:09.428 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:09 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:09.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:09 smithi027 ceph-mon[18892]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:09.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:09 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:09.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:09 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:09.459 INFO:teuthology.orchestra.run.smithi027.stdout:163208757289 2024-08-29T14:56:09.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-29T14:56:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:09 smithi171 ceph-mon[22959]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:09 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:09 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:10.033 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:10.034 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:10.036 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:10.071 INFO:teuthology.orchestra.run.smithi027.stdout:77309411384 2024-08-29T14:56:10.409 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:10 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1413252469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-29T14:56:10.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:10 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1413252469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-29T14:56:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:10 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1413252469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-29T14:56:11.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:11 smithi027 ceph-mon[29342]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:11 smithi027 ceph-mon[18892]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:11 smithi171 ceph-mon[22959]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:12.299 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411382 got 77309411384 for osd.2 2024-08-29T14:56:12.299 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:13.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:13 smithi027 ceph-mon[29342]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:13.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:13 smithi027 ceph-mon[18892]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:13.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:13 smithi171 ceph-mon[22959]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:13.648 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:13.827 INFO:teuthology.orchestra.run.smithi027.stdout:197568495653 2024-08-29T14:56:14.024 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:14.259 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:14 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2677186883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-29T14:56:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:14 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2677186883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-29T14:56:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:14 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2677186883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-29T14:56:14.761 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495651 got 197568495653 for osd.6 2024-08-29T14:56:14.761 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:15.369 INFO:teuthology.orchestra.run.smithi027.stdout:231928234017 2024-08-29T14:56:15.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:15 smithi027 ceph-mon[18892]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:15.370 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:15 smithi027 ceph-mon[29342]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:15 smithi171 ceph-mon[22959]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2168704467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2168704467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-08-29T14:56:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-08-29T14:56:16.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:16 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2168704467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-29T14:56:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-08-29T14:56:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-08-29T14:56:16.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-08-29T14:56:16.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:16.657 INFO:teuthology.orchestra.run.smithi027.stdout:55834574909 2024-08-29T14:56:16.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-29T14:56:17.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[29342]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:17.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-29T14:56:17.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-08-29T14:56:17.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-08-29T14:56:17.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[29342]: osdmap e72: 8 total, 8 up, 8 in 2024-08-29T14:56:17.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[18892]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:17.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-29T14:56:17.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-08-29T14:56:17.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-08-29T14:56:17.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-mon[18892]: osdmap e72: 8 total, 8 up, 8 in 2024-08-29T14:56:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:17 smithi171 ceph-mon[22959]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:17 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-29T14:56:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:17 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-08-29T14:56:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:17 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-08-29T14:56:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:17 smithi171 ceph-mon[22959]: osdmap e72: 8 total, 8 up, 8 in 2024-08-29T14:56:17.890 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:56:17] "GET /metrics HTTP/1.1" 200 33700 "" "Prometheus/2.43.0" 2024-08-29T14:56:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:18 smithi171 ceph-mon[22959]: osdmap e73: 8 total, 8 up, 8 in 2024-08-29T14:56:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:18 smithi027 ceph-mon[29342]: osdmap e73: 8 total, 8 up, 8 in 2024-08-29T14:56:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:18 smithi027 ceph-mon[18892]: osdmap e73: 8 total, 8 up, 8 in 2024-08-29T14:56:18.995 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234013 got 231928234017 for osd.7 2024-08-29T14:56:18.995 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:19.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:19 smithi027 ceph-mon[29342]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:19.254 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:19 smithi027 ceph-mon[18892]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:19.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:19 smithi171 ceph-mon[22959]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:20.436 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:21.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:21 smithi027 ceph-mon[29342]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:21.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:21 smithi027 ceph-mon[18892]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:21 smithi171 ceph-mon[22959]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:22.597 INFO:teuthology.orchestra.run.smithi027.stdout:128849018929 2024-08-29T14:56:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:23 smithi171 ceph-mon[22959]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:23 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3914396157' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-29T14:56:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:23 smithi027 ceph-mon[18892]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:23 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3914396157' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-29T14:56:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:23 smithi027 ceph-mon[29342]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:23 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3914396157' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-29T14:56:23.814 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018925 got 128849018929 for osd.4 2024-08-29T14:56:23.814 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:24.218 INFO:teuthology.orchestra.run.smithi027.stdout:163208757292 2024-08-29T14:56:25.012 INFO:teuthology.orchestra.run.smithi027.stdout:55834574912 2024-08-29T14:56:25.264 INFO:teuthology.orchestra.run.smithi027.stdout:107374182454 2024-08-29T14:56:25.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:25 smithi027 ceph-mon[29342]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:25.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:25 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1694173780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-29T14:56:25.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:25 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1357455843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-29T14:56:25.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:25 smithi027 ceph-mon[18892]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:25.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:25 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1694173780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-29T14:56:25.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:25 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1357455843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-29T14:56:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:25 smithi171 ceph-mon[22959]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:25 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1694173780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-29T14:56:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:25 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1357455843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-29T14:56:25.846 INFO:teuthology.orchestra.run.smithi027.stdout:34359738436 2024-08-29T14:56:26.215 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182449 got 107374182454 for osd.3 2024-08-29T14:56:26.215 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:26 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/156150831' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-29T14:56:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:26 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3091051488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-29T14:56:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:26 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/156150831' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-29T14:56:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:26 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3091051488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-29T14:56:26.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:26 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/156150831' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-29T14:56:26.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:26 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3091051488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-29T14:56:26.740 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757289 got 163208757292 for osd.5 2024-08-29T14:56:26.740 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:27.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:27 smithi027 ceph-mon[18892]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:27 smithi027 ceph-mon[29342]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:27 smithi171 ceph-mon[22959]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:27.661 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:27 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:56:27] "GET /metrics HTTP/1.1" 200 33700 "" "Prometheus/2.43.0" 2024-08-29T14:56:27.684 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738436 for osd.0 2024-08-29T14:56:27.684 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:28.090 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574909 got 55834574912 for osd.1 2024-08-29T14:56:28.090 DEBUG:teuthology.parallel:result is None 2024-08-29T14:56:28.090 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-29T14:56:28.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T14:56:28.536 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:29.426 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:29 smithi027 ceph-mon[29342]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:29.426 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:29 smithi027 ceph-mon[18892]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:29 smithi171 ceph-mon[22959]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:30.621 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.621 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:30.622 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:30 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:30 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:30 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:30 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:30 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:30.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:30 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:31.030 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:56:31.031 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T14:56:31.669 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:31 smithi027 ceph-mon[18892]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:31.669 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:31 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:31.669 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:31 smithi027 ceph-mon[29342]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:31.670 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:31 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:31.691 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":84,"stamp":"2024-08-29T14:56:29.575095+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":228012,"kb_used_data":7744,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733204,"statfs":{"total":767960285184,"available":767726800896,"internally_reserved":0,"allocated":7929856,"data_stored":4444923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002664"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.621599+0000","last_change":"2024-08-29T14:56:17.621599+0000","last_active":"2024-08-29T14:56:17.621599+0000","last_peered":"2024-08-29T14:56:17.621599+0000","last_clean":"2024-08-29T14:56:17.621599+0000","last_became_active":"2024-08-29T14:56:17.621289+0000","last_became_peered":"2024-08-29T14:56:17.621289+0000","last_unstale":"2024-08-29T14:56:17.621599+0000","last_undegraded":"2024-08-29T14:56:17.621599+0000","last_fullsized":"2024-08-29T14:56:17.621599+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:14:26.987112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145064+0000","last_change":"2024-08-29T14:55:32.414985+0000","last_active":"2024-08-29T14:56:16.145064+0000","last_peered":"2024-08-29T14:56:16.145064+0000","last_clean":"2024-08-29T14:56:16.145064+0000","last_became_active":"2024-08-29T14:55:32.414740+0000","last_became_peered":"2024-08-29T14:55:32.414740+0000","last_unstale":"2024-08-29T14:56:16.145064+0000","last_undegraded":"2024-08-29T14:56:16.145064+0000","last_fullsized":"2024-08-29T14:56:16.145064+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:51:32.533114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141986+0000","last_change":"2024-08-29T14:55:33.424931+0000","last_active":"2024-08-29T14:56:16.141986+0000","last_peered":"2024-08-29T14:56:16.141986+0000","last_clean":"2024-08-29T14:56:16.141986+0000","last_became_active":"2024-08-29T14:55:33.424592+0000","last_became_peered":"2024-08-29T14:55:33.424592+0000","last_unstale":"2024-08-29T14:56:16.141986+0000","last_undegraded":"2024-08-29T14:56:16.141986+0000","last_fullsized":"2024-08-29T14:56:16.141986+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:46:34.886494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141721+0000","last_change":"2024-08-29T14:55:32.419134+0000","last_active":"2024-08-29T14:56:16.141721+0000","last_peered":"2024-08-29T14:56:16.141721+0000","last_clean":"2024-08-29T14:56:16.141721+0000","last_became_active":"2024-08-29T14:55:32.418858+0000","last_became_peered":"2024-08-29T14:55:32.418858+0000","last_unstale":"2024-08-29T14:56:16.141721+0000","last_undegraded":"2024-08-29T14:56:16.141721+0000","last_fullsized":"2024-08-29T14:56:16.141721+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:26:37.051836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141924+0000","last_change":"2024-08-29T14:55:33.418693+0000","last_active":"2024-08-29T14:56:16.141924+0000","last_peered":"2024-08-29T14:56:16.141924+0000","last_clean":"2024-08-29T14:56:16.141924+0000","last_became_active":"2024-08-29T14:55:33.418417+0000","last_became_peered":"2024-08-29T14:55:33.418417+0000","last_unstale":"2024-08-29T14:56:16.141924+0000","last_undegraded":"2024-08-29T14:56:16.141924+0000","last_fullsized":"2024-08-29T14:56:16.141924+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:10:00.617593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.144982+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T14:56:16.144982+0000","last_peered":"2024-08-29T14:56:16.144982+0000","last_clean":"2024-08-29T14:56:16.144982+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T14:56:16.144982+0000","last_undegraded":"2024-08-29T14:56:16.144982+0000","last_fullsized":"2024-08-29T14:56:16.144982+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143814+0000","last_change":"2024-08-29T14:55:32.458161+0000","last_active":"2024-08-29T14:56:16.143814+0000","last_peered":"2024-08-29T14:56:16.143814+0000","last_clean":"2024-08-29T14:56:16.143814+0000","last_became_active":"2024-08-29T14:55:32.457728+0000","last_became_peered":"2024-08-29T14:55:32.457728+0000","last_unstale":"2024-08-29T14:56:16.143814+0000","last_undegraded":"2024-08-29T14:56:16.143814+0000","last_fullsized":"2024-08-29T14:56:16.143814+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:34:27.153148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141483+0000","last_change":"2024-08-29T14:55:32.416161+0000","last_active":"2024-08-29T14:56:16.141483+0000","last_peered":"2024-08-29T14:56:16.141483+0000","last_clean":"2024-08-29T14:56:16.141483+0000","last_became_active":"2024-08-29T14:55:32.415844+0000","last_became_peered":"2024-08-29T14:55:32.415844+0000","last_unstale":"2024-08-29T14:56:16.141483+0000","last_undegraded":"2024-08-29T14:56:16.141483+0000","last_fullsized":"2024-08-29T14:56:16.141483+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:32.405251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145329+0000","last_change":"2024-08-29T14:55:32.417418+0000","last_active":"2024-08-29T14:56:16.145329+0000","last_peered":"2024-08-29T14:56:16.145329+0000","last_clean":"2024-08-29T14:56:16.145329+0000","last_became_active":"2024-08-29T14:55:32.416972+0000","last_became_peered":"2024-08-29T14:55:32.416972+0000","last_unstale":"2024-08-29T14:56:16.145329+0000","last_undegraded":"2024-08-29T14:56:16.145329+0000","last_fullsized":"2024-08-29T14:56:16.145329+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:03:55.923654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141182+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T14:56:16.141182+0000","last_peered":"2024-08-29T14:56:16.141182+0000","last_clean":"2024-08-29T14:56:16.141182+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T14:56:16.141182+0000","last_undegraded":"2024-08-29T14:56:16.141182+0000","last_fullsized":"2024-08-29T14:56:16.141182+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145274+0000","last_change":"2024-08-29T14:55:32.410470+0000","last_active":"2024-08-29T14:56:16.145274+0000","last_peered":"2024-08-29T14:56:16.145274+0000","last_clean":"2024-08-29T14:56:16.145274+0000","last_became_active":"2024-08-29T14:55:32.410055+0000","last_became_peered":"2024-08-29T14:55:32.410055+0000","last_unstale":"2024-08-29T14:56:16.145274+0000","last_undegraded":"2024-08-29T14:56:16.145274+0000","last_fullsized":"2024-08-29T14:56:16.145274+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:38.263764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138691+0000","last_change":"2024-08-29T14:55:33.427456+0000","last_active":"2024-08-29T14:56:16.138691+0000","last_peered":"2024-08-29T14:56:16.138691+0000","last_clean":"2024-08-29T14:56:16.138691+0000","last_became_active":"2024-08-29T14:55:33.426367+0000","last_became_peered":"2024-08-29T14:55:33.426367+0000","last_unstale":"2024-08-29T14:56:16.138691+0000","last_undegraded":"2024-08-29T14:56:16.138691+0000","last_fullsized":"2024-08-29T14:56:16.138691+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:49:34.584262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145349+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T14:56:16.145349+0000","last_peered":"2024-08-29T14:56:16.145349+0000","last_clean":"2024-08-29T14:56:16.145349+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T14:56:16.145349+0000","last_undegraded":"2024-08-29T14:56:16.145349+0000","last_fullsized":"2024-08-29T14:56:16.145349+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141307+0000","last_change":"2024-08-29T14:55:33.430355+0000","last_active":"2024-08-29T14:56:16.141307+0000","last_peered":"2024-08-29T14:56:16.141307+0000","last_clean":"2024-08-29T14:56:16.141307+0000","last_became_active":"2024-08-29T14:55:33.430104+0000","last_became_peered":"2024-08-29T14:55:33.430104+0000","last_unstale":"2024-08-29T14:56:16.141307+0000","last_undegraded":"2024-08-29T14:56:16.141307+0000","last_fullsized":"2024-08-29T14:56:16.141307+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:00:59.549195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141428+0000","last_change":"2024-08-29T14:55:32.403312+0000","last_active":"2024-08-29T14:56:16.141428+0000","last_peered":"2024-08-29T14:56:16.141428+0000","last_clean":"2024-08-29T14:56:16.141428+0000","last_became_active":"2024-08-29T14:55:32.403016+0000","last_became_peered":"2024-08-29T14:55:32.403016+0000","last_unstale":"2024-08-29T14:56:16.141428+0000","last_undegraded":"2024-08-29T14:56:16.141428+0000","last_fullsized":"2024-08-29T14:56:16.141428+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:47.927639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141688+0000","last_change":"2024-08-29T14:55:33.425651+0000","last_active":"2024-08-29T14:56:16.141688+0000","last_peered":"2024-08-29T14:56:16.141688+0000","last_clean":"2024-08-29T14:56:16.141688+0000","last_became_active":"2024-08-29T14:55:33.425353+0000","last_became_peered":"2024-08-29T14:55:33.425353+0000","last_unstale":"2024-08-29T14:56:16.141688+0000","last_undegraded":"2024-08-29T14:56:16.141688+0000","last_fullsized":"2024-08-29T14:56:16.141688+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:26.644824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143110+0000","last_change":"2024-08-29T14:55:32.416739+0000","last_active":"2024-08-29T14:56:16.143110+0000","last_peered":"2024-08-29T14:56:16.143110+0000","last_clean":"2024-08-29T14:56:16.143110+0000","last_became_active":"2024-08-29T14:55:32.415882+0000","last_became_peered":"2024-08-29T14:55:32.415882+0000","last_unstale":"2024-08-29T14:56:16.143110+0000","last_undegraded":"2024-08-29T14:56:16.143110+0000","last_fullsized":"2024-08-29T14:56:16.143110+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:23:04.499484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138971+0000","last_change":"2024-08-29T14:55:33.426892+0000","last_active":"2024-08-29T14:56:16.138971+0000","last_peered":"2024-08-29T14:56:16.138971+0000","last_clean":"2024-08-29T14:56:16.138971+0000","last_became_active":"2024-08-29T14:55:33.426558+0000","last_became_peered":"2024-08-29T14:55:33.426558+0000","last_unstale":"2024-08-29T14:56:16.138971+0000","last_undegraded":"2024-08-29T14:56:16.138971+0000","last_fullsized":"2024-08-29T14:56:16.138971+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:39:21.945011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141661+0000","last_change":"2024-08-29T14:55:32.409109+0000","last_active":"2024-08-29T14:56:16.141661+0000","last_peered":"2024-08-29T14:56:16.141661+0000","last_clean":"2024-08-29T14:56:16.141661+0000","last_became_active":"2024-08-29T14:55:32.408825+0000","last_became_peered":"2024-08-29T14:55:32.408825+0000","last_unstale":"2024-08-29T14:56:16.141661+0000","last_undegraded":"2024-08-29T14:56:16.141661+0000","last_fullsized":"2024-08-29T14:56:16.141661+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:24:26.892271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.210976+0000","last_change":"2024-08-29T14:56:17.210976+0000","last_active":"2024-08-29T14:56:17.210976+0000","last_peered":"2024-08-29T14:56:17.210976+0000","last_clean":"2024-08-29T14:56:17.210976+0000","last_became_active":"2024-08-29T14:56:17.210693+0000","last_became_peered":"2024-08-29T14:56:17.210693+0000","last_unstale":"2024-08-29T14:56:17.210976+0000","last_undegraded":"2024-08-29T14:56:17.210976+0000","last_fullsized":"2024-08-29T14:56:17.210976+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:03.990347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138750+0000","last_change":"2024-08-29T14:55:32.411681+0000","last_active":"2024-08-29T14:56:16.138750+0000","last_peered":"2024-08-29T14:56:16.138750+0000","last_clean":"2024-08-29T14:56:16.138750+0000","last_became_active":"2024-08-29T14:55:32.411408+0000","last_became_peered":"2024-08-29T14:55:32.411408+0000","last_unstale":"2024-08-29T14:56:16.138750+0000","last_undegraded":"2024-08-29T14:56:16.138750+0000","last_fullsized":"2024-08-29T14:56:16.138750+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:10:28.211248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139182+0000","last_change":"2024-08-29T14:55:33.428638+0000","last_active":"2024-08-29T14:56:16.139182+0000","last_peered":"2024-08-29T14:56:16.139182+0000","last_clean":"2024-08-29T14:56:16.139182+0000","last_became_active":"2024-08-29T14:55:33.428340+0000","last_became_peered":"2024-08-29T14:55:33.428340+0000","last_unstale":"2024-08-29T14:56:16.139182+0000","last_undegraded":"2024-08-29T14:56:16.139182+0000","last_fullsized":"2024-08-29T14:56:16.139182+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:14:40.982367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142342+0000","last_change":"2024-08-29T14:55:32.411188+0000","last_active":"2024-08-29T14:56:16.142342+0000","last_peered":"2024-08-29T14:56:16.142342+0000","last_clean":"2024-08-29T14:56:16.142342+0000","last_became_active":"2024-08-29T14:55:32.410863+0000","last_became_peered":"2024-08-29T14:55:32.410863+0000","last_unstale":"2024-08-29T14:56:16.142342+0000","last_undegraded":"2024-08-29T14:56:16.142342+0000","last_fullsized":"2024-08-29T14:56:16.142342+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:46:39.520261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139528+0000","last_change":"2024-08-29T14:55:33.423535+0000","last_active":"2024-08-29T14:56:16.139528+0000","last_peered":"2024-08-29T14:56:16.139528+0000","last_clean":"2024-08-29T14:56:16.139528+0000","last_became_active":"2024-08-29T14:55:33.422923+0000","last_became_peered":"2024-08-29T14:55:33.422923+0000","last_unstale":"2024-08-29T14:56:16.139528+0000","last_undegraded":"2024-08-29T14:56:16.139528+0000","last_fullsized":"2024-08-29T14:56:16.139528+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:56:15.097853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145098+0000","last_change":"2024-08-29T14:53:48.474059+0000","last_active":"2024-08-29T14:56:16.145098+0000","last_peered":"2024-08-29T14:56:16.145098+0000","last_clean":"2024-08-29T14:56:16.145098+0000","last_became_active":"2024-08-29T14:53:48.473622+0000","last_became_peered":"2024-08-29T14:53:48.473622+0000","last_unstale":"2024-08-29T14:56:16.145098+0000","last_undegraded":"2024-08-29T14:56:16.145098+0000","last_fullsized":"2024-08-29T14:56:16.145098+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:51:04.923918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.210161+0000","last_change":"2024-08-29T14:54:23.263036+0000","last_active":"2024-08-29T14:56:17.210161+0000","last_peered":"2024-08-29T14:56:17.210161+0000","last_clean":"2024-08-29T14:56:17.210161+0000","last_became_active":"2024-08-29T14:54:18.558518+0000","last_became_peered":"2024-08-29T14:54:18.558518+0000","last_unstale":"2024-08-29T14:56:17.210161+0000","last_undegraded":"2024-08-29T14:56:17.210161+0000","last_fullsized":"2024-08-29T14:56:17.210161+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:16.144414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001015642,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142952+0000","last_change":"2024-08-29T14:55:32.406853+0000","last_active":"2024-08-29T14:56:16.142952+0000","last_peered":"2024-08-29T14:56:16.142952+0000","last_clean":"2024-08-29T14:56:16.142952+0000","last_became_active":"2024-08-29T14:55:32.405314+0000","last_became_peered":"2024-08-29T14:55:32.405314+0000","last_unstale":"2024-08-29T14:56:16.142952+0000","last_undegraded":"2024-08-29T14:56:16.142952+0000","last_fullsized":"2024-08-29T14:56:16.142952+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.143846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141541+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T14:56:16.141541+0000","last_peered":"2024-08-29T14:56:16.141541+0000","last_clean":"2024-08-29T14:56:16.141541+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T14:56:16.141541+0000","last_undegraded":"2024-08-29T14:56:16.141541+0000","last_fullsized":"2024-08-29T14:56:16.141541+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.620828+0000","last_change":"2024-08-29T14:54:23.447441+0000","last_active":"2024-08-29T14:56:17.620828+0000","last_peered":"2024-08-29T14:56:17.620828+0000","last_clean":"2024-08-29T14:56:17.620828+0000","last_became_active":"2024-08-29T14:54:18.562058+0000","last_became_peered":"2024-08-29T14:54:18.562058+0000","last_unstale":"2024-08-29T14:56:17.620828+0000","last_undegraded":"2024-08-29T14:56:17.620828+0000","last_fullsized":"2024-08-29T14:56:17.620828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:27:17.193516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032779599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141454+0000","last_change":"2024-08-29T14:55:32.419759+0000","last_active":"2024-08-29T14:56:16.141454+0000","last_peered":"2024-08-29T14:56:16.141454+0000","last_clean":"2024-08-29T14:56:16.141454+0000","last_became_active":"2024-08-29T14:55:32.419469+0000","last_became_peered":"2024-08-29T14:55:32.419469+0000","last_unstale":"2024-08-29T14:56:16.141454+0000","last_undegraded":"2024-08-29T14:56:16.141454+0000","last_fullsized":"2024-08-29T14:56:16.141454+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:33:02.468625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141826+0000","last_change":"2024-08-29T14:55:33.457590+0000","last_active":"2024-08-29T14:56:16.141826+0000","last_peered":"2024-08-29T14:56:16.141826+0000","last_clean":"2024-08-29T14:56:16.141826+0000","last_became_active":"2024-08-29T14:55:33.457258+0000","last_became_peered":"2024-08-29T14:55:33.457258+0000","last_unstale":"2024-08-29T14:56:16.141826+0000","last_undegraded":"2024-08-29T14:56:16.141826+0000","last_fullsized":"2024-08-29T14:56:16.141826+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:36.871100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.144713+0000","last_change":"2024-08-29T14:54:23.256858+0000","last_active":"2024-08-29T14:56:17.144713+0000","last_peered":"2024-08-29T14:56:17.144713+0000","last_clean":"2024-08-29T14:56:17.144713+0000","last_became_active":"2024-08-29T14:54:18.877989+0000","last_became_peered":"2024-08-29T14:54:18.877989+0000","last_unstale":"2024-08-29T14:56:17.144713+0000","last_undegraded":"2024-08-29T14:56:17.144713+0000","last_fullsized":"2024-08-29T14:56:17.144713+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:25:02.401747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033926300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138721+0000","last_change":"2024-08-29T14:55:32.414237+0000","last_active":"2024-08-29T14:56:16.138721+0000","last_peered":"2024-08-29T14:56:16.138721+0000","last_clean":"2024-08-29T14:56:16.138721+0000","last_became_active":"2024-08-29T14:55:32.413566+0000","last_became_peered":"2024-08-29T14:55:32.413566+0000","last_unstale":"2024-08-29T14:56:16.138721+0000","last_undegraded":"2024-08-29T14:56:16.138721+0000","last_fullsized":"2024-08-29T14:56:16.138721+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:42:12.636422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139592+0000","last_change":"2024-08-29T14:55:33.424683+0000","last_active":"2024-08-29T14:56:16.139592+0000","last_peered":"2024-08-29T14:56:16.139592+0000","last_clean":"2024-08-29T14:56:16.139592+0000","last_became_active":"2024-08-29T14:55:33.424387+0000","last_became_peered":"2024-08-29T14:55:33.424387+0000","last_unstale":"2024-08-29T14:56:16.139592+0000","last_undegraded":"2024-08-29T14:56:16.139592+0000","last_fullsized":"2024-08-29T14:56:16.139592+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:32:48.772407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.210267+0000","last_change":"2024-08-29T14:54:23.262948+0000","last_active":"2024-08-29T14:56:17.210267+0000","last_peered":"2024-08-29T14:56:17.210267+0000","last_clean":"2024-08-29T14:56:17.210267+0000","last_became_active":"2024-08-29T14:54:18.561354+0000","last_became_peered":"2024-08-29T14:54:18.561354+0000","last_unstale":"2024-08-29T14:56:17.210267+0000","last_undegraded":"2024-08-29T14:56:17.210267+0000","last_fullsized":"2024-08-29T14:56:17.210267+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:14.431009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093147100000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139515+0000","last_change":"2024-08-29T14:55:32.417502+0000","last_active":"2024-08-29T14:56:16.139515+0000","last_peered":"2024-08-29T14:56:16.139515+0000","last_clean":"2024-08-29T14:56:16.139515+0000","last_became_active":"2024-08-29T14:55:32.417182+0000","last_became_peered":"2024-08-29T14:55:32.417182+0000","last_unstale":"2024-08-29T14:56:16.139515+0000","last_undegraded":"2024-08-29T14:56:16.139515+0000","last_fullsized":"2024-08-29T14:56:16.139515+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:10:10.560754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139776+0000","last_change":"2024-08-29T14:55:33.457850+0000","last_active":"2024-08-29T14:56:16.139776+0000","last_peered":"2024-08-29T14:56:16.139776+0000","last_clean":"2024-08-29T14:56:16.139776+0000","last_became_active":"2024-08-29T14:55:33.457524+0000","last_became_peered":"2024-08-29T14:55:33.457524+0000","last_unstale":"2024-08-29T14:56:16.139776+0000","last_undegraded":"2024-08-29T14:56:16.139776+0000","last_fullsized":"2024-08-29T14:56:16.139776+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:41.003146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142099+0000","last_change":"2024-08-29T14:55:32.403376+0000","last_active":"2024-08-29T14:56:16.142099+0000","last_peered":"2024-08-29T14:56:16.142099+0000","last_clean":"2024-08-29T14:56:16.142099+0000","last_became_active":"2024-08-29T14:55:32.403032+0000","last_became_peered":"2024-08-29T14:55:32.403032+0000","last_unstale":"2024-08-29T14:56:16.142099+0000","last_undegraded":"2024-08-29T14:56:16.142099+0000","last_fullsized":"2024-08-29T14:56:16.142099+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:09:57.298172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.662884+0000","last_change":"2024-08-29T14:54:23.263824+0000","last_active":"2024-08-29T14:56:17.662884+0000","last_peered":"2024-08-29T14:56:17.662884+0000","last_clean":"2024-08-29T14:56:17.662884+0000","last_became_active":"2024-08-29T14:54:18.562017+0000","last_became_peered":"2024-08-29T14:54:18.562017+0000","last_unstale":"2024-08-29T14:56:17.662884+0000","last_undegraded":"2024-08-29T14:56:17.662884+0000","last_fullsized":"2024-08-29T14:56:17.662884+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:07:43.010038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000266333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139226+0000","last_change":"2024-08-29T14:55:33.427421+0000","last_active":"2024-08-29T14:56:16.139226+0000","last_peered":"2024-08-29T14:56:16.139226+0000","last_clean":"2024-08-29T14:56:16.139226+0000","last_became_active":"2024-08-29T14:55:33.426982+0000","last_became_peered":"2024-08-29T14:55:33.426982+0000","last_unstale":"2024-08-29T14:56:16.139226+0000","last_undegraded":"2024-08-29T14:56:16.139226+0000","last_fullsized":"2024-08-29T14:56:16.139226+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:40:57.784217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.140070+0000","last_change":"2024-08-29T14:55:32.414359+0000","last_active":"2024-08-29T14:56:16.140070+0000","last_peered":"2024-08-29T14:56:16.140070+0000","last_clean":"2024-08-29T14:56:16.140070+0000","last_became_active":"2024-08-29T14:55:32.413645+0000","last_became_peered":"2024-08-29T14:55:32.413645+0000","last_unstale":"2024-08-29T14:56:16.140070+0000","last_undegraded":"2024-08-29T14:56:16.140070+0000","last_fullsized":"2024-08-29T14:56:16.140070+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:36.790125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.621063+0000","last_change":"2024-08-29T14:54:23.447440+0000","last_active":"2024-08-29T14:56:17.621063+0000","last_peered":"2024-08-29T14:56:17.621063+0000","last_clean":"2024-08-29T14:56:17.621063+0000","last_became_active":"2024-08-29T14:54:18.771432+0000","last_became_peered":"2024-08-29T14:54:18.771432+0000","last_unstale":"2024-08-29T14:56:17.621063+0000","last_undegraded":"2024-08-29T14:56:17.621063+0000","last_fullsized":"2024-08-29T14:56:17.621063+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:18:29.200943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033832300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141924+0000","last_change":"2024-08-29T14:55:33.424845+0000","last_active":"2024-08-29T14:56:16.141924+0000","last_peered":"2024-08-29T14:56:16.141924+0000","last_clean":"2024-08-29T14:56:16.141924+0000","last_became_active":"2024-08-29T14:55:33.424540+0000","last_became_peered":"2024-08-29T14:55:33.424540+0000","last_unstale":"2024-08-29T14:56:16.141924+0000","last_undegraded":"2024-08-29T14:56:16.141924+0000","last_fullsized":"2024-08-29T14:56:16.141924+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:19:57.218079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143909+0000","last_change":"2024-08-29T14:55:32.417346+0000","last_active":"2024-08-29T14:56:16.143909+0000","last_peered":"2024-08-29T14:56:16.143909+0000","last_clean":"2024-08-29T14:56:16.143909+0000","last_became_active":"2024-08-29T14:55:32.415866+0000","last_became_peered":"2024-08-29T14:55:32.415866+0000","last_unstale":"2024-08-29T14:56:16.143909+0000","last_undegraded":"2024-08-29T14:56:16.143909+0000","last_fullsized":"2024-08-29T14:56:16.143909+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:27:19.248959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139785+0000","last_change":"2024-08-29T14:54:23.263862+0000","last_active":"2024-08-29T14:56:16.139785+0000","last_peered":"2024-08-29T14:56:16.139785+0000","last_clean":"2024-08-29T14:56:16.139785+0000","last_became_active":"2024-08-29T14:54:18.771613+0000","last_became_peered":"2024-08-29T14:54:18.771613+0000","last_unstale":"2024-08-29T14:56:16.139785+0000","last_undegraded":"2024-08-29T14:56:16.139785+0000","last_fullsized":"2024-08-29T14:56:16.139785+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:22:56.189804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029192399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142180+0000","last_change":"2024-08-29T14:55:32.456928+0000","last_active":"2024-08-29T14:56:16.142180+0000","last_peered":"2024-08-29T14:56:16.142180+0000","last_clean":"2024-08-29T14:56:16.142180+0000","last_became_active":"2024-08-29T14:55:32.456573+0000","last_became_peered":"2024-08-29T14:55:32.456573+0000","last_unstale":"2024-08-29T14:56:16.142180+0000","last_undegraded":"2024-08-29T14:56:16.142180+0000","last_fullsized":"2024-08-29T14:56:16.142180+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:10:30.462224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142015+0000","last_change":"2024-08-29T14:55:32.418286+0000","last_active":"2024-08-29T14:56:16.142015+0000","last_peered":"2024-08-29T14:56:16.142015+0000","last_clean":"2024-08-29T14:56:16.142015+0000","last_became_active":"2024-08-29T14:55:32.417967+0000","last_became_peered":"2024-08-29T14:55:32.417967+0000","last_unstale":"2024-08-29T14:56:16.142015+0000","last_undegraded":"2024-08-29T14:56:16.142015+0000","last_fullsized":"2024-08-29T14:56:16.142015+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:41:02.043386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141659+0000","last_change":"2024-08-29T14:54:23.263882+0000","last_active":"2024-08-29T14:56:16.141659+0000","last_peered":"2024-08-29T14:56:16.141659+0000","last_clean":"2024-08-29T14:56:16.141659+0000","last_became_active":"2024-08-29T14:54:18.559285+0000","last_became_peered":"2024-08-29T14:54:18.559285+0000","last_unstale":"2024-08-29T14:56:16.141659+0000","last_undegraded":"2024-08-29T14:56:16.141659+0000","last_fullsized":"2024-08-29T14:56:16.141659+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:26:21.042994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027796,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138507+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T14:56:16.138507+0000","last_peered":"2024-08-29T14:56:16.138507+0000","last_clean":"2024-08-29T14:56:16.138507+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T14:56:16.138507+0000","last_undegraded":"2024-08-29T14:56:16.138507+0000","last_fullsized":"2024-08-29T14:56:16.138507+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.147658+0000","last_change":"2024-08-29T14:56:17.147658+0000","last_active":"2024-08-29T14:56:17.147658+0000","last_peered":"2024-08-29T14:56:17.147658+0000","last_clean":"2024-08-29T14:56:17.147658+0000","last_became_active":"2024-08-29T14:56:17.147310+0000","last_became_peered":"2024-08-29T14:56:17.147310+0000","last_unstale":"2024-08-29T14:56:17.147658+0000","last_undegraded":"2024-08-29T14:56:17.147658+0000","last_fullsized":"2024-08-29T14:56:17.147658+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:09:50.769621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139432+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T14:56:16.139432+0000","last_peered":"2024-08-29T14:56:16.139432+0000","last_clean":"2024-08-29T14:56:16.139432+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T14:56:16.139432+0000","last_undegraded":"2024-08-29T14:56:16.139432+0000","last_fullsized":"2024-08-29T14:56:16.139432+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139275+0000","last_change":"2024-08-29T14:55:33.457675+0000","last_active":"2024-08-29T14:56:16.139275+0000","last_peered":"2024-08-29T14:56:16.139275+0000","last_clean":"2024-08-29T14:56:16.139275+0000","last_became_active":"2024-08-29T14:55:33.457375+0000","last_became_peered":"2024-08-29T14:55:33.457375+0000","last_unstale":"2024-08-29T14:56:16.139275+0000","last_undegraded":"2024-08-29T14:56:16.139275+0000","last_fullsized":"2024-08-29T14:56:16.139275+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:07:54.123509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145212+0000","last_change":"2024-08-29T14:55:32.416203+0000","last_active":"2024-08-29T14:56:16.145212+0000","last_peered":"2024-08-29T14:56:16.145212+0000","last_clean":"2024-08-29T14:56:16.145212+0000","last_became_active":"2024-08-29T14:55:32.415929+0000","last_became_peered":"2024-08-29T14:55:32.415929+0000","last_unstale":"2024-08-29T14:56:16.145212+0000","last_undegraded":"2024-08-29T14:56:16.145212+0000","last_fullsized":"2024-08-29T14:56:16.145212+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:50:24.083726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141519+0000","last_change":"2024-08-29T14:55:32.406011+0000","last_active":"2024-08-29T14:56:16.141519+0000","last_peered":"2024-08-29T14:56:16.141519+0000","last_clean":"2024-08-29T14:56:16.141519+0000","last_became_active":"2024-08-29T14:55:32.405740+0000","last_became_peered":"2024-08-29T14:55:32.405740+0000","last_unstale":"2024-08-29T14:56:16.141519+0000","last_undegraded":"2024-08-29T14:56:16.141519+0000","last_fullsized":"2024-08-29T14:56:16.141519+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:50:38.662773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142605+0000","last_change":"2024-08-29T14:55:32.458001+0000","last_active":"2024-08-29T14:56:16.142605+0000","last_peered":"2024-08-29T14:56:16.142605+0000","last_clean":"2024-08-29T14:56:16.142605+0000","last_became_active":"2024-08-29T14:55:32.457535+0000","last_became_peered":"2024-08-29T14:55:32.457535+0000","last_unstale":"2024-08-29T14:56:16.142605+0000","last_undegraded":"2024-08-29T14:56:16.142605+0000","last_fullsized":"2024-08-29T14:56:16.142605+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:50:49.359499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142047+0000","last_change":"2024-08-29T14:55:32.421690+0000","last_active":"2024-08-29T14:56:16.142047+0000","last_peered":"2024-08-29T14:56:16.142047+0000","last_clean":"2024-08-29T14:56:16.142047+0000","last_became_active":"2024-08-29T14:55:32.421228+0000","last_became_peered":"2024-08-29T14:55:32.421228+0000","last_unstale":"2024-08-29T14:56:16.142047+0000","last_undegraded":"2024-08-29T14:56:16.142047+0000","last_fullsized":"2024-08-29T14:56:16.142047+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:04:50.537451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142105+0000","last_change":"2024-08-29T14:55:32.456585+0000","last_active":"2024-08-29T14:56:16.142105+0000","last_peered":"2024-08-29T14:56:16.142105+0000","last_clean":"2024-08-29T14:56:16.142105+0000","last_became_active":"2024-08-29T14:55:32.456253+0000","last_became_peered":"2024-08-29T14:55:32.456253+0000","last_unstale":"2024-08-29T14:56:16.142105+0000","last_undegraded":"2024-08-29T14:56:16.142105+0000","last_fullsized":"2024-08-29T14:56:16.142105+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:53:34.913950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141673+0000","last_change":"2024-08-29T14:55:33.425422+0000","last_active":"2024-08-29T14:56:16.141673+0000","last_peered":"2024-08-29T14:56:16.141673+0000","last_clean":"2024-08-29T14:56:16.141673+0000","last_became_active":"2024-08-29T14:55:33.425148+0000","last_became_peered":"2024-08-29T14:55:33.425148+0000","last_unstale":"2024-08-29T14:56:16.141673+0000","last_undegraded":"2024-08-29T14:56:16.141673+0000","last_fullsized":"2024-08-29T14:56:16.141673+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:23:45.318590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142260+0000","last_change":"2024-08-29T14:55:32.417249+0000","last_active":"2024-08-29T14:56:16.142260+0000","last_peered":"2024-08-29T14:56:16.142260+0000","last_clean":"2024-08-29T14:56:16.142260+0000","last_became_active":"2024-08-29T14:55:32.416933+0000","last_became_peered":"2024-08-29T14:55:32.416933+0000","last_unstale":"2024-08-29T14:56:16.142260+0000","last_undegraded":"2024-08-29T14:56:16.142260+0000","last_fullsized":"2024-08-29T14:56:16.142260+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:45.305340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141828+0000","last_change":"2024-08-29T14:55:33.418912+0000","last_active":"2024-08-29T14:56:16.141828+0000","last_peered":"2024-08-29T14:56:16.141828+0000","last_clean":"2024-08-29T14:56:16.141828+0000","last_became_active":"2024-08-29T14:55:33.418557+0000","last_became_peered":"2024-08-29T14:55:33.418557+0000","last_unstale":"2024-08-29T14:56:16.141828+0000","last_undegraded":"2024-08-29T14:56:16.141828+0000","last_fullsized":"2024-08-29T14:56:16.141828+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:49:51.406477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138995+0000","last_change":"2024-08-29T14:55:32.415954+0000","last_active":"2024-08-29T14:56:16.138995+0000","last_peered":"2024-08-29T14:56:16.138995+0000","last_clean":"2024-08-29T14:56:16.138995+0000","last_became_active":"2024-08-29T14:55:32.415628+0000","last_became_peered":"2024-08-29T14:55:32.415628+0000","last_unstale":"2024-08-29T14:56:16.138995+0000","last_undegraded":"2024-08-29T14:56:16.138995+0000","last_fullsized":"2024-08-29T14:56:16.138995+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:25:05.088448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143373+0000","last_change":"2024-08-29T14:55:32.428650+0000","last_active":"2024-08-29T14:56:16.143373+0000","last_peered":"2024-08-29T14:56:16.143373+0000","last_clean":"2024-08-29T14:56:16.143373+0000","last_became_active":"2024-08-29T14:55:32.428322+0000","last_became_peered":"2024-08-29T14:55:32.428322+0000","last_unstale":"2024-08-29T14:56:16.143373+0000","last_undegraded":"2024-08-29T14:56:16.143373+0000","last_fullsized":"2024-08-29T14:56:16.143373+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:12:06.472131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139006+0000","last_change":"2024-08-29T14:55:32.412725+0000","last_active":"2024-08-29T14:56:16.139006+0000","last_peered":"2024-08-29T14:56:16.139006+0000","last_clean":"2024-08-29T14:56:16.139006+0000","last_became_active":"2024-08-29T14:55:32.412326+0000","last_became_peered":"2024-08-29T14:55:32.412326+0000","last_unstale":"2024-08-29T14:56:16.139006+0000","last_undegraded":"2024-08-29T14:56:16.139006+0000","last_fullsized":"2024-08-29T14:56:16.139006+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:14:46.707561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142838+0000","last_change":"2024-08-29T14:55:32.427358+0000","last_active":"2024-08-29T14:56:16.142838+0000","last_peered":"2024-08-29T14:56:16.142838+0000","last_clean":"2024-08-29T14:56:16.142838+0000","last_became_active":"2024-08-29T14:55:32.427022+0000","last_became_peered":"2024-08-29T14:55:32.427022+0000","last_unstale":"2024-08-29T14:56:16.142838+0000","last_undegraded":"2024-08-29T14:56:16.142838+0000","last_fullsized":"2024-08-29T14:56:16.142838+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:08:16.401961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141856+0000","last_change":"2024-08-29T14:55:32.417961+0000","last_active":"2024-08-29T14:56:16.141856+0000","last_peered":"2024-08-29T14:56:16.141856+0000","last_clean":"2024-08-29T14:56:16.141856+0000","last_became_active":"2024-08-29T14:55:32.417521+0000","last_became_peered":"2024-08-29T14:55:32.417521+0000","last_unstale":"2024-08-29T14:56:16.141856+0000","last_undegraded":"2024-08-29T14:56:16.141856+0000","last_fullsized":"2024-08-29T14:56:16.141856+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:42:40.270730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139296+0000","last_change":"2024-08-29T14:55:33.425215+0000","last_active":"2024-08-29T14:56:16.139296+0000","last_peered":"2024-08-29T14:56:16.139296+0000","last_clean":"2024-08-29T14:56:16.139296+0000","last_became_active":"2024-08-29T14:55:33.424916+0000","last_became_peered":"2024-08-29T14:55:33.424916+0000","last_unstale":"2024-08-29T14:56:16.139296+0000","last_undegraded":"2024-08-29T14:56:16.139296+0000","last_fullsized":"2024-08-29T14:56:16.139296+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:59:19.600324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141343+0000","last_change":"2024-08-29T14:55:32.408756+0000","last_active":"2024-08-29T14:56:16.141343+0000","last_peered":"2024-08-29T14:56:16.141343+0000","last_clean":"2024-08-29T14:56:16.141343+0000","last_became_active":"2024-08-29T14:55:32.408446+0000","last_became_peered":"2024-08-29T14:55:32.408446+0000","last_unstale":"2024-08-29T14:56:16.141343+0000","last_undegraded":"2024-08-29T14:56:16.141343+0000","last_fullsized":"2024-08-29T14:56:16.141343+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:19:14.278842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138842+0000","last_change":"2024-08-29T14:55:33.425604+0000","last_active":"2024-08-29T14:56:16.138842+0000","last_peered":"2024-08-29T14:56:16.138842+0000","last_clean":"2024-08-29T14:56:16.138842+0000","last_became_active":"2024-08-29T14:55:33.424342+0000","last_became_peered":"2024-08-29T14:55:33.424342+0000","last_unstale":"2024-08-29T14:56:16.138842+0000","last_undegraded":"2024-08-29T14:56:16.138842+0000","last_fullsized":"2024-08-29T14:56:16.138842+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:56.303118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142106+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T14:56:16.142106+0000","last_peered":"2024-08-29T14:56:16.142106+0000","last_clean":"2024-08-29T14:56:16.142106+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T14:56:16.142106+0000","last_undegraded":"2024-08-29T14:56:16.142106+0000","last_fullsized":"2024-08-29T14:56:16.142106+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142671+0000","last_change":"2024-08-29T14:55:32.431109+0000","last_active":"2024-08-29T14:56:16.142671+0000","last_peered":"2024-08-29T14:56:16.142671+0000","last_clean":"2024-08-29T14:56:16.142671+0000","last_became_active":"2024-08-29T14:55:32.430729+0000","last_became_peered":"2024-08-29T14:55:32.430729+0000","last_unstale":"2024-08-29T14:56:16.142671+0000","last_undegraded":"2024-08-29T14:56:16.142671+0000","last_fullsized":"2024-08-29T14:56:16.142671+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:34:45.964223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143575+0000","last_change":"2024-08-29T14:55:32.405511+0000","last_active":"2024-08-29T14:56:16.143575+0000","last_peered":"2024-08-29T14:56:16.143575+0000","last_clean":"2024-08-29T14:56:16.143575+0000","last_became_active":"2024-08-29T14:55:32.405141+0000","last_became_peered":"2024-08-29T14:55:32.405141+0000","last_unstale":"2024-08-29T14:56:16.143575+0000","last_undegraded":"2024-08-29T14:56:16.143575+0000","last_fullsized":"2024-08-29T14:56:16.143575+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:27:04.590663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138468+0000","last_change":"2024-08-29T14:55:33.425740+0000","last_active":"2024-08-29T14:56:16.138468+0000","last_peered":"2024-08-29T14:56:16.138468+0000","last_clean":"2024-08-29T14:56:16.138468+0000","last_became_active":"2024-08-29T14:55:33.425454+0000","last_became_peered":"2024-08-29T14:55:33.425454+0000","last_unstale":"2024-08-29T14:56:16.138468+0000","last_undegraded":"2024-08-29T14:56:16.138468+0000","last_fullsized":"2024-08-29T14:56:16.138468+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:14.050131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141978+0000","last_change":"2024-08-29T14:55:32.422520+0000","last_active":"2024-08-29T14:56:16.141978+0000","last_peered":"2024-08-29T14:56:16.141978+0000","last_clean":"2024-08-29T14:56:16.141978+0000","last_became_active":"2024-08-29T14:55:32.421601+0000","last_became_peered":"2024-08-29T14:55:32.421601+0000","last_unstale":"2024-08-29T14:56:16.141978+0000","last_undegraded":"2024-08-29T14:56:16.141978+0000","last_fullsized":"2024-08-29T14:56:16.141978+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:57.355165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28372,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716780,"statfs":{"total":95995035648,"available":95965982720,"internally_reserved":0,"allocated":1380352,"data_stored":925744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.47499999999999998,"15min":0.47199999999999998},"min":{"1min":0.35299999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.59099999999999997}]},{"osd":1,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.51600000000000001,"15min":0.51800000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.41699999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.50900000000000001,"15min":0.50800000000000001},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39400000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.46899999999999997}]},{"osd":3,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.36799999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76100000000000001,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.39100000000000001}]},{"osd":4,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.314,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.71699999999999997,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.45200000000000001,"15min":0.44500000000000001},"min":{"1min":0.314,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64900000000000002}]},{"osd":5,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.498,"15min":0.498},"min":{"1min":0.32500000000000001,"5min":0.309,"15min":0.309},"max":{"1min":0.70999999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.77400000000000002,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.629}]},{"osd":6,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.39100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.50700000000000001,"15min":0.502},"min":{"1min":0.41099999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.56999999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495656,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1368064,"data_stored":925344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54700000000000004,"15min":0.55300000000000005},"min":{"1min":0.374,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.749,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.749},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.76400000000000001,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.65600000000000003}]},{"osd":1,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.64000000000000001,"15min":0.57099999999999995},"min":{"1min":0.40500000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.449,"5min":10.449,"15min":10.449},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72699999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.64100000000000001}]},{"osd":2,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73099999999999998,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.63800000000000001,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.484999999999999,"5min":10.484999999999999,"15min":10.484999999999999},"last":0.68100000000000005}]},{"osd":3,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.31900000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.80500000000000005,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.53800000000000003},"min":{"1min":0.373,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.755,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.69199999999999995}]},{"osd":4,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.42499999999999999,"15min":0.40300000000000002},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68300000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.45300000000000001,"15min":0.433},"min":{"1min":0.315,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.45100000000000001,"15min":0.433},"min":{"1min":0.33800000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.46899999999999997,"15min":0.44400000000000001},"min":{"1min":0.35499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.73799999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.622}]},{"osd":7,"last update":"Thu Aug 29 14:55:45 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.35899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.38200000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.72799999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.67000000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757293,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":765952,"data_stored":333936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.38900000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72999999999999998,"5min":0.747,"15min":0.747},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53400000000000003,"15min":0.52500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70699999999999996,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.71399999999999997}]},{"osd":1,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53900000000000003,"15min":0.54800000000000004},"min":{"1min":0.40300000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.64600000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.51400000000000001,"15min":0.50800000000000001},"min":{"1min":0.374,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.73399999999999999}]},{"osd":2,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.501,"15min":0.505},"min":{"1min":0.42699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.497,"5min":0.53100000000000003,"15min":0.53900000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.67500000000000004,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.64700000000000002}]},{"osd":3,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.51000000000000001,"15min":0.51700000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.63600000000000001,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.53600000000000003},"min":{"1min":0.35799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.77200000000000002}]},{"osd":4,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.28799999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.56399999999999995,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.625},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.371},"min":{"1min":0.311,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.58199999999999996,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.59999999999999998}]},{"osd":6,"last update":"Thu Aug 29 14:55:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.39500000000000002,"15min":0.38700000000000001},"min":{"1min":0.29999999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.57499999999999996,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.39200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.67800000000000005,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.66500000000000004}]},{"osd":7,"last update":"Thu Aug 29 14:55:51 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.44500000000000001,"15min":0.437},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.628,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.49399999999999999,"15min":0.48199999999999998},"min":{"1min":0.41499999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.69099999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27784,"kb_used_data":760,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717368,"statfs":{"total":95995035648,"available":95966584832,"internally_reserved":0,"allocated":778240,"data_stored":335033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.81399999999999995,"5min":0.871,"15min":0.871},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76800000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.47999999999999998}]},{"osd":1,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.38200000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.71499999999999997,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.53600000000000003,"15min":0.54800000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.46300000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.76500000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72099999999999997,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.57099999999999995}]},{"osd":3,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.54500000000000004,"15min":0.53900000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.74199999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56200000000000006,"15min":0.56399999999999995},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78300000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.53500000000000003}]},{"osd":5,"last update":"Thu Aug 29 14:56:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.438,"15min":0.41799999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.70899999999999996,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.46700000000000003,"15min":0.442},"min":{"1min":0.40100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.52000000000000002}]},{"osd":6,"last update":"Thu Aug 29 14:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.42799999999999999,"15min":0.42299999999999999},"min":{"1min":0.29799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.44,"15min":0.438},"min":{"1min":0.316,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.65700000000000003,"5min":0.748,"15min":0.748},"last":0.55700000000000005}]},{"osd":7,"last update":"Thu Aug 29 14:55:50 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.47499999999999998,"15min":0.47099999999999997},"min":{"1min":0.33400000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70199999999999996,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.441}]}]},{"osd":3,"up_from":25,"seq":107374182455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27764,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717388,"statfs":{"total":95995035648,"available":95966605312,"internally_reserved":0,"allocated":761856,"data_stored":333686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38500000000000001,"15min":0.373},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.60699999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41499999999999998,"15min":0.41699999999999998},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61099999999999999,"5min":1.006,"15min":1.006},"last":0.32500000000000001}]},{"osd":1,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.41299999999999998,"15min":0.42499999999999999},"min":{"1min":0.27500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68799999999999994,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.38},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.42299999999999999,"15min":0.42599999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.35599999999999998}]},{"osd":2,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.44,"15min":0.45700000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.443,"15min":0.46100000000000002},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70899999999999996,"5min":0.873,"15min":0.873},"last":0.36899999999999999}]},{"osd":4,"last update":"Thu Aug 29 14:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.55300000000000005,"15min":0.54800000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.54400000000000004}]},{"osd":5,"last update":"Thu Aug 29 14:56:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56599999999999995,"15min":0.55600000000000005},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.60399999999999998}]},{"osd":6,"last update":"Thu Aug 29 14:55:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.56100000000000005,"15min":0.55100000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.57799999999999996,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.49099999999999999}]},{"osd":7,"last update":"Thu Aug 29 14:55:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.45300000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.57399999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31852,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713300,"statfs":{"total":95995035648,"available":95962419200,"internally_reserved":0,"allocated":757760,"data_stored":333270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.34899999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39400000000000002,"15min":0.38200000000000001},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.41499999999999998}]},{"osd":1,"last update":"Thu Aug 29 14:55:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.39700000000000002,"15min":0.375},"min":{"1min":0.28999999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.62,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.374,"15min":0.37},"min":{"1min":0.26600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.41799999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.27300000000000002}]},{"osd":3,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.45900000000000002,"15min":0.46700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.40999999999999998,"15min":0.42499999999999999},"min":{"1min":0.27900000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55400000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.34899999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:55:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.57899999999999996,"15min":0.59099999999999997},"min":{"1min":0.439,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80100000000000005,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.627,"15min":0.63200000000000001},"min":{"1min":0.439,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81000000000000005,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.75}]},{"osd":5,"last update":"Thu Aug 29 14:55:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.59499999999999997,"15min":0.60299999999999998},"min":{"1min":0.45100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.67700000000000005,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41199999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73999999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.79900000000000004}]},{"osd":6,"last update":"Thu Aug 29 14:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.57599999999999996,"15min":0.57399999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.78200000000000003,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.61499999999999999,"15min":0.61899999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.81799999999999995}]},{"osd":7,"last update":"Thu Aug 29 14:55:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.752,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.66100000000000003,"15min":0.67200000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.81200000000000006,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.62}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:36 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39000000000000001,"15min":0.38500000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Aug 29 14:55:56 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.36699999999999999,"15min":0.36499999999999999},"min":{"1min":0.23100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.50800000000000001,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.434},{"interface":"front","average":{"1min":0.375,"5min":0.40799999999999997,"15min":0.39900000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53600000000000003,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.29399999999999998}]},{"osd":3,"last update":"Thu Aug 29 14:56:15 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44800000000000001,"15min":0.45700000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.58799999999999997,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.42799999999999999,"15min":0.439},"min":{"1min":0.28199999999999997,"5min":0.252,"15min":0.252},"max":{"1min":0.59699999999999998,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48999999999999999}]},{"osd":4,"last update":"Thu Aug 29 14:55:42 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.45400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.57199999999999995,"15min":0.56599999999999995},"min":{"1min":0.47799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86299999999999999,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.53700000000000003}]},{"osd":5,"last update":"Thu Aug 29 14:56:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.46200000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85999999999999999,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.621,"15min":0.627},"min":{"1min":0.433,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.83399999999999996,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.59499999999999997}]},{"osd":6,"last update":"Thu Aug 29 14:55:25 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.60699999999999998,"15min":0.60999999999999999},"min":{"1min":0.40400000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.58699999999999997,"15min":0.58599999999999997},"min":{"1min":0.42099999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.72999999999999998}]},{"osd":7,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.623,"15min":0.628},"min":{"1min":0.45400000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79900000000000004,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.60899999999999999,"15min":0.61299999999999999},"min":{"1min":0.50700000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.78200000000000003,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.60699999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28344,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716808,"statfs":{"total":95995035648,"available":95966011392,"internally_reserved":0,"allocated":1355776,"data_stored":924295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Aug 29 14:55:34 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.36099999999999999,"15min":0.33100000000000002},"min":{"1min":0.251,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.52900000000000003,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.372},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.37,"15min":0.33400000000000002},"min":{"1min":0.26200000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.59099999999999997,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.247}]},{"osd":2,"last update":"Thu Aug 29 14:55:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39300000000000002,"15min":0.40300000000000002},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":0.47799999999999998,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.42399999999999999,"15min":0.42599999999999999},"min":{"1min":0.27600000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.63400000000000001,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.28299999999999997}]},{"osd":3,"last update":"Thu Aug 29 14:56:18 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.46600000000000003,"15min":0.49199999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57999999999999996,"5min":0.745,"15min":0.745},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.443,"15min":0.46500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.59099999999999997,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.35499999999999998}]},{"osd":4,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.57999999999999996,"15min":0.57599999999999996},"min":{"1min":0.40100000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.69499999999999995,"15min":0.65700000000000003},"min":{"1min":0.47099999999999997,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.282,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":0.67100000000000004}]},{"osd":5,"last update":"Thu Aug 29 14:56:02 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.85199999999999998,"15min":0.96399999999999997},"min":{"1min":0.43099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.262,"5min":8.5920000000000005,"15min":8.5920000000000005},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.61399999999999999,"15min":0.60999999999999999},"min":{"1min":0.45300000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.72699999999999998}]},{"osd":6,"last update":"Thu Aug 29 14:56:24 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.3140000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.628,"15min":0.626},"min":{"1min":0.45000000000000001,"5min":0.439,"15min":0.439},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.50600000000000001}]},{"osd":7,"last update":"Thu Aug 29 14:55:53 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.84299999999999997,"15min":0.877},"min":{"1min":0.505,"5min":0.438,"15min":0.438},"max":{"1min":1.2390000000000001,"5min":8.5280000000000005,"15min":8.5280000000000005},"last":0.625},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.623,"15min":0.61699999999999999},"min":{"1min":0.45400000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.48999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T14:56:31.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T14:56:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:31 smithi171 ceph-mon[22959]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:31 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:31.901 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:32.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:32 smithi027 ceph-mon[29342]: from='client.14733 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:56:32.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:32 smithi027 ceph-mon[18892]: from='client.14733 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:56:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:32 smithi171 ceph-mon[22959]: from='client.14733 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:56:33.638 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:56:33.638 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T14:56:33.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:33 smithi171 ceph-mon[22959]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:33.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:33 smithi027 ceph-mon[18892]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:33.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:33 smithi027 ceph-mon[29342]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:34.293 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":86,"stamp":"2024-08-29T14:56:33.575886+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":228012,"kb_used_data":7744,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733204,"statfs":{"total":767960285184,"available":767726800896,"internally_reserved":0,"allocated":7929856,"data_stored":4444923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":225431096},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002603"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.621599+0000","last_change":"2024-08-29T14:56:17.621599+0000","last_active":"2024-08-29T14:56:17.621599+0000","last_peered":"2024-08-29T14:56:17.621599+0000","last_clean":"2024-08-29T14:56:17.621599+0000","last_became_active":"2024-08-29T14:56:17.621289+0000","last_became_peered":"2024-08-29T14:56:17.621289+0000","last_unstale":"2024-08-29T14:56:17.621599+0000","last_undegraded":"2024-08-29T14:56:17.621599+0000","last_fullsized":"2024-08-29T14:56:17.621599+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:14:26.987112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145064+0000","last_change":"2024-08-29T14:55:32.414985+0000","last_active":"2024-08-29T14:56:16.145064+0000","last_peered":"2024-08-29T14:56:16.145064+0000","last_clean":"2024-08-29T14:56:16.145064+0000","last_became_active":"2024-08-29T14:55:32.414740+0000","last_became_peered":"2024-08-29T14:55:32.414740+0000","last_unstale":"2024-08-29T14:56:16.145064+0000","last_undegraded":"2024-08-29T14:56:16.145064+0000","last_fullsized":"2024-08-29T14:56:16.145064+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:51:32.533114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141986+0000","last_change":"2024-08-29T14:55:33.424931+0000","last_active":"2024-08-29T14:56:16.141986+0000","last_peered":"2024-08-29T14:56:16.141986+0000","last_clean":"2024-08-29T14:56:16.141986+0000","last_became_active":"2024-08-29T14:55:33.424592+0000","last_became_peered":"2024-08-29T14:55:33.424592+0000","last_unstale":"2024-08-29T14:56:16.141986+0000","last_undegraded":"2024-08-29T14:56:16.141986+0000","last_fullsized":"2024-08-29T14:56:16.141986+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:46:34.886494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141721+0000","last_change":"2024-08-29T14:55:32.419134+0000","last_active":"2024-08-29T14:56:16.141721+0000","last_peered":"2024-08-29T14:56:16.141721+0000","last_clean":"2024-08-29T14:56:16.141721+0000","last_became_active":"2024-08-29T14:55:32.418858+0000","last_became_peered":"2024-08-29T14:55:32.418858+0000","last_unstale":"2024-08-29T14:56:16.141721+0000","last_undegraded":"2024-08-29T14:56:16.141721+0000","last_fullsized":"2024-08-29T14:56:16.141721+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:26:37.051836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141924+0000","last_change":"2024-08-29T14:55:33.418693+0000","last_active":"2024-08-29T14:56:16.141924+0000","last_peered":"2024-08-29T14:56:16.141924+0000","last_clean":"2024-08-29T14:56:16.141924+0000","last_became_active":"2024-08-29T14:55:33.418417+0000","last_became_peered":"2024-08-29T14:55:33.418417+0000","last_unstale":"2024-08-29T14:56:16.141924+0000","last_undegraded":"2024-08-29T14:56:16.141924+0000","last_fullsized":"2024-08-29T14:56:16.141924+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:10:00.617593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.144982+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T14:56:16.144982+0000","last_peered":"2024-08-29T14:56:16.144982+0000","last_clean":"2024-08-29T14:56:16.144982+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T14:56:16.144982+0000","last_undegraded":"2024-08-29T14:56:16.144982+0000","last_fullsized":"2024-08-29T14:56:16.144982+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143814+0000","last_change":"2024-08-29T14:55:32.458161+0000","last_active":"2024-08-29T14:56:16.143814+0000","last_peered":"2024-08-29T14:56:16.143814+0000","last_clean":"2024-08-29T14:56:16.143814+0000","last_became_active":"2024-08-29T14:55:32.457728+0000","last_became_peered":"2024-08-29T14:55:32.457728+0000","last_unstale":"2024-08-29T14:56:16.143814+0000","last_undegraded":"2024-08-29T14:56:16.143814+0000","last_fullsized":"2024-08-29T14:56:16.143814+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:34:27.153148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141483+0000","last_change":"2024-08-29T14:55:32.416161+0000","last_active":"2024-08-29T14:56:16.141483+0000","last_peered":"2024-08-29T14:56:16.141483+0000","last_clean":"2024-08-29T14:56:16.141483+0000","last_became_active":"2024-08-29T14:55:32.415844+0000","last_became_peered":"2024-08-29T14:55:32.415844+0000","last_unstale":"2024-08-29T14:56:16.141483+0000","last_undegraded":"2024-08-29T14:56:16.141483+0000","last_fullsized":"2024-08-29T14:56:16.141483+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:32.405251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145329+0000","last_change":"2024-08-29T14:55:32.417418+0000","last_active":"2024-08-29T14:56:16.145329+0000","last_peered":"2024-08-29T14:56:16.145329+0000","last_clean":"2024-08-29T14:56:16.145329+0000","last_became_active":"2024-08-29T14:55:32.416972+0000","last_became_peered":"2024-08-29T14:55:32.416972+0000","last_unstale":"2024-08-29T14:56:16.145329+0000","last_undegraded":"2024-08-29T14:56:16.145329+0000","last_fullsized":"2024-08-29T14:56:16.145329+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:03:55.923654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141182+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T14:56:16.141182+0000","last_peered":"2024-08-29T14:56:16.141182+0000","last_clean":"2024-08-29T14:56:16.141182+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T14:56:16.141182+0000","last_undegraded":"2024-08-29T14:56:16.141182+0000","last_fullsized":"2024-08-29T14:56:16.141182+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145274+0000","last_change":"2024-08-29T14:55:32.410470+0000","last_active":"2024-08-29T14:56:16.145274+0000","last_peered":"2024-08-29T14:56:16.145274+0000","last_clean":"2024-08-29T14:56:16.145274+0000","last_became_active":"2024-08-29T14:55:32.410055+0000","last_became_peered":"2024-08-29T14:55:32.410055+0000","last_unstale":"2024-08-29T14:56:16.145274+0000","last_undegraded":"2024-08-29T14:56:16.145274+0000","last_fullsized":"2024-08-29T14:56:16.145274+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:38.263764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138691+0000","last_change":"2024-08-29T14:55:33.427456+0000","last_active":"2024-08-29T14:56:16.138691+0000","last_peered":"2024-08-29T14:56:16.138691+0000","last_clean":"2024-08-29T14:56:16.138691+0000","last_became_active":"2024-08-29T14:55:33.426367+0000","last_became_peered":"2024-08-29T14:55:33.426367+0000","last_unstale":"2024-08-29T14:56:16.138691+0000","last_undegraded":"2024-08-29T14:56:16.138691+0000","last_fullsized":"2024-08-29T14:56:16.138691+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:49:34.584262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145349+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T14:56:16.145349+0000","last_peered":"2024-08-29T14:56:16.145349+0000","last_clean":"2024-08-29T14:56:16.145349+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T14:56:16.145349+0000","last_undegraded":"2024-08-29T14:56:16.145349+0000","last_fullsized":"2024-08-29T14:56:16.145349+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141307+0000","last_change":"2024-08-29T14:55:33.430355+0000","last_active":"2024-08-29T14:56:16.141307+0000","last_peered":"2024-08-29T14:56:16.141307+0000","last_clean":"2024-08-29T14:56:16.141307+0000","last_became_active":"2024-08-29T14:55:33.430104+0000","last_became_peered":"2024-08-29T14:55:33.430104+0000","last_unstale":"2024-08-29T14:56:16.141307+0000","last_undegraded":"2024-08-29T14:56:16.141307+0000","last_fullsized":"2024-08-29T14:56:16.141307+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:00:59.549195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141428+0000","last_change":"2024-08-29T14:55:32.403312+0000","last_active":"2024-08-29T14:56:16.141428+0000","last_peered":"2024-08-29T14:56:16.141428+0000","last_clean":"2024-08-29T14:56:16.141428+0000","last_became_active":"2024-08-29T14:55:32.403016+0000","last_became_peered":"2024-08-29T14:55:32.403016+0000","last_unstale":"2024-08-29T14:56:16.141428+0000","last_undegraded":"2024-08-29T14:56:16.141428+0000","last_fullsized":"2024-08-29T14:56:16.141428+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:47.927639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141688+0000","last_change":"2024-08-29T14:55:33.425651+0000","last_active":"2024-08-29T14:56:16.141688+0000","last_peered":"2024-08-29T14:56:16.141688+0000","last_clean":"2024-08-29T14:56:16.141688+0000","last_became_active":"2024-08-29T14:55:33.425353+0000","last_became_peered":"2024-08-29T14:55:33.425353+0000","last_unstale":"2024-08-29T14:56:16.141688+0000","last_undegraded":"2024-08-29T14:56:16.141688+0000","last_fullsized":"2024-08-29T14:56:16.141688+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:26.644824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143110+0000","last_change":"2024-08-29T14:55:32.416739+0000","last_active":"2024-08-29T14:56:16.143110+0000","last_peered":"2024-08-29T14:56:16.143110+0000","last_clean":"2024-08-29T14:56:16.143110+0000","last_became_active":"2024-08-29T14:55:32.415882+0000","last_became_peered":"2024-08-29T14:55:32.415882+0000","last_unstale":"2024-08-29T14:56:16.143110+0000","last_undegraded":"2024-08-29T14:56:16.143110+0000","last_fullsized":"2024-08-29T14:56:16.143110+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:23:04.499484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138971+0000","last_change":"2024-08-29T14:55:33.426892+0000","last_active":"2024-08-29T14:56:16.138971+0000","last_peered":"2024-08-29T14:56:16.138971+0000","last_clean":"2024-08-29T14:56:16.138971+0000","last_became_active":"2024-08-29T14:55:33.426558+0000","last_became_peered":"2024-08-29T14:55:33.426558+0000","last_unstale":"2024-08-29T14:56:16.138971+0000","last_undegraded":"2024-08-29T14:56:16.138971+0000","last_fullsized":"2024-08-29T14:56:16.138971+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:39:21.945011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141661+0000","last_change":"2024-08-29T14:55:32.409109+0000","last_active":"2024-08-29T14:56:16.141661+0000","last_peered":"2024-08-29T14:56:16.141661+0000","last_clean":"2024-08-29T14:56:16.141661+0000","last_became_active":"2024-08-29T14:55:32.408825+0000","last_became_peered":"2024-08-29T14:55:32.408825+0000","last_unstale":"2024-08-29T14:56:16.141661+0000","last_undegraded":"2024-08-29T14:56:16.141661+0000","last_fullsized":"2024-08-29T14:56:16.141661+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:24:26.892271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.210976+0000","last_change":"2024-08-29T14:56:17.210976+0000","last_active":"2024-08-29T14:56:17.210976+0000","last_peered":"2024-08-29T14:56:17.210976+0000","last_clean":"2024-08-29T14:56:17.210976+0000","last_became_active":"2024-08-29T14:56:17.210693+0000","last_became_peered":"2024-08-29T14:56:17.210693+0000","last_unstale":"2024-08-29T14:56:17.210976+0000","last_undegraded":"2024-08-29T14:56:17.210976+0000","last_fullsized":"2024-08-29T14:56:17.210976+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:03.990347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138750+0000","last_change":"2024-08-29T14:55:32.411681+0000","last_active":"2024-08-29T14:56:16.138750+0000","last_peered":"2024-08-29T14:56:16.138750+0000","last_clean":"2024-08-29T14:56:16.138750+0000","last_became_active":"2024-08-29T14:55:32.411408+0000","last_became_peered":"2024-08-29T14:55:32.411408+0000","last_unstale":"2024-08-29T14:56:16.138750+0000","last_undegraded":"2024-08-29T14:56:16.138750+0000","last_fullsized":"2024-08-29T14:56:16.138750+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:10:28.211248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139182+0000","last_change":"2024-08-29T14:55:33.428638+0000","last_active":"2024-08-29T14:56:16.139182+0000","last_peered":"2024-08-29T14:56:16.139182+0000","last_clean":"2024-08-29T14:56:16.139182+0000","last_became_active":"2024-08-29T14:55:33.428340+0000","last_became_peered":"2024-08-29T14:55:33.428340+0000","last_unstale":"2024-08-29T14:56:16.139182+0000","last_undegraded":"2024-08-29T14:56:16.139182+0000","last_fullsized":"2024-08-29T14:56:16.139182+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:14:40.982367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142342+0000","last_change":"2024-08-29T14:55:32.411188+0000","last_active":"2024-08-29T14:56:16.142342+0000","last_peered":"2024-08-29T14:56:16.142342+0000","last_clean":"2024-08-29T14:56:16.142342+0000","last_became_active":"2024-08-29T14:55:32.410863+0000","last_became_peered":"2024-08-29T14:55:32.410863+0000","last_unstale":"2024-08-29T14:56:16.142342+0000","last_undegraded":"2024-08-29T14:56:16.142342+0000","last_fullsized":"2024-08-29T14:56:16.142342+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:46:39.520261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139528+0000","last_change":"2024-08-29T14:55:33.423535+0000","last_active":"2024-08-29T14:56:16.139528+0000","last_peered":"2024-08-29T14:56:16.139528+0000","last_clean":"2024-08-29T14:56:16.139528+0000","last_became_active":"2024-08-29T14:55:33.422923+0000","last_became_peered":"2024-08-29T14:55:33.422923+0000","last_unstale":"2024-08-29T14:56:16.139528+0000","last_undegraded":"2024-08-29T14:56:16.139528+0000","last_fullsized":"2024-08-29T14:56:16.139528+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:56:15.097853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145098+0000","last_change":"2024-08-29T14:53:48.474059+0000","last_active":"2024-08-29T14:56:16.145098+0000","last_peered":"2024-08-29T14:56:16.145098+0000","last_clean":"2024-08-29T14:56:16.145098+0000","last_became_active":"2024-08-29T14:53:48.473622+0000","last_became_peered":"2024-08-29T14:53:48.473622+0000","last_unstale":"2024-08-29T14:56:16.145098+0000","last_undegraded":"2024-08-29T14:56:16.145098+0000","last_fullsized":"2024-08-29T14:56:16.145098+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:51:04.923918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.210161+0000","last_change":"2024-08-29T14:54:23.263036+0000","last_active":"2024-08-29T14:56:17.210161+0000","last_peered":"2024-08-29T14:56:17.210161+0000","last_clean":"2024-08-29T14:56:17.210161+0000","last_became_active":"2024-08-29T14:54:18.558518+0000","last_became_peered":"2024-08-29T14:54:18.558518+0000","last_unstale":"2024-08-29T14:56:17.210161+0000","last_undegraded":"2024-08-29T14:56:17.210161+0000","last_fullsized":"2024-08-29T14:56:17.210161+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:16.144414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001015642,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142952+0000","last_change":"2024-08-29T14:55:32.406853+0000","last_active":"2024-08-29T14:56:16.142952+0000","last_peered":"2024-08-29T14:56:16.142952+0000","last_clean":"2024-08-29T14:56:16.142952+0000","last_became_active":"2024-08-29T14:55:32.405314+0000","last_became_peered":"2024-08-29T14:55:32.405314+0000","last_unstale":"2024-08-29T14:56:16.142952+0000","last_undegraded":"2024-08-29T14:56:16.142952+0000","last_fullsized":"2024-08-29T14:56:16.142952+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.143846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141541+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T14:56:16.141541+0000","last_peered":"2024-08-29T14:56:16.141541+0000","last_clean":"2024-08-29T14:56:16.141541+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T14:56:16.141541+0000","last_undegraded":"2024-08-29T14:56:16.141541+0000","last_fullsized":"2024-08-29T14:56:16.141541+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.620828+0000","last_change":"2024-08-29T14:54:23.447441+0000","last_active":"2024-08-29T14:56:17.620828+0000","last_peered":"2024-08-29T14:56:17.620828+0000","last_clean":"2024-08-29T14:56:17.620828+0000","last_became_active":"2024-08-29T14:54:18.562058+0000","last_became_peered":"2024-08-29T14:54:18.562058+0000","last_unstale":"2024-08-29T14:56:17.620828+0000","last_undegraded":"2024-08-29T14:56:17.620828+0000","last_fullsized":"2024-08-29T14:56:17.620828+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:27:17.193516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032779599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141454+0000","last_change":"2024-08-29T14:55:32.419759+0000","last_active":"2024-08-29T14:56:16.141454+0000","last_peered":"2024-08-29T14:56:16.141454+0000","last_clean":"2024-08-29T14:56:16.141454+0000","last_became_active":"2024-08-29T14:55:32.419469+0000","last_became_peered":"2024-08-29T14:55:32.419469+0000","last_unstale":"2024-08-29T14:56:16.141454+0000","last_undegraded":"2024-08-29T14:56:16.141454+0000","last_fullsized":"2024-08-29T14:56:16.141454+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:33:02.468625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141826+0000","last_change":"2024-08-29T14:55:33.457590+0000","last_active":"2024-08-29T14:56:16.141826+0000","last_peered":"2024-08-29T14:56:16.141826+0000","last_clean":"2024-08-29T14:56:16.141826+0000","last_became_active":"2024-08-29T14:55:33.457258+0000","last_became_peered":"2024-08-29T14:55:33.457258+0000","last_unstale":"2024-08-29T14:56:16.141826+0000","last_undegraded":"2024-08-29T14:56:16.141826+0000","last_fullsized":"2024-08-29T14:56:16.141826+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:36.871100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.144713+0000","last_change":"2024-08-29T14:54:23.256858+0000","last_active":"2024-08-29T14:56:17.144713+0000","last_peered":"2024-08-29T14:56:17.144713+0000","last_clean":"2024-08-29T14:56:17.144713+0000","last_became_active":"2024-08-29T14:54:18.877989+0000","last_became_peered":"2024-08-29T14:54:18.877989+0000","last_unstale":"2024-08-29T14:56:17.144713+0000","last_undegraded":"2024-08-29T14:56:17.144713+0000","last_fullsized":"2024-08-29T14:56:17.144713+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:25:02.401747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033926300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138721+0000","last_change":"2024-08-29T14:55:32.414237+0000","last_active":"2024-08-29T14:56:16.138721+0000","last_peered":"2024-08-29T14:56:16.138721+0000","last_clean":"2024-08-29T14:56:16.138721+0000","last_became_active":"2024-08-29T14:55:32.413566+0000","last_became_peered":"2024-08-29T14:55:32.413566+0000","last_unstale":"2024-08-29T14:56:16.138721+0000","last_undegraded":"2024-08-29T14:56:16.138721+0000","last_fullsized":"2024-08-29T14:56:16.138721+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:42:12.636422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139592+0000","last_change":"2024-08-29T14:55:33.424683+0000","last_active":"2024-08-29T14:56:16.139592+0000","last_peered":"2024-08-29T14:56:16.139592+0000","last_clean":"2024-08-29T14:56:16.139592+0000","last_became_active":"2024-08-29T14:55:33.424387+0000","last_became_peered":"2024-08-29T14:55:33.424387+0000","last_unstale":"2024-08-29T14:56:16.139592+0000","last_undegraded":"2024-08-29T14:56:16.139592+0000","last_fullsized":"2024-08-29T14:56:16.139592+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:32:48.772407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.210267+0000","last_change":"2024-08-29T14:54:23.262948+0000","last_active":"2024-08-29T14:56:17.210267+0000","last_peered":"2024-08-29T14:56:17.210267+0000","last_clean":"2024-08-29T14:56:17.210267+0000","last_became_active":"2024-08-29T14:54:18.561354+0000","last_became_peered":"2024-08-29T14:54:18.561354+0000","last_unstale":"2024-08-29T14:56:17.210267+0000","last_undegraded":"2024-08-29T14:56:17.210267+0000","last_fullsized":"2024-08-29T14:56:17.210267+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:14.431009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093147100000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139515+0000","last_change":"2024-08-29T14:55:32.417502+0000","last_active":"2024-08-29T14:56:16.139515+0000","last_peered":"2024-08-29T14:56:16.139515+0000","last_clean":"2024-08-29T14:56:16.139515+0000","last_became_active":"2024-08-29T14:55:32.417182+0000","last_became_peered":"2024-08-29T14:55:32.417182+0000","last_unstale":"2024-08-29T14:56:16.139515+0000","last_undegraded":"2024-08-29T14:56:16.139515+0000","last_fullsized":"2024-08-29T14:56:16.139515+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:10:10.560754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139776+0000","last_change":"2024-08-29T14:55:33.457850+0000","last_active":"2024-08-29T14:56:16.139776+0000","last_peered":"2024-08-29T14:56:16.139776+0000","last_clean":"2024-08-29T14:56:16.139776+0000","last_became_active":"2024-08-29T14:55:33.457524+0000","last_became_peered":"2024-08-29T14:55:33.457524+0000","last_unstale":"2024-08-29T14:56:16.139776+0000","last_undegraded":"2024-08-29T14:56:16.139776+0000","last_fullsized":"2024-08-29T14:56:16.139776+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:41.003146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142099+0000","last_change":"2024-08-29T14:55:32.403376+0000","last_active":"2024-08-29T14:56:16.142099+0000","last_peered":"2024-08-29T14:56:16.142099+0000","last_clean":"2024-08-29T14:56:16.142099+0000","last_became_active":"2024-08-29T14:55:32.403032+0000","last_became_peered":"2024-08-29T14:55:32.403032+0000","last_unstale":"2024-08-29T14:56:16.142099+0000","last_undegraded":"2024-08-29T14:56:16.142099+0000","last_fullsized":"2024-08-29T14:56:16.142099+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:09:57.298172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.662884+0000","last_change":"2024-08-29T14:54:23.263824+0000","last_active":"2024-08-29T14:56:17.662884+0000","last_peered":"2024-08-29T14:56:17.662884+0000","last_clean":"2024-08-29T14:56:17.662884+0000","last_became_active":"2024-08-29T14:54:18.562017+0000","last_became_peered":"2024-08-29T14:54:18.562017+0000","last_unstale":"2024-08-29T14:56:17.662884+0000","last_undegraded":"2024-08-29T14:56:17.662884+0000","last_fullsized":"2024-08-29T14:56:17.662884+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:07:43.010038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000266333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139226+0000","last_change":"2024-08-29T14:55:33.427421+0000","last_active":"2024-08-29T14:56:16.139226+0000","last_peered":"2024-08-29T14:56:16.139226+0000","last_clean":"2024-08-29T14:56:16.139226+0000","last_became_active":"2024-08-29T14:55:33.426982+0000","last_became_peered":"2024-08-29T14:55:33.426982+0000","last_unstale":"2024-08-29T14:56:16.139226+0000","last_undegraded":"2024-08-29T14:56:16.139226+0000","last_fullsized":"2024-08-29T14:56:16.139226+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:40:57.784217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.140070+0000","last_change":"2024-08-29T14:55:32.414359+0000","last_active":"2024-08-29T14:56:16.140070+0000","last_peered":"2024-08-29T14:56:16.140070+0000","last_clean":"2024-08-29T14:56:16.140070+0000","last_became_active":"2024-08-29T14:55:32.413645+0000","last_became_peered":"2024-08-29T14:55:32.413645+0000","last_unstale":"2024-08-29T14:56:16.140070+0000","last_undegraded":"2024-08-29T14:56:16.140070+0000","last_fullsized":"2024-08-29T14:56:16.140070+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:36.790125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.621063+0000","last_change":"2024-08-29T14:54:23.447440+0000","last_active":"2024-08-29T14:56:17.621063+0000","last_peered":"2024-08-29T14:56:17.621063+0000","last_clean":"2024-08-29T14:56:17.621063+0000","last_became_active":"2024-08-29T14:54:18.771432+0000","last_became_peered":"2024-08-29T14:54:18.771432+0000","last_unstale":"2024-08-29T14:56:17.621063+0000","last_undegraded":"2024-08-29T14:56:17.621063+0000","last_fullsized":"2024-08-29T14:56:17.621063+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:18:29.200943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033832300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141924+0000","last_change":"2024-08-29T14:55:33.424845+0000","last_active":"2024-08-29T14:56:16.141924+0000","last_peered":"2024-08-29T14:56:16.141924+0000","last_clean":"2024-08-29T14:56:16.141924+0000","last_became_active":"2024-08-29T14:55:33.424540+0000","last_became_peered":"2024-08-29T14:55:33.424540+0000","last_unstale":"2024-08-29T14:56:16.141924+0000","last_undegraded":"2024-08-29T14:56:16.141924+0000","last_fullsized":"2024-08-29T14:56:16.141924+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:19:57.218079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143909+0000","last_change":"2024-08-29T14:55:32.417346+0000","last_active":"2024-08-29T14:56:16.143909+0000","last_peered":"2024-08-29T14:56:16.143909+0000","last_clean":"2024-08-29T14:56:16.143909+0000","last_became_active":"2024-08-29T14:55:32.415866+0000","last_became_peered":"2024-08-29T14:55:32.415866+0000","last_unstale":"2024-08-29T14:56:16.143909+0000","last_undegraded":"2024-08-29T14:56:16.143909+0000","last_fullsized":"2024-08-29T14:56:16.143909+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:27:19.248959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.663003+0000","last_change":"2024-08-29T14:54:23.263862+0000","last_active":"2024-08-29T14:56:17.663003+0000","last_peered":"2024-08-29T14:56:17.663003+0000","last_clean":"2024-08-29T14:56:17.663003+0000","last_became_active":"2024-08-29T14:54:18.771613+0000","last_became_peered":"2024-08-29T14:54:18.771613+0000","last_unstale":"2024-08-29T14:56:17.663003+0000","last_undegraded":"2024-08-29T14:56:17.663003+0000","last_fullsized":"2024-08-29T14:56:17.663003+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:22:56.189804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029192399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142180+0000","last_change":"2024-08-29T14:55:32.456928+0000","last_active":"2024-08-29T14:56:16.142180+0000","last_peered":"2024-08-29T14:56:16.142180+0000","last_clean":"2024-08-29T14:56:16.142180+0000","last_became_active":"2024-08-29T14:55:32.456573+0000","last_became_peered":"2024-08-29T14:55:32.456573+0000","last_unstale":"2024-08-29T14:56:16.142180+0000","last_undegraded":"2024-08-29T14:56:16.142180+0000","last_fullsized":"2024-08-29T14:56:16.142180+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:10:30.462224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142015+0000","last_change":"2024-08-29T14:55:32.418286+0000","last_active":"2024-08-29T14:56:16.142015+0000","last_peered":"2024-08-29T14:56:16.142015+0000","last_clean":"2024-08-29T14:56:16.142015+0000","last_became_active":"2024-08-29T14:55:32.417967+0000","last_became_peered":"2024-08-29T14:55:32.417967+0000","last_unstale":"2024-08-29T14:56:16.142015+0000","last_undegraded":"2024-08-29T14:56:16.142015+0000","last_fullsized":"2024-08-29T14:56:16.142015+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:41:02.043386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.141910+0000","last_change":"2024-08-29T14:54:23.263882+0000","last_active":"2024-08-29T14:56:17.141910+0000","last_peered":"2024-08-29T14:56:17.141910+0000","last_clean":"2024-08-29T14:56:17.141910+0000","last_became_active":"2024-08-29T14:54:18.559285+0000","last_became_peered":"2024-08-29T14:54:18.559285+0000","last_unstale":"2024-08-29T14:56:17.141910+0000","last_undegraded":"2024-08-29T14:56:17.141910+0000","last_fullsized":"2024-08-29T14:56:17.141910+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:26:21.042994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027796,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138507+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T14:56:16.138507+0000","last_peered":"2024-08-29T14:56:16.138507+0000","last_clean":"2024-08-29T14:56:16.138507+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T14:56:16.138507+0000","last_undegraded":"2024-08-29T14:56:16.138507+0000","last_fullsized":"2024-08-29T14:56:16.138507+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-29T14:56:17.147658+0000","last_change":"2024-08-29T14:56:17.147658+0000","last_active":"2024-08-29T14:56:17.147658+0000","last_peered":"2024-08-29T14:56:17.147658+0000","last_clean":"2024-08-29T14:56:17.147658+0000","last_became_active":"2024-08-29T14:56:17.147310+0000","last_became_peered":"2024-08-29T14:56:17.147310+0000","last_unstale":"2024-08-29T14:56:17.147658+0000","last_undegraded":"2024-08-29T14:56:17.147658+0000","last_fullsized":"2024-08-29T14:56:17.147658+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:09:50.769621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139432+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T14:56:16.139432+0000","last_peered":"2024-08-29T14:56:16.139432+0000","last_clean":"2024-08-29T14:56:16.139432+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T14:56:16.139432+0000","last_undegraded":"2024-08-29T14:56:16.139432+0000","last_fullsized":"2024-08-29T14:56:16.139432+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139275+0000","last_change":"2024-08-29T14:55:33.457675+0000","last_active":"2024-08-29T14:56:16.139275+0000","last_peered":"2024-08-29T14:56:16.139275+0000","last_clean":"2024-08-29T14:56:16.139275+0000","last_became_active":"2024-08-29T14:55:33.457375+0000","last_became_peered":"2024-08-29T14:55:33.457375+0000","last_unstale":"2024-08-29T14:56:16.139275+0000","last_undegraded":"2024-08-29T14:56:16.139275+0000","last_fullsized":"2024-08-29T14:56:16.139275+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:07:54.123509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.145212+0000","last_change":"2024-08-29T14:55:32.416203+0000","last_active":"2024-08-29T14:56:16.145212+0000","last_peered":"2024-08-29T14:56:16.145212+0000","last_clean":"2024-08-29T14:56:16.145212+0000","last_became_active":"2024-08-29T14:55:32.415929+0000","last_became_peered":"2024-08-29T14:55:32.415929+0000","last_unstale":"2024-08-29T14:56:16.145212+0000","last_undegraded":"2024-08-29T14:56:16.145212+0000","last_fullsized":"2024-08-29T14:56:16.145212+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:50:24.083726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141519+0000","last_change":"2024-08-29T14:55:32.406011+0000","last_active":"2024-08-29T14:56:16.141519+0000","last_peered":"2024-08-29T14:56:16.141519+0000","last_clean":"2024-08-29T14:56:16.141519+0000","last_became_active":"2024-08-29T14:55:32.405740+0000","last_became_peered":"2024-08-29T14:55:32.405740+0000","last_unstale":"2024-08-29T14:56:16.141519+0000","last_undegraded":"2024-08-29T14:56:16.141519+0000","last_fullsized":"2024-08-29T14:56:16.141519+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:50:38.662773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142605+0000","last_change":"2024-08-29T14:55:32.458001+0000","last_active":"2024-08-29T14:56:16.142605+0000","last_peered":"2024-08-29T14:56:16.142605+0000","last_clean":"2024-08-29T14:56:16.142605+0000","last_became_active":"2024-08-29T14:55:32.457535+0000","last_became_peered":"2024-08-29T14:55:32.457535+0000","last_unstale":"2024-08-29T14:56:16.142605+0000","last_undegraded":"2024-08-29T14:56:16.142605+0000","last_fullsized":"2024-08-29T14:56:16.142605+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:50:49.359499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142047+0000","last_change":"2024-08-29T14:55:32.421690+0000","last_active":"2024-08-29T14:56:16.142047+0000","last_peered":"2024-08-29T14:56:16.142047+0000","last_clean":"2024-08-29T14:56:16.142047+0000","last_became_active":"2024-08-29T14:55:32.421228+0000","last_became_peered":"2024-08-29T14:55:32.421228+0000","last_unstale":"2024-08-29T14:56:16.142047+0000","last_undegraded":"2024-08-29T14:56:16.142047+0000","last_fullsized":"2024-08-29T14:56:16.142047+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:04:50.537451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142105+0000","last_change":"2024-08-29T14:55:32.456585+0000","last_active":"2024-08-29T14:56:16.142105+0000","last_peered":"2024-08-29T14:56:16.142105+0000","last_clean":"2024-08-29T14:56:16.142105+0000","last_became_active":"2024-08-29T14:55:32.456253+0000","last_became_peered":"2024-08-29T14:55:32.456253+0000","last_unstale":"2024-08-29T14:56:16.142105+0000","last_undegraded":"2024-08-29T14:56:16.142105+0000","last_fullsized":"2024-08-29T14:56:16.142105+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:53:34.913950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141673+0000","last_change":"2024-08-29T14:55:33.425422+0000","last_active":"2024-08-29T14:56:16.141673+0000","last_peered":"2024-08-29T14:56:16.141673+0000","last_clean":"2024-08-29T14:56:16.141673+0000","last_became_active":"2024-08-29T14:55:33.425148+0000","last_became_peered":"2024-08-29T14:55:33.425148+0000","last_unstale":"2024-08-29T14:56:16.141673+0000","last_undegraded":"2024-08-29T14:56:16.141673+0000","last_fullsized":"2024-08-29T14:56:16.141673+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:23:45.318590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142260+0000","last_change":"2024-08-29T14:55:32.417249+0000","last_active":"2024-08-29T14:56:16.142260+0000","last_peered":"2024-08-29T14:56:16.142260+0000","last_clean":"2024-08-29T14:56:16.142260+0000","last_became_active":"2024-08-29T14:55:32.416933+0000","last_became_peered":"2024-08-29T14:55:32.416933+0000","last_unstale":"2024-08-29T14:56:16.142260+0000","last_undegraded":"2024-08-29T14:56:16.142260+0000","last_fullsized":"2024-08-29T14:56:16.142260+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:45.305340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141828+0000","last_change":"2024-08-29T14:55:33.418912+0000","last_active":"2024-08-29T14:56:16.141828+0000","last_peered":"2024-08-29T14:56:16.141828+0000","last_clean":"2024-08-29T14:56:16.141828+0000","last_became_active":"2024-08-29T14:55:33.418557+0000","last_became_peered":"2024-08-29T14:55:33.418557+0000","last_unstale":"2024-08-29T14:56:16.141828+0000","last_undegraded":"2024-08-29T14:56:16.141828+0000","last_fullsized":"2024-08-29T14:56:16.141828+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:49:51.406477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138995+0000","last_change":"2024-08-29T14:55:32.415954+0000","last_active":"2024-08-29T14:56:16.138995+0000","last_peered":"2024-08-29T14:56:16.138995+0000","last_clean":"2024-08-29T14:56:16.138995+0000","last_became_active":"2024-08-29T14:55:32.415628+0000","last_became_peered":"2024-08-29T14:55:32.415628+0000","last_unstale":"2024-08-29T14:56:16.138995+0000","last_undegraded":"2024-08-29T14:56:16.138995+0000","last_fullsized":"2024-08-29T14:56:16.138995+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:25:05.088448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143373+0000","last_change":"2024-08-29T14:55:32.428650+0000","last_active":"2024-08-29T14:56:16.143373+0000","last_peered":"2024-08-29T14:56:16.143373+0000","last_clean":"2024-08-29T14:56:16.143373+0000","last_became_active":"2024-08-29T14:55:32.428322+0000","last_became_peered":"2024-08-29T14:55:32.428322+0000","last_unstale":"2024-08-29T14:56:16.143373+0000","last_undegraded":"2024-08-29T14:56:16.143373+0000","last_fullsized":"2024-08-29T14:56:16.143373+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:12:06.472131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139006+0000","last_change":"2024-08-29T14:55:32.412725+0000","last_active":"2024-08-29T14:56:16.139006+0000","last_peered":"2024-08-29T14:56:16.139006+0000","last_clean":"2024-08-29T14:56:16.139006+0000","last_became_active":"2024-08-29T14:55:32.412326+0000","last_became_peered":"2024-08-29T14:55:32.412326+0000","last_unstale":"2024-08-29T14:56:16.139006+0000","last_undegraded":"2024-08-29T14:56:16.139006+0000","last_fullsized":"2024-08-29T14:56:16.139006+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:14:46.707561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142838+0000","last_change":"2024-08-29T14:55:32.427358+0000","last_active":"2024-08-29T14:56:16.142838+0000","last_peered":"2024-08-29T14:56:16.142838+0000","last_clean":"2024-08-29T14:56:16.142838+0000","last_became_active":"2024-08-29T14:55:32.427022+0000","last_became_peered":"2024-08-29T14:55:32.427022+0000","last_unstale":"2024-08-29T14:56:16.142838+0000","last_undegraded":"2024-08-29T14:56:16.142838+0000","last_fullsized":"2024-08-29T14:56:16.142838+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:08:16.401961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141856+0000","last_change":"2024-08-29T14:55:32.417961+0000","last_active":"2024-08-29T14:56:16.141856+0000","last_peered":"2024-08-29T14:56:16.141856+0000","last_clean":"2024-08-29T14:56:16.141856+0000","last_became_active":"2024-08-29T14:55:32.417521+0000","last_became_peered":"2024-08-29T14:55:32.417521+0000","last_unstale":"2024-08-29T14:56:16.141856+0000","last_undegraded":"2024-08-29T14:56:16.141856+0000","last_fullsized":"2024-08-29T14:56:16.141856+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:42:40.270730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.139296+0000","last_change":"2024-08-29T14:55:33.425215+0000","last_active":"2024-08-29T14:56:16.139296+0000","last_peered":"2024-08-29T14:56:16.139296+0000","last_clean":"2024-08-29T14:56:16.139296+0000","last_became_active":"2024-08-29T14:55:33.424916+0000","last_became_peered":"2024-08-29T14:55:33.424916+0000","last_unstale":"2024-08-29T14:56:16.139296+0000","last_undegraded":"2024-08-29T14:56:16.139296+0000","last_fullsized":"2024-08-29T14:56:16.139296+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:59:19.600324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141343+0000","last_change":"2024-08-29T14:55:32.408756+0000","last_active":"2024-08-29T14:56:16.141343+0000","last_peered":"2024-08-29T14:56:16.141343+0000","last_clean":"2024-08-29T14:56:16.141343+0000","last_became_active":"2024-08-29T14:55:32.408446+0000","last_became_peered":"2024-08-29T14:55:32.408446+0000","last_unstale":"2024-08-29T14:56:16.141343+0000","last_undegraded":"2024-08-29T14:56:16.141343+0000","last_fullsized":"2024-08-29T14:56:16.141343+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:19:14.278842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138842+0000","last_change":"2024-08-29T14:55:33.425604+0000","last_active":"2024-08-29T14:56:16.138842+0000","last_peered":"2024-08-29T14:56:16.138842+0000","last_clean":"2024-08-29T14:56:16.138842+0000","last_became_active":"2024-08-29T14:55:33.424342+0000","last_became_peered":"2024-08-29T14:55:33.424342+0000","last_unstale":"2024-08-29T14:56:16.138842+0000","last_undegraded":"2024-08-29T14:56:16.138842+0000","last_fullsized":"2024-08-29T14:56:16.138842+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:56.303118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142106+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T14:56:16.142106+0000","last_peered":"2024-08-29T14:56:16.142106+0000","last_clean":"2024-08-29T14:56:16.142106+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T14:56:16.142106+0000","last_undegraded":"2024-08-29T14:56:16.142106+0000","last_fullsized":"2024-08-29T14:56:16.142106+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.142671+0000","last_change":"2024-08-29T14:55:32.431109+0000","last_active":"2024-08-29T14:56:16.142671+0000","last_peered":"2024-08-29T14:56:16.142671+0000","last_clean":"2024-08-29T14:56:16.142671+0000","last_became_active":"2024-08-29T14:55:32.430729+0000","last_became_peered":"2024-08-29T14:55:32.430729+0000","last_unstale":"2024-08-29T14:56:16.142671+0000","last_undegraded":"2024-08-29T14:56:16.142671+0000","last_fullsized":"2024-08-29T14:56:16.142671+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:34:45.964223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.143575+0000","last_change":"2024-08-29T14:55:32.405511+0000","last_active":"2024-08-29T14:56:16.143575+0000","last_peered":"2024-08-29T14:56:16.143575+0000","last_clean":"2024-08-29T14:56:16.143575+0000","last_became_active":"2024-08-29T14:55:32.405141+0000","last_became_peered":"2024-08-29T14:55:32.405141+0000","last_unstale":"2024-08-29T14:56:16.143575+0000","last_undegraded":"2024-08-29T14:56:16.143575+0000","last_fullsized":"2024-08-29T14:56:16.143575+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:27:04.590663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.138468+0000","last_change":"2024-08-29T14:55:33.425740+0000","last_active":"2024-08-29T14:56:16.138468+0000","last_peered":"2024-08-29T14:56:16.138468+0000","last_clean":"2024-08-29T14:56:16.138468+0000","last_became_active":"2024-08-29T14:55:33.425454+0000","last_became_peered":"2024-08-29T14:55:33.425454+0000","last_unstale":"2024-08-29T14:56:16.138468+0000","last_undegraded":"2024-08-29T14:56:16.138468+0000","last_fullsized":"2024-08-29T14:56:16.138468+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:14.050131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-29T14:56:16.141978+0000","last_change":"2024-08-29T14:55:32.422520+0000","last_active":"2024-08-29T14:56:16.141978+0000","last_peered":"2024-08-29T14:56:16.141978+0000","last_clean":"2024-08-29T14:56:16.141978+0000","last_became_active":"2024-08-29T14:55:32.421601+0000","last_became_peered":"2024-08-29T14:55:32.421601+0000","last_unstale":"2024-08-29T14:56:16.141978+0000","last_undegraded":"2024-08-29T14:56:16.141978+0000","last_fullsized":"2024-08-29T14:56:16.141978+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:57.355165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234021,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28372,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716780,"statfs":{"total":95995035648,"available":95965982720,"internally_reserved":0,"allocated":1380352,"data_stored":925744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.47499999999999998,"15min":0.47199999999999998},"min":{"1min":0.35299999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.66300000000000003}]},{"osd":1,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.51600000000000001,"15min":0.51800000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.501,"15min":0.497},"min":{"1min":0.41699999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.70599999999999996}]},{"osd":2,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.50900000000000001,"15min":0.50800000000000001},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39400000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.72199999999999998}]},{"osd":3,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.36799999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76100000000000001,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.77000000000000002}]},{"osd":4,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.314,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.71699999999999997,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.45200000000000001,"15min":0.44500000000000001},"min":{"1min":0.314,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.60899999999999999}]},{"osd":5,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.498,"15min":0.498},"min":{"1min":0.32500000000000001,"5min":0.309,"15min":0.309},"max":{"1min":0.70999999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.502,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.77400000000000002,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.746}]},{"osd":6,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.39100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.50700000000000001,"15min":0.502},"min":{"1min":0.41099999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.68400000000000005}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1368064,"data_stored":925344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54700000000000004,"15min":0.55300000000000005},"min":{"1min":0.374,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.749,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.628},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.38600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.76400000000000001,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.42899999999999999}]},{"osd":1,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.64000000000000001,"15min":0.57099999999999995},"min":{"1min":0.40500000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.449,"5min":10.449,"15min":10.449},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72699999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52500000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51300000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73099999999999998,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.63800000000000001,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.484999999999999,"5min":10.484999999999999,"15min":10.484999999999999},"last":0.38500000000000001}]},{"osd":3,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.31900000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.80500000000000005,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54000000000000004,"15min":0.53800000000000003},"min":{"1min":0.373,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.755,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.44500000000000001}]},{"osd":4,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.42499999999999999,"15min":0.40300000000000002},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68300000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.45300000000000001,"15min":0.433},"min":{"1min":0.315,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.66500000000000004}]},{"osd":5,"last update":"Thu Aug 29 14:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.45100000000000001,"15min":0.433},"min":{"1min":0.33800000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.505},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.46899999999999997,"15min":0.44400000000000001},"min":{"1min":0.35499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.73799999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.70299999999999996}]},{"osd":7,"last update":"Thu Aug 29 14:55:45 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.35899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.38200000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.72799999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.745}]}]},{"osd":5,"up_from":38,"seq":163208757294,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":765952,"data_stored":333936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.38900000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72999999999999998,"5min":0.747,"15min":0.747},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53400000000000003,"15min":0.52500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70699999999999996,"5min":1.4390000000000001,"15min":1.4390000000000001},"last":0.45600000000000002}]},{"osd":1,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53900000000000003,"15min":0.54800000000000004},"min":{"1min":0.40300000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.64600000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.51400000000000001,"15min":0.50800000000000001},"min":{"1min":0.374,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.499}]},{"osd":2,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.501,"15min":0.505},"min":{"1min":0.42699999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.53100000000000003,"15min":0.53900000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.67500000000000004,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.63100000000000001}]},{"osd":3,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.51000000000000001,"15min":0.51700000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.63600000000000001,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.53600000000000003},"min":{"1min":0.35799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.67700000000000005}]},{"osd":4,"last update":"Thu Aug 29 14:56:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.28799999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.56399999999999995,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.38400000000000001,"15min":0.371},"min":{"1min":0.311,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.58199999999999996,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.60899999999999999}]},{"osd":6,"last update":"Thu Aug 29 14:56:30 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.41399999999999998,"15min":0.39300000000000002},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.47299999999999998,"15min":0.45100000000000001},"min":{"1min":0.35999999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.69599999999999995}]},{"osd":7,"last update":"Thu Aug 29 14:55:51 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.44500000000000001,"15min":0.437},"min":{"1min":0.373,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.628,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.49399999999999999,"15min":0.48199999999999998},"min":{"1min":0.41499999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.71999999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27784,"kb_used_data":760,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717368,"statfs":{"total":95995035648,"available":95966584832,"internally_reserved":0,"allocated":778240,"data_stored":335033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.81399999999999995,"5min":0.871,"15min":0.871},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76800000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.621}]},{"osd":1,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.38200000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.71499999999999997,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.53600000000000003,"15min":0.54800000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.64300000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54100000000000004,"15min":0.55300000000000005},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.76500000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72099999999999997,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.72899999999999998}]},{"osd":3,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.54500000000000004,"15min":0.53900000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.74199999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56200000000000006,"15min":0.56399999999999995},"min":{"1min":0.36299999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78300000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.66200000000000003}]},{"osd":5,"last update":"Thu Aug 29 14:56:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.438,"15min":0.41799999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.70899999999999996,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.46700000000000003,"15min":0.442},"min":{"1min":0.40100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.47899999999999998}]},{"osd":6,"last update":"Thu Aug 29 14:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.438,"15min":0.42599999999999999},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.68000000000000005,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.501,"5min":0.45300000000000001,"15min":0.443},"min":{"1min":0.34100000000000003,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.68799999999999994,"5min":0.748,"15min":0.748},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Aug 29 14:55:50 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.47499999999999998,"15min":0.47099999999999997},"min":{"1min":0.33400000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.316,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70199999999999996,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.74199999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182456,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27764,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717388,"statfs":{"total":95995035648,"available":95966605312,"internally_reserved":0,"allocated":761856,"data_stored":333686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.38500000000000001,"15min":0.373},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.60699999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41499999999999998,"15min":0.41699999999999998},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61099999999999999,"5min":1.006,"15min":1.006},"last":0.27400000000000002}]},{"osd":1,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.41299999999999998,"15min":0.42499999999999999},"min":{"1min":0.27500000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68799999999999994,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.42299999999999999,"15min":0.42599999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.38200000000000001}]},{"osd":2,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.44,"15min":0.45700000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.443,"15min":0.46100000000000002},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70899999999999996,"5min":0.873,"15min":0.873},"last":0.314}]},{"osd":4,"last update":"Thu Aug 29 14:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.55300000000000005,"15min":0.54800000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.58299999999999996}]},{"osd":5,"last update":"Thu Aug 29 14:56:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56599999999999995,"15min":0.55600000000000005},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.52300000000000002}]},{"osd":6,"last update":"Thu Aug 29 14:56:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57099999999999995,"15min":0.55500000000000005},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.89200000000000002,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.58599999999999997,"15min":0.57599999999999996},"min":{"1min":0.42599999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82899999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.63}]},{"osd":7,"last update":"Thu Aug 29 14:55:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.45300000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57599999999999996,"15min":0.57699999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.502}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31852,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713300,"statfs":{"total":95995035648,"available":95962419200,"internally_reserved":0,"allocated":757760,"data_stored":333270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.34899999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39400000000000002,"15min":0.38200000000000001},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.435}]},{"osd":1,"last update":"Thu Aug 29 14:55:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.39700000000000002,"15min":0.375},"min":{"1min":0.28999999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.62,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.374,"15min":0.37},"min":{"1min":0.26600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.41799999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.373}]},{"osd":3,"last update":"Thu Aug 29 14:56:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.45900000000000002,"15min":0.46700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.40999999999999998,"15min":0.42499999999999999},"min":{"1min":0.27900000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55400000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.42599999999999999}]},{"osd":4,"last update":"Thu Aug 29 14:55:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.57899999999999996,"15min":0.59099999999999997},"min":{"1min":0.439,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80100000000000005,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.627,"15min":0.63200000000000001},"min":{"1min":0.439,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81000000000000005,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.71199999999999997}]},{"osd":5,"last update":"Thu Aug 29 14:55:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.59499999999999997,"15min":0.60299999999999998},"min":{"1min":0.45100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.67700000000000005,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41199999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73999999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.60399999999999998}]},{"osd":6,"last update":"Thu Aug 29 14:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.79800000000000004,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60999999999999999,"15min":0.61799999999999999},"min":{"1min":0.44500000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.624}]},{"osd":7,"last update":"Thu Aug 29 14:55:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.59299999999999997,"15min":0.59799999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.752,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.66100000000000003,"15min":0.67200000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.81200000000000006,"5min":1.0369999999999999,"15min":1.0369999999999999},"last":0.67800000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Aug 29 14:55:36 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.308},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39000000000000001,"15min":0.38500000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.32700000000000001}]},{"osd":2,"last update":"Thu Aug 29 14:55:56 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.36699999999999999,"15min":0.36499999999999999},"min":{"1min":0.23100000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.50800000000000001,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.375,"5min":0.40799999999999997,"15min":0.39900000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53600000000000003,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.38200000000000001}]},{"osd":3,"last update":"Thu Aug 29 14:56:15 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44800000000000001,"15min":0.45700000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.58799999999999997,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.42799999999999999,"15min":0.439},"min":{"1min":0.28199999999999997,"5min":0.252,"15min":0.252},"max":{"1min":0.59699999999999998,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.39800000000000002}]},{"osd":4,"last update":"Thu Aug 29 14:55:42 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.45400000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.57199999999999995,"15min":0.56599999999999995},"min":{"1min":0.47799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86299999999999999,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.77000000000000002}]},{"osd":5,"last update":"Thu Aug 29 14:56:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.46200000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85999999999999999,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.755},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.621,"15min":0.627},"min":{"1min":0.433,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.83399999999999996,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.57199999999999995}]},{"osd":6,"last update":"Thu Aug 29 14:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60799999999999998,"15min":0.60999999999999999},"min":{"1min":0.44700000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82599999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.59299999999999997,"15min":0.58799999999999997},"min":{"1min":0.46000000000000002,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82999999999999996,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.60499999999999998}]},{"osd":7,"last update":"Thu Aug 29 14:55:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.623,"15min":0.628},"min":{"1min":0.45400000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79900000000000004,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.60899999999999999,"15min":0.61299999999999999},"min":{"1min":0.50700000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.78200000000000003,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.622}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28344,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716808,"statfs":{"total":95995035648,"available":95966011392,"internally_reserved":0,"allocated":1355776,"data_stored":924295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Aug 29 14:55:34 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.36099999999999999,"15min":0.33100000000000002},"min":{"1min":0.251,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.52900000000000003,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.37,"15min":0.33400000000000002},"min":{"1min":0.26200000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.59099999999999997,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.33600000000000002}]},{"osd":2,"last update":"Thu Aug 29 14:55:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39300000000000002,"15min":0.40300000000000002},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":0.47799999999999998,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.42399999999999999,"15min":0.42599999999999999},"min":{"1min":0.27600000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.63400000000000001,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.313}]},{"osd":3,"last update":"Thu Aug 29 14:56:18 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.46600000000000003,"15min":0.49199999999999999},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57999999999999996,"5min":0.745,"15min":0.745},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.443,"15min":0.46500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.59099999999999997,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.378}]},{"osd":4,"last update":"Thu Aug 29 14:55:39 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.57999999999999996,"15min":0.57599999999999996},"min":{"1min":0.40100000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.622},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.69499999999999995,"15min":0.65700000000000003},"min":{"1min":0.47099999999999997,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.282,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":0.59299999999999997}]},{"osd":5,"last update":"Thu Aug 29 14:56:02 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.85199999999999998,"15min":0.96399999999999997},"min":{"1min":0.43099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.262,"5min":8.5920000000000005,"15min":8.5920000000000005},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.61399999999999999,"15min":0.60999999999999999},"min":{"1min":0.45300000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.51700000000000002}]},{"osd":6,"last update":"Thu Aug 29 14:56:24 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.3140000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.628,"15min":0.626},"min":{"1min":0.45000000000000001,"5min":0.439,"15min":0.439},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.51100000000000001}]},{"osd":7,"last update":"Thu Aug 29 14:55:53 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.84299999999999997,"15min":0.877},"min":{"1min":0.505,"5min":0.438,"15min":0.438},"max":{"1min":1.2390000000000001,"5min":8.5280000000000005,"15min":8.5280000000000005},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.623,"15min":0.61699999999999999},"min":{"1min":0.45400000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.56499999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T14:56:34.297 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-29T14:56:34.297 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-29T14:56:34.297 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-29T14:56:34.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph health --format=json 2024-08-29T14:56:34.501 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:35.714 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:35 smithi027 ceph-mon[29342]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:35.715 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:35 smithi027 ceph-mon[29342]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:56:35.715 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:35 smithi027 ceph-mon[18892]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:35.715 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:35 smithi027 ceph-mon[18892]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:56:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:35 smithi171 ceph-mon[22959]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:35 smithi171 ceph-mon[22959]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T14:56:36.411 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T14:56:36.411 INFO:teuthology.orchestra.run.smithi027.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-29T14:56:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:36 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/955640040' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-29T14:56:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:36 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/955640040' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-29T14:56:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:36 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/955640040' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-29T14:56:37.019 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-29T14:56:37.020 INFO:teuthology.run_tasks:Running task print... 2024-08-29T14:56:37.040 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-08-29T14:56:37.040 INFO:teuthology.run_tasks:Running task parallel... 2024-08-29T14:56:37.094 INFO:teuthology.task.parallel:starting parallel... 2024-08-29T14:56:37.094 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-29T14:56:37.095 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-29T14:56:37.101 INFO:tasks.workunit:Pulling workunits from ref reef 2024-08-29T14:56:37.101 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-29T14:56:37.101 DEBUG:teuthology.orchestra.run.smithi027:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-29T14:56:37.102 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-29T14:56:37.102 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-08-29T14:56:37.102 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-08-29T14:56:37.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-08-29T14:56:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T14:56:37.120 INFO:teuthology.orchestra.run.smithi027.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-29T14:56:37.120 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-29T14:56:37.176 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-29T14:56:37.176 DEBUG:teuthology.orchestra.run.smithi027:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-29T14:56:37.232 INFO:tasks.workunit:timeout=3h 2024-08-29T14:56:37.232 INFO:tasks.workunit:cleanup=True 2024-08-29T14:56:37.233 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-29T14:56:37.292 INFO:tasks.workunit.client.0.smithi027.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-29T14:56:37.315 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:37.550 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:37 smithi027 ceph-mon[29342]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:37.551 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:37 smithi027 ceph-mon[18892]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:37 smithi171 ceph-mon[22959]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:37.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:37 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:56:37] "GET /metrics HTTP/1.1" 200 33713 "" "Prometheus/2.43.0" 2024-08-29T14:56:39.497 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:56:39] ENGINE Bus STOPPING 2024-08-29T14:56:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:39 smithi171 ceph-mon[22959]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:39 smithi171 ceph-mon[22959]: from='client.? ' entity='client.admin' 2024-08-29T14:56:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:39 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:56:39.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-mon[18892]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:39.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-mon[18892]: from='client.? ' entity='client.admin' 2024-08-29T14:56:39.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:56:39.797 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-mon[29342]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:39.797 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-mon[29342]: from='client.? ' entity='client.admin' 2024-08-29T14:56:39.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:56:39.798 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:56:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:56:39.798 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:56:39] ENGINE Bus STOPPED 2024-08-29T14:56:39.798 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:56:39] ENGINE Bus STARTING 2024-08-29T14:56:39.798 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:56:39] ENGINE Serving on http://:::9283 2024-08-29T14:56:39.798 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:56:39] ENGINE Bus STARTED 2024-08-29T14:56:39.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'ceph config set global log_to_journald false --force' 2024-08-29T14:56:40.070 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:40 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:40 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:40 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:40.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:40 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:40.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:40 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:40.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:40 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:40 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:56:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:40 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:56:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:40 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:56:41.794 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:41 smithi027 ceph-mon[29342]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:41.794 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:41 smithi027 ceph-mon[18892]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:41 smithi171 ceph-mon[22959]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:43.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-08-29T14:56:43.388 DEBUG:teuthology.orchestra.run.smithi027:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-29T14:56:43.416 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:43.541 INFO:tasks.workunit.client.0.smithi027.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-29T14:56:43.543 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-29T14:56:43.543 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-29T14:56:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:43 smithi027 ceph-mon[18892]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:43 smithi027 ceph-mon[29342]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:43 smithi171 ceph-mon[22959]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:45.416 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-29T14:56:45.496 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-29T14:56:45.539 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-29T14:56:45.541 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-29T14:56:45.541 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-29T14:56:45.583 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-29T14:56:45.586 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T14:56:45.586 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-29T14:56:45.644 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-08-29T14:56:45.645 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-08-29T14:56:45.645 DEBUG:teuthology.orchestra.run.smithi027:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-08-29T14:56:45.725 INFO:tasks.workunit.client.0.smithi027.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-08-29T14:56:45.725 INFO:tasks.workunit.client.0.smithi027.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-08-29T14:56:45.725 INFO:tasks.workunit.client.0.smithi027.stderr:+ TMPDIR=/tmp/rbd_import_export_69295 2024-08-29T14:56:45.725 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm -rf /tmp/rbd_import_export_69295 2024-08-29T14:56:45.726 INFO:tasks.workunit.client.0.smithi027.stderr:+ mkdir /tmp/rbd_import_export_69295 2024-08-29T14:56:45.727 INFO:tasks.workunit.client.0.smithi027.stderr:+ trap 'rm -rf /tmp/rbd_import_export_69295' INT TERM EXIT 2024-08-29T14:56:45.727 INFO:tasks.workunit.client.0.smithi027.stderr:+ mkdir foo.69295 2024-08-29T14:56:45.727 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import foo.69295 foo.dir 2024-08-29T14:56:45.746 INFO:teuthology.orchestra.run.smithi027.stdout:wait for mgr daemons to upgrade 2024-08-29T14:56:45.746 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:45 smithi027 ceph-mon[29342]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:45.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:45 smithi027 ceph-mon[18892]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:45.758 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: cannot import a directory 2024-08-29T14:56:45.758 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 0% complete...failed. 2024-08-29T14:56:45.758 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: import failed: (21) Is a directory 2024-08-29T14:56:45.775 INFO:tasks.workunit.client.0.smithi027.stderr:+ true 2024-08-29T14:56:45.775 INFO:tasks.workunit.client.0.smithi027.stderr:+ rmdir foo.69295 2024-08-29T14:56:45.808 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_69295/img bs=1k count=1 seek=10 2024-08-29T14:56:45.809 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records in 2024-08-29T14:56:45.809 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records out 2024-08-29T14:56:45.809 INFO:tasks.workunit.client.0.smithi027.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.7042e-05 s, 21.8 MB/s 2024-08-29T14:56:45.809 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_69295/img bs=1k count=10 seek=100 2024-08-29T14:56:45.810 INFO:tasks.workunit.client.0.smithi027.stderr:10+0 records in 2024-08-29T14:56:45.810 INFO:tasks.workunit.client.0.smithi027.stderr:10+0 records out 2024-08-29T14:56:45.810 INFO:tasks.workunit.client.0.smithi027.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.6483e-05 s, 181 MB/s 2024-08-29T14:56:45.810 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_69295/img bs=1k count=100 seek=1000 2024-08-29T14:56:45.811 INFO:tasks.workunit.client.0.smithi027.stderr:60+1 records in 2024-08-29T14:56:45.811 INFO:tasks.workunit.client.0.smithi027.stderr:60+1 records out 2024-08-29T14:56:45.811 INFO:tasks.workunit.client.0.smithi027.stderr:61640 bytes (62 kB, 60 KiB) copied, 0.000165946 s, 371 MB/s 2024-08-29T14:56:45.811 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_69295/img bs=1k seek=10000 2024-08-29T14:56:45.812 INFO:tasks.workunit.client.0.smithi027.stderr:137+1 records in 2024-08-29T14:56:45.812 INFO:tasks.workunit.client.0.smithi027.stderr:137+1 records out 2024-08-29T14:56:45.812 INFO:tasks.workunit.client.0.smithi027.stderr:140928 bytes (141 kB, 138 KiB) copied, 0.000536478 s, 263 MB/s 2024-08-29T14:56:45.812 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_69295/img bs=1k seek=100000 2024-08-29T14:56:45.817 INFO:tasks.workunit.client.0.smithi027.stderr:60+1 records in 2024-08-29T14:56:45.817 INFO:tasks.workunit.client.0.smithi027.stderr:60+1 records out 2024-08-29T14:56:45.817 INFO:tasks.workunit.client.0.smithi027.stderr:61592 bytes (62 kB, 60 KiB) copied, 0.000176084 s, 350 MB/s 2024-08-29T14:56:45.817 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_69295/img bs=1k seek=1000000 2024-08-29T14:56:45.818 INFO:tasks.workunit.client.0.smithi027.stderr:154+1 records in 2024-08-29T14:56:45.818 INFO:tasks.workunit.client.0.smithi027.stderr:154+1 records out 2024-08-29T14:56:45.818 INFO:tasks.workunit.client.0.smithi027.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.000386342 s, 409 MB/s 2024-08-29T14:56:45.818 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm testimg 2024-08-29T14:56:45.856 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 0% complete...failed. 2024-08-29T14:56:45.856 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: delete error: (2) No such file or directory 2024-08-29T14:56:45.859 INFO:tasks.workunit.client.0.smithi027.stderr:+ true 2024-08-29T14:56:45.859 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format /tmp/rbd_import_export_69295/img testimg 2024-08-29T14:56:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:45 smithi171 ceph-mon[22959]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:46.361 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-08-29T14:56:46.361 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:56:46.364 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export testimg /tmp/rbd_import_export_69295/img2 2024-08-29T14:56:46.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-08-29T14:56:46.557 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:46.563 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export testimg - 2024-08-29T14:56:46.736 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:56:46.781 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:46 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:46.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:46 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:46 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:56:47.192 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:47.197 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm testimg 2024-08-29T14:56:47.289 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-29T14:56:47.292 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp /tmp/rbd_import_export_69295/img /tmp/rbd_import_export_69295/img2 2024-08-29T14:56:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:47 smithi027 ceph-mon[29342]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:47 smithi027 ceph-mon[18892]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:47.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:56:47] "GET /metrics HTTP/1.1" 200 33711 "" "Prometheus/2.43.0" 2024-08-29T14:56:47.927 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp /tmp/rbd_import_export_69295/img /tmp/rbd_import_export_69295/img3 2024-08-29T14:56:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:47 smithi171 ceph-mon[22959]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:56:48.409 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img2 /tmp/rbd_import_export_69295/img3 2024-08-29T14:56:48.581 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format - testimg 2024-08-29T14:56:48.967 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:56:48.968 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:56:48.971 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export testimg /tmp/rbd_import_export_69295/img2 2024-08-29T14:56:49.154 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:49.160 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export testimg - 2024-08-29T14:56:49.817 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:49.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:49 smithi027 ceph-mon[29342]: pgmap v93: 73 pgs: 73 active+clean; 5.9 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 38 KiB/s wr, 10 op/s 2024-08-29T14:56:49.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:49 smithi027 ceph-mon[18892]: pgmap v93: 73 pgs: 73 active+clean; 5.9 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 38 KiB/s wr, 10 op/s 2024-08-29T14:56:49.818 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:49 smithi171 ceph-mon[22959]: pgmap v93: 73 pgs: 73 active+clean; 5.9 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 38 KiB/s wr, 10 op/s 2024-08-29T14:56:49.822 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm testimg 2024-08-29T14:56:49.917 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-29T14:56:49.920 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp /tmp/rbd_import_export_69295/img /tmp/rbd_import_export_69295/img2 2024-08-29T14:56:50.607 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp /tmp/rbd_import_export_69295/img /tmp/rbd_import_export_69295/img3 2024-08-29T14:56:50.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:50 smithi027 ceph-mon[18892]: from='client.24695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:56:50.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:50 smithi027 ceph-mon[29342]: from='client.24695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:56:51.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:50 smithi171 ceph-mon[22959]: from='client.24695 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:56:51.107 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img /tmp/rbd_import_export_69295/img2 /tmp/rbd_import_export_69295/img3 2024-08-29T14:56:51.345 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd help export 2024-08-29T14:56:51.345 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep -q export-format 2024-08-29T14:56:51.370 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_69295/img bs=1k count=10 seek=100 2024-08-29T14:56:51.371 INFO:tasks.workunit.client.0.smithi027.stderr:10+0 records in 2024-08-29T14:56:51.371 INFO:tasks.workunit.client.0.smithi027.stderr:10+0 records out 2024-08-29T14:56:51.371 INFO:tasks.workunit.client.0.smithi027.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.5682e-05 s, 156 MB/s 2024-08-29T14:56:51.371 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format /tmp/rbd_import_export_69295/img testimg 2024-08-29T14:56:51.439 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:56:51.439 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:56:51.442 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap create testimg@snap 2024-08-29T14:56:51.579 INFO:tasks.workunit.client.0.smithi027.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-08-29T14:56:51.586 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd image-meta set testimg key1 value1 2024-08-29T14:56:51.642 INFO:tasks.workunit.client.0.smithi027.stderr:++ rbd image-meta list testimg 2024-08-29T14:56:51.690 INFO:tasks.workunit.client.0.smithi027.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-08-29T14:56:51.690 INFO:tasks.workunit.client.0.smithi027.stderr: 2024-08-29T14:56:51.690 INFO:tasks.workunit.client.0.smithi027.stderr:Key Value 2024-08-29T14:56:51.690 INFO:tasks.workunit.client.0.smithi027.stderr:key1 value1' 2024-08-29T14:56:51.691 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:51.735 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:51.739 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69295/img_v2 testimg_import 2024-08-29T14:56:51.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:51 smithi027 ceph-mon[29342]: pgmap v94: 73 pgs: 73 active+clean; 5.9 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 38 KiB/s wr, 10 op/s 2024-08-29T14:56:51.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:51 smithi027 ceph-mon[18892]: pgmap v94: 73 pgs: 73 active+clean; 5.9 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 38 KiB/s wr, 10 op/s 2024-08-29T14:56:52.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:51 smithi171 ceph-mon[22959]: pgmap v94: 73 pgs: 73 active+clean; 5.9 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 38 KiB/s wr, 10 op/s 2024-08-29T14:56:52.590 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-29T14:56:52.593 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd info testimg_import 2024-08-29T14:56:52.635 INFO:tasks.workunit.client.0.smithi027.stdout:rbd image 'testimg_import': 2024-08-29T14:56:52.635 INFO:tasks.workunit.client.0.smithi027.stdout: size 110 KiB in 1 objects 2024-08-29T14:56:52.635 INFO:tasks.workunit.client.0.smithi027.stdout: order 22 (4 MiB objects) 2024-08-29T14:56:52.635 INFO:tasks.workunit.client.0.smithi027.stdout: snapshot_count: 1 2024-08-29T14:56:52.635 INFO:tasks.workunit.client.0.smithi027.stdout: id: 60b3c9426338 2024-08-29T14:56:52.635 INFO:tasks.workunit.client.0.smithi027.stdout: block_name_prefix: rbd_data.60b3c9426338 2024-08-29T14:56:52.635 INFO:tasks.workunit.client.0.smithi027.stdout: format: 2 2024-08-29T14:56:52.636 INFO:tasks.workunit.client.0.smithi027.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-08-29T14:56:52.636 INFO:tasks.workunit.client.0.smithi027.stdout: op_features: 2024-08-29T14:56:52.636 INFO:tasks.workunit.client.0.smithi027.stdout: flags: 2024-08-29T14:56:52.636 INFO:tasks.workunit.client.0.smithi027.stdout: create_timestamp: Thu Aug 29 14:56:51 2024 2024-08-29T14:56:52.636 INFO:tasks.workunit.client.0.smithi027.stdout: access_timestamp: Thu Aug 29 14:56:51 2024 2024-08-29T14:56:52.636 INFO:tasks.workunit.client.0.smithi027.stdout: modify_timestamp: Thu Aug 29 14:56:51 2024 2024-08-29T14:56:52.638 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd info testimg_import@snap 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout:rbd image 'testimg_import': 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: size 110 KiB in 1 objects 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: order 22 (4 MiB objects) 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: snapshot_count: 1 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: id: 60b3c9426338 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: block_name_prefix: rbd_data.60b3c9426338 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: format: 2 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: op_features: 2024-08-29T14:56:52.680 INFO:tasks.workunit.client.0.smithi027.stdout: flags: 2024-08-29T14:56:52.681 INFO:tasks.workunit.client.0.smithi027.stdout: create_timestamp: Thu Aug 29 14:56:51 2024 2024-08-29T14:56:52.681 INFO:tasks.workunit.client.0.smithi027.stdout: access_timestamp: Thu Aug 29 14:56:51 2024 2024-08-29T14:56:52.681 INFO:tasks.workunit.client.0.smithi027.stdout: modify_timestamp: Thu Aug 29 14:56:51 2024 2024-08-29T14:56:52.681 INFO:tasks.workunit.client.0.smithi027.stdout: protected: False 2024-08-29T14:56:52.683 INFO:tasks.workunit.client.0.smithi027.stderr:++ rbd image-meta list testimg_import 2024-08-29T14:56:52.731 INFO:tasks.workunit.client.0.smithi027.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-08-29T14:56:52.731 INFO:tasks.workunit.client.0.smithi027.stderr: 2024-08-29T14:56:52.731 INFO:tasks.workunit.client.0.smithi027.stderr:Key Value 2024-08-29T14:56:52.731 INFO:tasks.workunit.client.0.smithi027.stderr:key1 value1' 2024-08-29T14:56:52.731 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 'There is 1 metadatum on this image: 2024-08-29T14:56:52.732 INFO:tasks.workunit.client.0.smithi027.stderr: 2024-08-29T14:56:52.732 INFO:tasks.workunit.client.0.smithi027.stderr:Key Value 2024-08-29T14:56:52.732 INFO:tasks.workunit.client.0.smithi027.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-08-29T14:56:52.732 INFO:tasks.workunit.client.0.smithi027.stderr: 2024-08-29T14:56:52.732 INFO:tasks.workunit.client.0.smithi027.stderr:Key Value 2024-08-29T14:56:52.733 INFO:tasks.workunit.client.0.smithi027.stderr:key1 value1' ']' 2024-08-29T14:56:52.733 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export testimg_import /tmp/rbd_import_export_69295/img_import 2024-08-29T14:56:52.773 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete...done. 2024-08-29T14:56:52.777 INFO:tasks.workunit.client.0.smithi027.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69295/img /tmp/rbd_import_export_69295/img_import 2024-08-29T14:56:52.777 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp -l /tmp/rbd_import_export_69295/img /tmp/rbd_import_export_69295/img_import 2024-08-29T14:56:52.778 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/img --format %b 2024-08-29T14:56:52.779 INFO:tasks.workunit.client.0.smithi027.stderr:+ origsize=24 2024-08-29T14:56:52.780 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/img_import --format %b 2024-08-29T14:56:52.781 INFO:tasks.workunit.client.0.smithi027.stderr:+ exportsize=224 2024-08-29T14:56:52.782 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=200 2024-08-29T14:56:52.782 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=200 2024-08-29T14:56:52.782 INFO:tasks.workunit.client.0.smithi027.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-08-29T14:56:52.782 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_69295/img_snap 2024-08-29T14:56:52.826 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete...done. 2024-08-29T14:56:52.829 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_69295/img_snap_import 2024-08-29T14:56:52.875 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete...done. 2024-08-29T14:56:52.878 INFO:tasks.workunit.client.0.smithi027.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69295/img_snap /tmp/rbd_import_export_69295/img_snap_import 2024-08-29T14:56:52.878 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp -l /tmp/rbd_import_export_69295/img_snap /tmp/rbd_import_export_69295/img_snap_import 2024-08-29T14:56:52.879 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/img_snap --format %b 2024-08-29T14:56:52.880 INFO:tasks.workunit.client.0.smithi027.stderr:+ origsize=224 2024-08-29T14:56:52.880 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/img_snap_import --format %b 2024-08-29T14:56:52.882 INFO:tasks.workunit.client.0.smithi027.stderr:+ exportsize=224 2024-08-29T14:56:52.882 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:56:52.882 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:56:52.882 INFO:tasks.workunit.client.0.smithi027.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-29T14:56:52.882 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:52.883 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img_import 2024-08-29T14:56:52.883 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img_snap 2024-08-29T14:56:52.884 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img_snap_import 2024-08-29T14:56:52.885 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap rm testimg_import@snap 2024-08-29T14:56:53.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:52 smithi171 ceph-mon[22959]: osdmap e74: 8 total, 8 up, 8 in 2024-08-29T14:56:53.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:52 smithi171 ceph-mon[22959]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 95 KiB/s wr, 89 op/s 2024-08-29T14:56:53.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:52 smithi027 ceph-mon[29342]: osdmap e74: 8 total, 8 up, 8 in 2024-08-29T14:56:53.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:52 smithi027 ceph-mon[29342]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 95 KiB/s wr, 89 op/s 2024-08-29T14:56:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:52 smithi027 ceph-mon[18892]: osdmap e74: 8 total, 8 up, 8 in 2024-08-29T14:56:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:52 smithi027 ceph-mon[18892]: pgmap v96: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 95 KiB/s wr, 89 op/s 2024-08-29T14:56:53.582 INFO:tasks.workunit.client.0.smithi027.stderr: Removing snap: 100% complete...done. 2024-08-29T14:56:53.589 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg_import 2024-08-29T14:56:53.687 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:56:53.691 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap rm testimg@snap 2024-08-29T14:56:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:53 smithi171 ceph-mon[22959]: osdmap e75: 8 total, 8 up, 8 in 2024-08-29T14:56:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:53 smithi027 ceph-mon[29342]: osdmap e75: 8 total, 8 up, 8 in 2024-08-29T14:56:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:53 smithi027 ceph-mon[18892]: osdmap e75: 8 total, 8 up, 8 in 2024-08-29T14:56:54.587 INFO:tasks.workunit.client.0.smithi027.stderr: Removing snap: 100% complete...done. 2024-08-29T14:56:54.594 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm testimg 2024-08-29T14:56:54.698 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:56:54.702 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --order 20 /tmp/rbd_import_export_69295/img testimg 2024-08-29T14:56:54.767 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:56:54.768 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --order is deprecated, use --object-size 2024-08-29T14:56:54.770 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:54.816 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:54.820 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69295/img_v2 testimg_import 2024-08-29T14:56:54.884 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-29T14:56:54.888 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd info testimg_import 2024-08-29T14:56:54.888 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep order 2024-08-29T14:56:54.888 INFO:tasks.workunit.client.0.smithi027.stderr:+ awk '{print $2}' 2024-08-29T14:56:54.888 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep 20 2024-08-29T14:56:54.931 INFO:tasks.workunit.client.0.smithi027.stdout:20 2024-08-29T14:56:54.931 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:54.932 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg_import 2024-08-29T14:56:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:54 smithi171 ceph-mon[22959]: osdmap e76: 8 total, 8 up, 8 in 2024-08-29T14:56:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:54 smithi171 ceph-mon[22959]: pgmap v99: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 312 KiB/s rd, 83 KiB/s wr, 129 op/s 2024-08-29T14:56:55.021 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:56:55.024 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg 2024-08-29T14:56:55.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:54 smithi027 ceph-mon[29342]: osdmap e76: 8 total, 8 up, 8 in 2024-08-29T14:56:55.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:54 smithi027 ceph-mon[29342]: pgmap v99: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 312 KiB/s rd, 83 KiB/s wr, 129 op/s 2024-08-29T14:56:55.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:54 smithi027 ceph-mon[18892]: osdmap e76: 8 total, 8 up, 8 in 2024-08-29T14:56:55.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:54 smithi027 ceph-mon[18892]: pgmap v99: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 312 KiB/s rd, 83 KiB/s wr, 129 op/s 2024-08-29T14:56:55.117 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:56:55.120 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_69295/img testimg 2024-08-29T14:56:55.175 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:56:55.179 INFO:tasks.workunit.client.0.smithi027.stderr:++ rbd info testimg 2024-08-29T14:56:55.179 INFO:tasks.workunit.client.0.smithi027.stderr:++ grep features 2024-08-29T14:56:55.222 INFO:tasks.workunit.client.0.smithi027.stderr:+ FEATURES_BEFORE=' features: layering 2024-08-29T14:56:55.222 INFO:tasks.workunit.client.0.smithi027.stderr: op_features: ' 2024-08-29T14:56:55.222 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:55.266 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:55.269 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69295/img_v2 testimg_import 2024-08-29T14:56:55.329 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-29T14:56:55.333 INFO:tasks.workunit.client.0.smithi027.stderr:++ rbd info testimg_import 2024-08-29T14:56:55.333 INFO:tasks.workunit.client.0.smithi027.stderr:++ grep features 2024-08-29T14:56:55.376 INFO:tasks.workunit.client.0.smithi027.stderr:+ FEATURES_AFTER=' features: layering 2024-08-29T14:56:55.377 INFO:tasks.workunit.client.0.smithi027.stderr: op_features: ' 2024-08-29T14:56:55.377 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' ' features: layering 2024-08-29T14:56:55.377 INFO:tasks.workunit.client.0.smithi027.stderr: op_features: ' '!=' ' features: layering 2024-08-29T14:56:55.377 INFO:tasks.workunit.client.0.smithi027.stderr: op_features: ' ']' 2024-08-29T14:56:55.377 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:55.377 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg_import 2024-08-29T14:56:55.451 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:56:55.454 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg 2024-08-29T14:56:55.530 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:56:55.533 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_69295/img testimg 2024-08-29T14:56:55.605 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:56:55.608 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:55.650 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:55.653 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69295/img_v2 testimg_import 2024-08-29T14:56:55.715 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-29T14:56:55.718 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd info testimg_import 2024-08-29T14:56:55.718 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep 'stripe unit' 2024-08-29T14:56:55.719 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep -Ei '(4 KiB|4096)' 2024-08-29T14:56:55.758 INFO:tasks.workunit.client.0.smithi027.stdout: stripe unit: 4 KiB 2024-08-29T14:56:55.759 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd info testimg_import 2024-08-29T14:56:55.759 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep 'stripe count' 2024-08-29T14:56:55.759 INFO:tasks.workunit.client.0.smithi027.stderr:+ awk '{print $3}' 2024-08-29T14:56:55.759 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep 1000 2024-08-29T14:56:55.799 INFO:tasks.workunit.client.0.smithi027.stdout:1000 2024-08-29T14:56:55.799 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/img_v2 2024-08-29T14:56:55.799 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg_import 2024-08-29T14:56:55.884 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-29T14:56:55.887 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg 2024-08-29T14:56:55.984 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-29T14:56:55.987 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_69295/img testimg 2024-08-29T14:56:56.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:55 smithi171 ceph-mon[22959]: osdmap e77: 8 total, 8 up, 8 in 2024-08-29T14:56:56.049 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:56:56.052 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap create testimg@snap1 2024-08-29T14:56:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:55 smithi027 ceph-mon[29342]: osdmap e77: 8 total, 8 up, 8 in 2024-08-29T14:56:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:55 smithi027 ceph-mon[18892]: osdmap e77: 8 total, 8 up, 8 in 2024-08-29T14:56:56.599 INFO:tasks.workunit.client.0.smithi027.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-08-29T14:56:56.605 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap create testimg@snap2 2024-08-29T14:56:57.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:56 smithi171 ceph-mon[22959]: pgmap v101: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 449 KiB/s rd, 124 KiB/s wr, 186 op/s 2024-08-29T14:56:57.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:56 smithi027 ceph-mon[18892]: pgmap v101: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 449 KiB/s rd, 124 KiB/s wr, 186 op/s 2024-08-29T14:56:57.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:56 smithi027 ceph-mon[29342]: pgmap v101: 73 pgs: 73 active+clean; 689 KiB data, 223 MiB used, 715 GiB / 715 GiB avail; 449 KiB/s rd, 124 KiB/s wr, 186 op/s 2024-08-29T14:56:57.606 INFO:tasks.workunit.client.0.smithi027.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-08-29T14:56:57.613 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap protect testimg@snap2 2024-08-29T14:56:57.662 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69295/snap_protect 2024-08-29T14:56:57.710 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-08-29T14:56:57.713 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69295/snap_protect testimg_import 2024-08-29T14:56:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:57 smithi171 ceph-mon[22959]: osdmap e78: 8 total, 8 up, 8 in 2024-08-29T14:56:58.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:57 smithi027 ceph-mon[29342]: osdmap e78: 8 total, 8 up, 8 in 2024-08-29T14:56:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:57 smithi027 ceph-mon[18892]: osdmap e78: 8 total, 8 up, 8 in 2024-08-29T14:56:58.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:56:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:56:57] "GET /metrics HTTP/1.1" 200 33711 "" "Prometheus/2.43.0" 2024-08-29T14:56:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:58 smithi171 ceph-mon[22959]: pgmap v103: 73 pgs: 73 active+clean; 689 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 389 KiB/s rd, 42 KiB/s wr, 510 op/s 2024-08-29T14:56:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:58 smithi171 ceph-mon[22959]: osdmap e79: 8 total, 8 up, 8 in 2024-08-29T14:56:59.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:58 smithi027 ceph-mon[29342]: pgmap v103: 73 pgs: 73 active+clean; 689 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 389 KiB/s rd, 42 KiB/s wr, 510 op/s 2024-08-29T14:56:59.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:58 smithi027 ceph-mon[29342]: osdmap e79: 8 total, 8 up, 8 in 2024-08-29T14:56:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:58 smithi027 ceph-mon[18892]: pgmap v103: 73 pgs: 73 active+clean; 689 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 389 KiB/s rd, 42 KiB/s wr, 510 op/s 2024-08-29T14:56:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:58 smithi027 ceph-mon[18892]: osdmap e79: 8 total, 8 up, 8 in 2024-08-29T14:56:59.510 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-08-29T14:56:59.516 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd info testimg_import@snap1 2024-08-29T14:56:59.516 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep 'protected: False' 2024-08-29T14:56:59.557 INFO:tasks.workunit.client.0.smithi027.stdout: protected: False 2024-08-29T14:56:59.558 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd info testimg_import@snap2 2024-08-29T14:56:59.558 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep 'protected: True' 2024-08-29T14:56:59.604 INFO:tasks.workunit.client.0.smithi027.stdout: protected: True 2024-08-29T14:56:59.604 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/snap_protect 2024-08-29T14:56:59.605 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap unprotect testimg@snap2 2024-08-29T14:56:59.653 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap unprotect testimg_import@snap2 2024-08-29T14:56:59.699 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap purge testimg 2024-08-29T14:57:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:59 smithi171 ceph-mon[22959]: osdmap e80: 8 total, 8 up, 8 in 2024-08-29T14:57:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:56:59 smithi171 ceph-mon[22959]: osdmap e81: 8 total, 8 up, 8 in 2024-08-29T14:57:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:59 smithi027 ceph-mon[18892]: osdmap e80: 8 total, 8 up, 8 in 2024-08-29T14:57:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:56:59 smithi027 ceph-mon[18892]: osdmap e81: 8 total, 8 up, 8 in 2024-08-29T14:57:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:59 smithi027 ceph-mon[29342]: osdmap e80: 8 total, 8 up, 8 in 2024-08-29T14:57:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:56:59 smithi027 ceph-mon[29342]: osdmap e81: 8 total, 8 up, 8 in 2024-08-29T14:57:01.611 INFO:tasks.workunit.client.0.smithi027.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-08-29T14:57:01.620 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd snap purge testimg_import 2024-08-29T14:57:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:01 smithi171 ceph-mon[22959]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 487 KiB/s rd, 52 KiB/s wr, 638 op/s 2024-08-29T14:57:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:01 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:01 smithi171 ceph-mon[22959]: osdmap e82: 8 total, 8 up, 8 in 2024-08-29T14:57:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:01 smithi027 ceph-mon[18892]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 487 KiB/s rd, 52 KiB/s wr, 638 op/s 2024-08-29T14:57:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:01 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:01 smithi027 ceph-mon[18892]: osdmap e82: 8 total, 8 up, 8 in 2024-08-29T14:57:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:01 smithi027 ceph-mon[29342]: pgmap v107: 73 pgs: 73 active+clean; 689 KiB data, 227 MiB used, 715 GiB / 715 GiB avail; 487 KiB/s rd, 52 KiB/s wr, 638 op/s 2024-08-29T14:57:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:01 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:01 smithi027 ceph-mon[29342]: osdmap e82: 8 total, 8 up, 8 in 2024-08-29T14:57:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:02 smithi171 ceph-mon[22959]: pgmap v109: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2024-08-29T14:57:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:02 smithi171 ceph-mon[22959]: osdmap e83: 8 total, 8 up, 8 in 2024-08-29T14:57:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:02 smithi027 ceph-mon[18892]: pgmap v109: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2024-08-29T14:57:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:02 smithi027 ceph-mon[18892]: osdmap e83: 8 total, 8 up, 8 in 2024-08-29T14:57:03.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:02 smithi027 ceph-mon[29342]: pgmap v109: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2024-08-29T14:57:03.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:02 smithi027 ceph-mon[29342]: osdmap e83: 8 total, 8 up, 8 in 2024-08-29T14:57:03.628 INFO:tasks.workunit.client.0.smithi027.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-08-29T14:57:03.638 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg 2024-08-29T14:57:03.734 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:57:03.737 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd remove testimg_import 2024-08-29T14:57:03.830 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:57:03.833 INFO:tasks.workunit.client.0.smithi027.stderr:+ tiered=0 2024-08-29T14:57:03.833 INFO:tasks.workunit.client.0.smithi027.stderr:+ ceph osd dump 2024-08-29T14:57:03.833 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep '^pool' 2024-08-29T14:57:03.833 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep ''\''rbd'\''' 2024-08-29T14:57:03.833 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep tier 2024-08-29T14:57:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:03 smithi171 ceph-mon[22959]: osdmap e84: 8 total, 8 up, 8 in 2024-08-29T14:57:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:03 smithi027 ceph-mon[29342]: osdmap e84: 8 total, 8 up, 8 in 2024-08-29T14:57:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:03 smithi027 ceph-mon[18892]: osdmap e84: 8 total, 8 up, 8 in 2024-08-29T14:57:04.253 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_69295/sparse1 2024-08-29T14:57:04.257 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records in 2024-08-29T14:57:04.257 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records out 2024-08-29T14:57:04.258 INFO:tasks.workunit.client.0.smithi027.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00354808 s, 296 MB/s 2024-08-29T14:57:04.258 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_69295/sparse2 2024-08-29T14:57:04.262 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records in 2024-08-29T14:57:04.262 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records out 2024-08-29T14:57:04.262 INFO:tasks.workunit.client.0.smithi027.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0032729 s, 320 MB/s 2024-08-29T14:57:04.262 INFO:tasks.workunit.client.0.smithi027.stderr:+ truncate /tmp/rbd_import_export_69295/sparse2 -s 2M 2024-08-29T14:57:04.271 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse1 2024-08-29T14:57:04.316 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 0% complete...failed. 2024-08-29T14:57:04.316 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: delete error: (2) No such file or directory 2024-08-29T14:57:04.319 INFO:tasks.workunit.client.0.smithi027.stderr:+ true 2024-08-29T14:57:04.320 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_69295/sparse1 2024-08-29T14:57:04.391 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-08-29T14:57:04.391 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:57:04.392 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --order is deprecated, use --object-size 2024-08-29T14:57:04.395 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd ls -l 2024-08-29T14:57:04.395 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep sparse1 2024-08-29T14:57:04.395 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-08-29T14:57:04.439 INFO:tasks.workunit.client.0.smithi027.stdout:sparse1 2 MiB 2 2024-08-29T14:57:04.440 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects sparse1 2024-08-29T14:57:04.440 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse1 2024-08-29T14:57:04.440 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse1 2024-08-29T14:57:04.440 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep block_name_prefix 2024-08-29T14:57:04.440 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk '{print $NF;}' 2024-08-29T14:57:04.485 INFO:tasks.workunit.client.0.smithi027.stderr:++ prefix=rbd_data.61cd8069b528 2024-08-29T14:57:04.487 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep rbd_data.61cd8069b528 2024-08-29T14:57:04.487 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sort -u 2024-08-29T14:57:04.487 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sed -e 's/rbd_data.61cd8069b528\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-29T14:57:04.488 INFO:tasks.workunit.client.0.smithi027.stderr:++++ get_image_data_pool sparse1 2024-08-29T14:57:04.488 INFO:tasks.workunit.client.0.smithi027.stderr:++++ image=sparse1 2024-08-29T14:57:04.488 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ rbd info sparse1 2024-08-29T14:57:04.489 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ grep 'data_pool: ' 2024-08-29T14:57:04.489 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ awk -F: '{ print $NF }' 2024-08-29T14:57:04.536 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool= 2024-08-29T14:57:04.536 INFO:tasks.workunit.client.0.smithi027.stderr:++++ '[' -z ']' 2024-08-29T14:57:04.537 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool=rbd 2024-08-29T14:57:04.537 INFO:tasks.workunit.client.0.smithi027.stderr:++++ echo rbd 2024-08-29T14:57:04.537 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rados ls -p rbd 2024-08-29T14:57:04.578 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects=1 2024-08-29T14:57:04.578 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo 1 2024-08-29T14:57:04.578 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-08-29T14:57:04.578 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export sparse1 /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:04.623 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-08-29T14:57:04.628 INFO:tasks.workunit.client.0.smithi027.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69295/sparse1 /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:04.629 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp -l /tmp/rbd_import_export_69295/sparse1 /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:04.629 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse1 --format %b 2024-08-29T14:57:04.630 INFO:tasks.workunit.client.0.smithi027.stderr:+ origsize=2048 2024-08-29T14:57:04.630 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse1.out --format %b 2024-08-29T14:57:04.632 INFO:tasks.workunit.client.0.smithi027.stderr:+ exportsize=2048 2024-08-29T14:57:04.632 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:04.632 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:04.632 INFO:tasks.workunit.client.0.smithi027.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-29T14:57:04.632 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:04.632 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse1 2024-08-29T14:57:04.729 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-08-29T14:57:04.732 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse2 2024-08-29T14:57:04.778 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 0% complete...failed. 2024-08-29T14:57:04.778 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: delete error: (2) No such file or directory 2024-08-29T14:57:04.781 INFO:tasks.workunit.client.0.smithi027.stderr:+ true 2024-08-29T14:57:04.781 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_69295/sparse2 2024-08-29T14:57:04.849 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-08-29T14:57:04.850 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:57:04.850 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --order is deprecated, use --object-size 2024-08-29T14:57:04.853 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd ls -l 2024-08-29T14:57:04.853 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep sparse2 2024-08-29T14:57:04.853 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-08-29T14:57:04.897 INFO:tasks.workunit.client.0.smithi027.stdout:sparse2 2 MiB 2 2024-08-29T14:57:04.897 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects sparse2 2024-08-29T14:57:04.898 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse2 2024-08-29T14:57:04.898 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse2 2024-08-29T14:57:04.898 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep block_name_prefix 2024-08-29T14:57:04.898 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk '{print $NF;}' 2024-08-29T14:57:04.941 INFO:tasks.workunit.client.0.smithi027.stderr:++ prefix=rbd_data.3b58469b2115 2024-08-29T14:57:04.942 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep rbd_data.3b58469b2115 2024-08-29T14:57:04.942 INFO:tasks.workunit.client.0.smithi027.stderr:++++ get_image_data_pool sparse2 2024-08-29T14:57:04.942 INFO:tasks.workunit.client.0.smithi027.stderr:++++ image=sparse2 2024-08-29T14:57:04.942 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sed -e 's/rbd_data.3b58469b2115\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-29T14:57:04.942 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sort -u 2024-08-29T14:57:04.943 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ rbd info sparse2 2024-08-29T14:57:04.943 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ grep 'data_pool: ' 2024-08-29T14:57:04.944 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ awk -F: '{ print $NF }' 2024-08-29T14:57:04.987 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool= 2024-08-29T14:57:04.987 INFO:tasks.workunit.client.0.smithi027.stderr:++++ '[' -z ']' 2024-08-29T14:57:04.987 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool=rbd 2024-08-29T14:57:04.988 INFO:tasks.workunit.client.0.smithi027.stderr:++++ echo rbd 2024-08-29T14:57:04.988 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rados ls -p rbd 2024-08-29T14:57:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:04 smithi171 ceph-mon[22959]: pgmap v112: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 11 KiB/s wr, 164 op/s 2024-08-29T14:57:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:04 smithi171 ceph-mon[22959]: osdmap e85: 8 total, 8 up, 8 in 2024-08-29T14:57:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:04 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3881025745' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-29T14:57:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:04 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3881025745' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-29T14:57:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:04 smithi171 ceph-mon[22959]: osdmap e86: 8 total, 8 up, 8 in 2024-08-29T14:57:05.028 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects=0 2024-08-29T14:57:05.028 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo 0 2024-08-29T14:57:05.028 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-08-29T14:57:05.028 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export sparse2 /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[18892]: pgmap v112: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 11 KiB/s wr, 164 op/s 2024-08-29T14:57:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[18892]: osdmap e85: 8 total, 8 up, 8 in 2024-08-29T14:57:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3881025745' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-29T14:57:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3881025745' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-29T14:57:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[18892]: osdmap e86: 8 total, 8 up, 8 in 2024-08-29T14:57:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[29342]: pgmap v112: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s rd, 11 KiB/s wr, 164 op/s 2024-08-29T14:57:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[29342]: osdmap e85: 8 total, 8 up, 8 in 2024-08-29T14:57:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3881025745' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-29T14:57:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3881025745' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-08-29T14:57:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:04 smithi027 ceph-mon[29342]: osdmap e86: 8 total, 8 up, 8 in 2024-08-29T14:57:05.071 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-08-29T14:57:05.074 INFO:tasks.workunit.client.0.smithi027.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69295/sparse2 /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.074 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp -l /tmp/rbd_import_export_69295/sparse2 /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.076 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse2 --format %b 2024-08-29T14:57:05.077 INFO:tasks.workunit.client.0.smithi027.stderr:+ origsize=2048 2024-08-29T14:57:05.077 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse2.out --format %b 2024-08-29T14:57:05.079 INFO:tasks.workunit.client.0.smithi027.stderr:+ exportsize=2048 2024-08-29T14:57:05.079 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:05.079 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:05.079 INFO:tasks.workunit.client.0.smithi027.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-29T14:57:05.079 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.080 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse2 2024-08-29T14:57:05.175 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-08-29T14:57:05.179 INFO:tasks.workunit.client.0.smithi027.stderr:+ truncate /tmp/rbd_import_export_69295/sparse1 -s 10M 2024-08-29T14:57:05.179 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format --order 20 - sparse1 2024-08-29T14:57:05.279 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:57:05.280 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:57:05.280 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --order is deprecated, use --object-size 2024-08-29T14:57:05.283 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd ls -l 2024-08-29T14:57:05.283 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep sparse1 2024-08-29T14:57:05.283 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-08-29T14:57:05.328 INFO:tasks.workunit.client.0.smithi027.stdout:sparse1 10 MiB 2 2024-08-29T14:57:05.329 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects sparse1 2024-08-29T14:57:05.329 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse1 2024-08-29T14:57:05.329 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse1 2024-08-29T14:57:05.329 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep block_name_prefix 2024-08-29T14:57:05.329 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk '{print $NF;}' 2024-08-29T14:57:05.373 INFO:tasks.workunit.client.0.smithi027.stderr:++ prefix=rbd_data.3b8564724793 2024-08-29T14:57:05.374 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep rbd_data.3b8564724793 2024-08-29T14:57:05.374 INFO:tasks.workunit.client.0.smithi027.stderr:++++ get_image_data_pool sparse1 2024-08-29T14:57:05.375 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sed -e 's/rbd_data.3b8564724793\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-29T14:57:05.375 INFO:tasks.workunit.client.0.smithi027.stderr:++++ image=sparse1 2024-08-29T14:57:05.375 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sort -u 2024-08-29T14:57:05.376 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ rbd info sparse1 2024-08-29T14:57:05.376 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ grep 'data_pool: ' 2024-08-29T14:57:05.376 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ awk -F: '{ print $NF }' 2024-08-29T14:57:05.420 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool= 2024-08-29T14:57:05.420 INFO:tasks.workunit.client.0.smithi027.stderr:++++ '[' -z ']' 2024-08-29T14:57:05.420 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool=rbd 2024-08-29T14:57:05.420 INFO:tasks.workunit.client.0.smithi027.stderr:++++ echo rbd 2024-08-29T14:57:05.420 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rados ls -p rbd 2024-08-29T14:57:05.461 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects=1 2024-08-29T14:57:05.462 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo 1 2024-08-29T14:57:05.462 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-08-29T14:57:05.462 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export sparse1 /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:05.509 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-08-29T14:57:05.513 INFO:tasks.workunit.client.0.smithi027.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69295/sparse1 /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:05.513 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp -l /tmp/rbd_import_export_69295/sparse1 /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:05.521 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse1 --format %b 2024-08-29T14:57:05.522 INFO:tasks.workunit.client.0.smithi027.stderr:+ origsize=2048 2024-08-29T14:57:05.522 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse1.out --format %b 2024-08-29T14:57:05.523 INFO:tasks.workunit.client.0.smithi027.stderr:+ exportsize=2048 2024-08-29T14:57:05.523 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:05.523 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:05.523 INFO:tasks.workunit.client.0.smithi027.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-29T14:57:05.524 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/sparse1.out 2024-08-29T14:57:05.525 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse1 2024-08-29T14:57:05.618 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-08-29T14:57:05.622 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_69295/sparse2 oflag=append conv=notrunc 2024-08-29T14:57:05.630 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records in 2024-08-29T14:57:05.630 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records out 2024-08-29T14:57:05.630 INFO:tasks.workunit.client.0.smithi027.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00673979 s, 311 MB/s 2024-08-29T14:57:05.630 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format --order 20 - sparse2 2024-08-29T14:57:05.723 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:57:05.723 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:57:05.724 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --order is deprecated, use --object-size 2024-08-29T14:57:05.727 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd ls -l 2024-08-29T14:57:05.727 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep sparse2 2024-08-29T14:57:05.727 INFO:tasks.workunit.client.0.smithi027.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-08-29T14:57:05.769 INFO:tasks.workunit.client.0.smithi027.stdout:sparse2 4 MiB 2 2024-08-29T14:57:05.770 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects sparse2 2024-08-29T14:57:05.770 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse2 2024-08-29T14:57:05.770 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse2 2024-08-29T14:57:05.771 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep block_name_prefix 2024-08-29T14:57:05.771 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk '{print $NF;}' 2024-08-29T14:57:05.813 INFO:tasks.workunit.client.0.smithi027.stderr:++ prefix=rbd_data.625a4b56cb6a 2024-08-29T14:57:05.814 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep rbd_data.625a4b56cb6a 2024-08-29T14:57:05.814 INFO:tasks.workunit.client.0.smithi027.stderr:++++ get_image_data_pool sparse2 2024-08-29T14:57:05.814 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sed -e 's/rbd_data.625a4b56cb6a\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-29T14:57:05.815 INFO:tasks.workunit.client.0.smithi027.stderr:++++ image=sparse2 2024-08-29T14:57:05.815 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sort -u 2024-08-29T14:57:05.815 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ rbd info sparse2 2024-08-29T14:57:05.815 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ grep 'data_pool: ' 2024-08-29T14:57:05.815 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ awk -F: '{ print $NF }' 2024-08-29T14:57:05.857 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool= 2024-08-29T14:57:05.858 INFO:tasks.workunit.client.0.smithi027.stderr:++++ '[' -z ']' 2024-08-29T14:57:05.858 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool=rbd 2024-08-29T14:57:05.858 INFO:tasks.workunit.client.0.smithi027.stderr:++++ echo rbd 2024-08-29T14:57:05.858 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rados ls -p rbd 2024-08-29T14:57:05.899 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects='0 2024-08-29T14:57:05.899 INFO:tasks.workunit.client.0.smithi027.stderr:2 2024-08-29T14:57:05.899 INFO:tasks.workunit.client.0.smithi027.stderr:3' 2024-08-29T14:57:05.899 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo 0 2 3 2024-08-29T14:57:05.899 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-08-29T14:57:05.899 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export sparse2 /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.944 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-08-29T14:57:05.948 INFO:tasks.workunit.client.0.smithi027.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69295/sparse2 /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.948 INFO:tasks.workunit.client.0.smithi027.stderr:+ cmp -l /tmp/rbd_import_export_69295/sparse2 /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.951 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse2 --format %b 2024-08-29T14:57:05.952 INFO:tasks.workunit.client.0.smithi027.stderr:+ origsize=6144 2024-08-29T14:57:05.952 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse2.out --format %b 2024-08-29T14:57:05.953 INFO:tasks.workunit.client.0.smithi027.stderr:+ exportsize=6144 2024-08-29T14:57:05.954 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:05.954 INFO:tasks.workunit.client.0.smithi027.stderr:+ difference=0 2024-08-29T14:57:05.954 INFO:tasks.workunit.client.0.smithi027.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-08-29T14:57:05.954 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/sparse2.out 2024-08-29T14:57:05.954 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse2 2024-08-29T14:57:06.047 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-08-29T14:57:06.051 INFO:tasks.workunit.client.0.smithi027.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-08-29T14:57:06.051 INFO:tasks.workunit.client.0.smithi027.stdout:partially-sparse file imports to partially-sparse image 2024-08-29T14:57:06.051 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_69295/sparse1 sparse 2024-08-29T14:57:06.116 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-08-29T14:57:06.117 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:57:06.117 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --order is deprecated, use --object-size 2024-08-29T14:57:06.120 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects sparse 2024-08-29T14:57:06.120 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse 2024-08-29T14:57:06.121 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse 2024-08-29T14:57:06.121 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep block_name_prefix 2024-08-29T14:57:06.121 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk '{print $NF;}' 2024-08-29T14:57:06.164 INFO:tasks.workunit.client.0.smithi027.stderr:++ prefix=rbd_data.6287cb3ee517 2024-08-29T14:57:06.165 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep rbd_data.6287cb3ee517 2024-08-29T14:57:06.165 INFO:tasks.workunit.client.0.smithi027.stderr:++++ get_image_data_pool sparse 2024-08-29T14:57:06.165 INFO:tasks.workunit.client.0.smithi027.stderr:++++ image=sparse 2024-08-29T14:57:06.165 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sed -e 's/rbd_data.6287cb3ee517\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-29T14:57:06.166 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sort -u 2024-08-29T14:57:06.166 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ rbd info sparse 2024-08-29T14:57:06.166 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ grep 'data_pool: ' 2024-08-29T14:57:06.166 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ awk -F: '{ print $NF }' 2024-08-29T14:57:06.210 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool= 2024-08-29T14:57:06.210 INFO:tasks.workunit.client.0.smithi027.stderr:++++ '[' -z ']' 2024-08-29T14:57:06.210 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool=rbd 2024-08-29T14:57:06.210 INFO:tasks.workunit.client.0.smithi027.stderr:++++ echo rbd 2024-08-29T14:57:06.210 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rados ls -p rbd 2024-08-29T14:57:06.249 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects=1 2024-08-29T14:57:06.249 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo 1 2024-08-29T14:57:06.250 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-08-29T14:57:06.250 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse 2024-08-29T14:57:06.336 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-08-29T14:57:06.339 INFO:tasks.workunit.client.0.smithi027.stdout:zeros import through stdin to sparse image 2024-08-29T14:57:06.340 INFO:tasks.workunit.client.0.smithi027.stderr:+ echo 'zeros import through stdin to sparse image' 2024-08-29T14:57:06.340 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/dev/zero bs=1M count=4 2024-08-29T14:57:06.340 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd import --new-format - sparse 2024-08-29T14:57:06.394 INFO:tasks.workunit.client.0.smithi027.stderr:4+0 records in 2024-08-29T14:57:06.395 INFO:tasks.workunit.client.0.smithi027.stderr:4+0 records out 2024-08-29T14:57:06.395 INFO:tasks.workunit.client.0.smithi027.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0539827 s, 77.7 MB/s 2024-08-29T14:57:06.404 INFO:tasks.workunit.client.0.smithi027.stderr: Importing image: 100% complete...done. 2024-08-29T14:57:06.404 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:57:06.408 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects sparse 2024-08-29T14:57:06.408 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse 2024-08-29T14:57:06.408 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse 2024-08-29T14:57:06.409 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep block_name_prefix 2024-08-29T14:57:06.409 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk '{print $NF;}' 2024-08-29T14:57:06.451 INFO:tasks.workunit.client.0.smithi027.stderr:++ prefix=rbd_data.3c00a5b2cb48 2024-08-29T14:57:06.452 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep rbd_data.3c00a5b2cb48 2024-08-29T14:57:06.452 INFO:tasks.workunit.client.0.smithi027.stderr:++++ get_image_data_pool sparse 2024-08-29T14:57:06.453 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sed -e 's/rbd_data.3c00a5b2cb48\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-29T14:57:06.453 INFO:tasks.workunit.client.0.smithi027.stderr:++++ image=sparse 2024-08-29T14:57:06.453 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sort -u 2024-08-29T14:57:06.453 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ rbd info sparse 2024-08-29T14:57:06.453 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ grep 'data_pool: ' 2024-08-29T14:57:06.453 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ awk -F: '{ print $NF }' 2024-08-29T14:57:06.497 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool= 2024-08-29T14:57:06.497 INFO:tasks.workunit.client.0.smithi027.stderr:++++ '[' -z ']' 2024-08-29T14:57:06.497 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool=rbd 2024-08-29T14:57:06.497 INFO:tasks.workunit.client.0.smithi027.stderr:++++ echo rbd 2024-08-29T14:57:06.497 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rados ls -p rbd 2024-08-29T14:57:06.536 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects= 2024-08-29T14:57:06.536 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo 2024-08-29T14:57:06.536 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-08-29T14:57:06.536 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse 2024-08-29T14:57:06.621 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:57:06.625 INFO:tasks.workunit.client.0.smithi027.stdout:zeros export to sparse file 2024-08-29T14:57:06.625 INFO:tasks.workunit.client.0.smithi027.stderr:+ echo 'zeros export to sparse file' 2024-08-29T14:57:06.625 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd create --new-format sparse --size 4 2024-08-29T14:57:06.669 INFO:tasks.workunit.client.0.smithi027.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-08-29T14:57:06.673 INFO:tasks.workunit.client.0.smithi027.stderr:++ rbd info sparse 2024-08-29T14:57:06.673 INFO:tasks.workunit.client.0.smithi027.stderr:++ grep block_name_prefix 2024-08-29T14:57:06.673 INFO:tasks.workunit.client.0.smithi027.stderr:++ awk '{print $NF;}' 2024-08-29T14:57:06.715 INFO:tasks.workunit.client.0.smithi027.stderr:+ prefix=rbd_data.3c2175fe8035 2024-08-29T14:57:06.715 INFO:tasks.workunit.client.0.smithi027.stderr:+ dd if=/dev/zero bs=4M count=1 2024-08-29T14:57:06.716 INFO:tasks.workunit.client.0.smithi027.stderr:++ get_image_data_pool sparse 2024-08-29T14:57:06.716 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse 2024-08-29T14:57:06.716 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse 2024-08-29T14:57:06.716 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep 'data_pool: ' 2024-08-29T14:57:06.716 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk -F: '{ print $NF }' 2024-08-29T14:57:06.759 INFO:tasks.workunit.client.0.smithi027.stderr:++ data_pool= 2024-08-29T14:57:06.759 INFO:tasks.workunit.client.0.smithi027.stderr:++ '[' -z ']' 2024-08-29T14:57:06.759 INFO:tasks.workunit.client.0.smithi027.stderr:++ data_pool=rbd 2024-08-29T14:57:06.760 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo rbd 2024-08-29T14:57:06.760 INFO:tasks.workunit.client.0.smithi027.stderr:+ rados -p rbd put rbd_data.3c2175fe8035.000000000000 - 2024-08-29T14:57:06.796 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records in 2024-08-29T14:57:06.796 INFO:tasks.workunit.client.0.smithi027.stderr:1+0 records out 2024-08-29T14:57:06.796 INFO:tasks.workunit.client.0.smithi027.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0800516 s, 52.4 MB/s 2024-08-29T14:57:06.815 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects sparse 2024-08-29T14:57:06.815 INFO:tasks.workunit.client.0.smithi027.stderr:++ image=sparse 2024-08-29T14:57:06.816 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rbd info sparse 2024-08-29T14:57:06.816 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep block_name_prefix 2024-08-29T14:57:06.816 INFO:tasks.workunit.client.0.smithi027.stderr:+++ awk '{print $NF;}' 2024-08-29T14:57:06.859 INFO:tasks.workunit.client.0.smithi027.stderr:++ prefix=rbd_data.3c2175fe8035 2024-08-29T14:57:06.860 INFO:tasks.workunit.client.0.smithi027.stderr:+++ grep rbd_data.3c2175fe8035 2024-08-29T14:57:06.860 INFO:tasks.workunit.client.0.smithi027.stderr:++++ get_image_data_pool sparse 2024-08-29T14:57:06.860 INFO:tasks.workunit.client.0.smithi027.stderr:++++ image=sparse 2024-08-29T14:57:06.860 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sed -e 's/rbd_data.3c2175fe8035\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-08-29T14:57:06.860 INFO:tasks.workunit.client.0.smithi027.stderr:+++ sort -u 2024-08-29T14:57:06.861 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ rbd info sparse 2024-08-29T14:57:06.861 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ grep 'data_pool: ' 2024-08-29T14:57:06.861 INFO:tasks.workunit.client.0.smithi027.stderr:+++++ awk -F: '{ print $NF }' 2024-08-29T14:57:06.883 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:06 smithi027 ceph-mon[29342]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:06.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:06 smithi027 ceph-mon[18892]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:06.909 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool= 2024-08-29T14:57:06.909 INFO:tasks.workunit.client.0.smithi027.stderr:++++ '[' -z ']' 2024-08-29T14:57:06.909 INFO:tasks.workunit.client.0.smithi027.stderr:++++ data_pool=rbd 2024-08-29T14:57:06.909 INFO:tasks.workunit.client.0.smithi027.stderr:++++ echo rbd 2024-08-29T14:57:06.909 INFO:tasks.workunit.client.0.smithi027.stderr:+++ rados ls -p rbd 2024-08-29T14:57:06.951 INFO:tasks.workunit.client.0.smithi027.stderr:++ objects=0 2024-08-29T14:57:06.951 INFO:tasks.workunit.client.0.smithi027.stderr:++ echo 0 2024-08-29T14:57:06.952 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-08-29T14:57:06.952 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/sparse 2024-08-29T14:57:06.952 INFO:tasks.workunit.client.0.smithi027.stderr:rm: cannot remove '/tmp/rbd_import_export_69295/sparse': No such file or directory 2024-08-29T14:57:06.952 INFO:tasks.workunit.client.0.smithi027.stderr:+ true 2024-08-29T14:57:06.953 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd export sparse /tmp/rbd_import_export_69295/sparse 2024-08-29T14:57:06.995 INFO:tasks.workunit.client.0.smithi027.stderr: Exporting image: 100% complete...done. 2024-08-29T14:57:06.999 INFO:tasks.workunit.client.0.smithi027.stderr:++ stat /tmp/rbd_import_export_69295/sparse --format=%b 2024-08-29T14:57:07.000 INFO:tasks.workunit.client.0.smithi027.stderr:+ '[' 0 = 0 ']' 2024-08-29T14:57:07.001 INFO:tasks.workunit.client.0.smithi027.stderr:+ rbd rm sparse 2024-08-29T14:57:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:06 smithi171 ceph-mon[22959]: pgmap v115: 73 pgs: 73 active+clean; 799 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:07.094 INFO:tasks.workunit.client.0.smithi027.stderr: Removing image: 100% complete...done. 2024-08-29T14:57:07.097 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm /tmp/rbd_import_export_69295/sparse /tmp/rbd_import_export_69295/sparse1 /tmp/rbd_import_export_69295/sparse2 /tmp/rbd_import_export_69295/sparse3 2024-08-29T14:57:07.100 INFO:tasks.workunit.client.0.smithi027.stderr:rm: cannot remove '/tmp/rbd_import_export_69295/sparse3': No such file or directory 2024-08-29T14:57:07.100 INFO:tasks.workunit.client.0.smithi027.stderr:+ true 2024-08-29T14:57:07.100 INFO:tasks.workunit.client.0.smithi027.stderr:+ echo OK 2024-08-29T14:57:07.100 INFO:tasks.workunit.client.0.smithi027.stdout:OK 2024-08-29T14:57:07.101 INFO:tasks.workunit.client.0.smithi027.stderr:+ rm -rf /tmp/rbd_import_export_69295 2024-08-29T14:57:07.101 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-08-29T14:57:07.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-29T14:57:07.173 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-08-29T14:57:07.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-29T14:57:07.479 DEBUG:teuthology.parallel:result is None 2024-08-29T14:57:07.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-08-29T14:57:07.510 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-08-29T14:57:07.510 DEBUG:teuthology.orchestra.run.smithi027:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-08-29T14:57:07.566 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-08-29T14:57:07.566 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-29T14:57:07.566 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-08-29T14:57:07.567 DEBUG:teuthology.parallel:result is None 2024-08-29T14:57:07.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:57:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:57:07] "GET /metrics HTTP/1.1" 200 33731 "" "Prometheus/2.43.0" 2024-08-29T14:57:09.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:08 smithi171 ceph-mon[22959]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 383 op/s 2024-08-29T14:57:09.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:08 smithi027 ceph-mon[29342]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 383 op/s 2024-08-29T14:57:09.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:08 smithi027 ceph-mon[18892]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 383 op/s 2024-08-29T14:57:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:10 smithi171 ceph-mon[22959]: osdmap e87: 8 total, 8 up, 8 in 2024-08-29T14:57:10.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:10 smithi027 ceph-mon[18892]: osdmap e87: 8 total, 8 up, 8 in 2024-08-29T14:57:10.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:10 smithi027 ceph-mon[29342]: osdmap e87: 8 total, 8 up, 8 in 2024-08-29T14:57:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:11 smithi171 ceph-mon[22959]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 381 op/s 2024-08-29T14:57:11.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:11 smithi027 ceph-mon[18892]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 381 op/s 2024-08-29T14:57:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:11 smithi027 ceph-mon[29342]: pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 381 op/s 2024-08-29T14:57:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:13 smithi171 ceph-mon[22959]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 340 op/s 2024-08-29T14:57:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:13 smithi027 ceph-mon[18892]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 340 op/s 2024-08-29T14:57:13.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:13 smithi027 ceph-mon[29342]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 340 op/s 2024-08-29T14:57:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:15 smithi171 ceph-mon[22959]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 898 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2024-08-29T14:57:15.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:15 smithi027 ceph-mon[29342]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 898 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2024-08-29T14:57:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:15 smithi027 ceph-mon[18892]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 898 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2024-08-29T14:57:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:16 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:16.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:16 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:16.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:16 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:17 smithi171 ceph-mon[22959]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-08-29T14:57:17.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:17 smithi027 ceph-mon[18892]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-08-29T14:57:17.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:17 smithi027 ceph-mon[29342]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-08-29T14:57:17.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:57:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:57:17] "GET /metrics HTTP/1.1" 200 33743 "" "Prometheus/2.43.0" 2024-08-29T14:57:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:19 smithi171 ceph-mon[22959]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 411 KiB/s wr, 42 op/s 2024-08-29T14:57:19.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:19 smithi027 ceph-mon[18892]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 411 KiB/s wr, 42 op/s 2024-08-29T14:57:19.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:19 smithi027 ceph-mon[29342]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 411 KiB/s wr, 42 op/s 2024-08-29T14:57:21.730 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:57:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:21 smithi171 ceph-mon[22959]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 407 KiB/s wr, 41 op/s 2024-08-29T14:57:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:21 smithi027 ceph-mon[18892]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 407 KiB/s wr, 41 op/s 2024-08-29T14:57:21.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:21 smithi027 ceph-mon[29342]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 407 KiB/s wr, 41 op/s 2024-08-29T14:57:22.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-29T14:57:22.514 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T14:57:23.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[18892]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 342 KiB/s wr, 35 op/s 2024-08-29T14:57:23.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[18892]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:57:23.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:23.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:57:23.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[29342]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 342 KiB/s wr, 35 op/s 2024-08-29T14:57:23.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[29342]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:57:23.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:23.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:22 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:57:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:22 smithi171 ceph-mon[22959]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 342 KiB/s wr, 35 op/s 2024-08-29T14:57:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:22 smithi171 ceph-mon[22959]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:57:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:22 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:22 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:57:24.292 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (117s) 55s ago 2m 16.3M - 0.25.0 c8568f914cd2 2e3fd226c647 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (2m) 76s ago 2m 84.2M - 9.4.7 954c08fa6188 f705829982dc 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (100s) 55s ago 100s 14.4M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (98s) 76s ago 97s 13.0M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,8765 running (6m) 76s ago 6m 440M - 18.2.4-774-gfe43f1c9 96384990183a fc352c65c32f 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283,8765 running (8m) 55s ago 8m 483M - 18.2.4-774-gfe43f1c9 96384990183a b6d2336620e1 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (8m) 55s ago 8m 44.8M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T14:57:24.850 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (7m) 76s ago 7m 40.6M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (7m) 55s ago 7m 37.0M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 55s ago 2m 15.6M - 1.5.0 0da6a335fe13 8caf48017aa9 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (2m) 76s ago 2m 8380k - 1.5.0 0da6a335fe13 91bd04afdd9c 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (6m) 55s ago 6m 44.2M 4016M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (6m) 55s ago 6m 43.0M 4016M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (5m) 55s ago 5m 49.5M 4016M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (5m) 55s ago 5m 40.3M 4016M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (4m) 76s ago 4m 49.8M 4016M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (4m) 76s ago 4m 40.8M 4016M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (4m) 76s ago 4m 40.1M 4016M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (3m) 76s ago 3m 41.0M 4016M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T14:57:24.851 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (111s) 76s ago 3m 35.7M - 2.43.0 a07b618ecd1d b057ad21b404 2024-08-29T14:57:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:24 smithi171 ceph-mon[22959]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:24 smithi171 ceph-mon[22959]: from='client.25349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:24 smithi027 ceph-mon[18892]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:24 smithi027 ceph-mon[18892]: from='client.25349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:24 smithi027 ceph-mon[29342]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:24 smithi027 ceph-mon[29342]: from='client.25349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:25.210 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:25.211 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T14:57:25.212 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 15 2024-08-29T14:57:25.212 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T14:57:25.212 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:57:25.537 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "", 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 image", 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T14:57:25.538 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:57:26.436 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/2602539248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:57:26.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[29342]: from='client.25373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/2602539248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:57:26.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:26 smithi027 ceph-mon[18892]: from='client.25373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='client.15459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:57:26.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:57:26.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:26.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2602539248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:57:26.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T14:57:26.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:26 smithi171 ceph-mon[22959]: from='client.25373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:27 smithi171 ceph-mon[22959]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:27 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:27 smithi171 ceph-mon[22959]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (unknown) 2024-08-29T14:57:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:27 smithi171 ceph-mon[22959]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T14:57:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:27 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:57:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:27 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:27.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:27 smithi171 ceph-mon[22959]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-29T14:57:27.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:57:27] "GET /metrics HTTP/1.1" 200 33743 "" "Prometheus/2.43.0" 2024-08-29T14:57:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[18892]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[18892]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (unknown) 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[18892]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[18892]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[29342]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[29342]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (unknown) 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[29342]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T14:57:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:57:27.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:57:27.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:27 smithi027 ceph-mon[29342]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-29T14:57:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:28 smithi171 ceph-mon[22959]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc on smithi171 2024-08-29T14:57:28.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:28 smithi027 ceph-mon[18892]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc on smithi171 2024-08-29T14:57:28.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:28 smithi027 ceph-mon[29342]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc on smithi171 2024-08-29T14:57:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:29 smithi171 ceph-mon[22959]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:29 smithi027 ceph-mon[18892]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:29.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:29 smithi027 ceph-mon[29342]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:31 smithi171 ceph-mon[22959]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:31 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:31 smithi027 ceph-mon[18892]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:31 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:31 smithi027 ceph-mon[29342]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:31.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:31 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:33 smithi171 ceph-mon[22959]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:33.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:33 smithi027 ceph-mon[18892]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:33.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:33 smithi027 ceph-mon[29342]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:35 smithi171 ceph-mon[22959]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:35.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:35 smithi027 ceph-mon[18892]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:35 smithi027 ceph-mon[29342]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:37 smithi171 ceph-mon[22959]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:37 smithi027 ceph-mon[18892]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:37 smithi027 ceph-mon[29342]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:37.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:57:37 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:57:37] "GET /metrics HTTP/1.1" 200 33743 "" "Prometheus/2.43.0" 2024-08-29T14:57:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:39 smithi171 ceph-mon[22959]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:39.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:39 smithi027 ceph-mon[29342]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:39.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:39 smithi027 ceph-mon[18892]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:41 smithi171 ceph-mon[22959]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:41 smithi027 ceph-mon[18892]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:41.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:41 smithi027 ceph-mon[29342]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:43 smithi171 ceph-mon[22959]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:43 smithi027 ceph-mon[29342]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:43 smithi027 ceph-mon[18892]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:45 smithi171 ceph-mon[22959]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:45 smithi027 ceph-mon[18892]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:45.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:45 smithi027 ceph-mon[29342]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:46 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:46.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:46 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:46 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:57:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:47 smithi027 ceph-mon[18892]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:47 smithi027 ceph-mon[29342]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:47.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:57:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:57:47] "GET /metrics HTTP/1.1" 200 33735 "" "Prometheus/2.43.0" 2024-08-29T14:57:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:47 smithi171 ceph-mon[22959]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:49.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:49 smithi027 ceph-mon[18892]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:49.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:49 smithi027 ceph-mon[29342]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:49.819 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:49 smithi171 ceph-mon[22959]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:51.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:51 smithi027 ceph-mon[18892]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:51.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:51 smithi027 ceph-mon[29342]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:52.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:51 smithi171 ceph-mon[22959]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:53.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:53 smithi027 ceph-mon[18892]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:53.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:53 smithi027 ceph-mon[29342]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:53 smithi171 ceph-mon[22959]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:55.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:55 smithi027 ceph-mon[29342]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:55 smithi027 ceph-mon[18892]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:55.848 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T14:57:56.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:55 smithi171 ceph-mon[22959]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:56.404 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T14:57:56.404 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (2m) 86s ago 3m 16.3M - 0.25.0 c8568f914cd2 2e3fd226c647 2024-08-29T14:57:56.404 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (3m) 31s ago 3m 88.3M - 9.4.7 954c08fa6188 f705829982dc 2024-08-29T14:57:56.404 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (2m) 86s ago 2m 14.4M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T14:57:56.404 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (2m) 31s ago 2m 13.3M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,8765 running (7m) 31s ago 7m 441M - 18.2.4-774-gfe43f1c9 96384990183a fc352c65c32f 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283,8765 running (9m) 86s ago 9m 483M - 18.2.4-774-gfe43f1c9 96384990183a b6d2336620e1 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (9m) 86s ago 9m 44.8M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (7m) 31s ago 7m 42.9M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (7m) 86s ago 7m 37.0M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 86s ago 2m 15.6M - 1.5.0 0da6a335fe13 8caf48017aa9 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (2m) 31s ago 2m 16.6M - 1.5.0 0da6a335fe13 91bd04afdd9c 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (7m) 86s ago 7m 44.2M 4016M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (6m) 86s ago 6m 43.0M 4016M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (6m) 86s ago 6m 49.5M 4016M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (5m) 86s ago 5m 40.3M 4016M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (5m) 31s ago 5m 52.8M 4016M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T14:57:56.405 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (5m) 31s ago 5m 45.4M 4016M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T14:57:56.406 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (4m) 31s ago 4m 44.1M 4016M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T14:57:56.406 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (4m) 31s ago 4m 47.9M 4016M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T14:57:56.406 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (2m) 31s ago 3m 43.1M - 2.43.0 a07b618ecd1d b057ad21b404 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 15 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T14:57:56.743 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:57:57.023 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:57:57.023 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T14:57:57.023 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T14:57:57.024 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-29T14:57:57.024 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T14:57:57.024 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/2 daemons upgraded", 2024-08-29T14:57:57.024 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc image on host smithi171", 2024-08-29T14:57:57.024 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T14:57:57.024 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:57:57.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:57:57] "GET /metrics HTTP/1.1" 200 33735 "" "Prometheus/2.43.0" 2024-08-29T14:57:57.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[18892]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[18892]: from='client.25379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[18892]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[18892]: from='client.25391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3143819891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[29342]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[29342]: from='client.25379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[29342]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[29342]: from='client.25391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:57.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:57 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3143819891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:57:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:57 smithi171 ceph-mon[22959]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:57 smithi171 ceph-mon[22959]: from='client.25379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:57 smithi171 ceph-mon[22959]: from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:57 smithi171 ceph-mon[22959]: from='client.25391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:57 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3143819891' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:57:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:58 smithi027 ceph-mon[18892]: from='client.25403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:58 smithi027 ceph-mon[29342]: from='client.25403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:58 smithi171 ceph-mon[22959]: from='client.25403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:57:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:57:59 smithi027 ceph-mon[18892]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:57:59.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:57:59 smithi027 ceph-mon[29342]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:57:59 smithi171 ceph-mon[22959]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:01 smithi027 ceph-mon[18892]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:01 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:01 smithi027 ceph-mon[29342]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:01 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:01.975 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:01 smithi171 ceph-mon[22959]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:01.975 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:01 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:03.704 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:03 smithi171 ceph-mon[22959]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:03.704 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:03 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:03.704 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:03 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:03.705 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:03 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:58:03.705 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:03 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[18892]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:58:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:03.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[29342]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:03.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:03.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:03.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:58:03.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:03 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:04.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:04 smithi171 ceph-mon[22959]: Upgrade: Updating mgr.x 2024-08-29T14:58:04.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:04 smithi171 ceph-mon[22959]: Deploying daemon mgr.x on smithi171 2024-08-29T14:58:04.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:04 smithi027 ceph-mon[18892]: Upgrade: Updating mgr.x 2024-08-29T14:58:04.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:04 smithi027 ceph-mon[18892]: Deploying daemon mgr.x on smithi171 2024-08-29T14:58:04.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:04 smithi027 ceph-mon[29342]: Upgrade: Updating mgr.x 2024-08-29T14:58:04.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:04 smithi027 ceph-mon[29342]: Deploying daemon mgr.x on smithi171 2024-08-29T14:58:04.984 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:04 smithi171 systemd[1]: Stopping Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:58:05.266 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:04 smithi171 podman[56248]: 2024-08-29 14:58:04.983793609 +0000 UTC m=+0.121374995 container died fc352c65c32f4eaf7cc7d6c94a3f8f6c58418e76d1562cd561c21c44dbad73ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, GIT_CLEAN=True, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.1) 2024-08-29T14:58:05.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:05 smithi171 podman[56248]: 2024-08-29 14:58:05.347424534 +0000 UTC m=+0.485005958 container cleanup fc352c65c32f4eaf7cc7d6c94a3f8f6c58418e76d1562cd561c21c44dbad73ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-08-29T14:58:05.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:05 smithi171 bash[56248]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x 2024-08-29T14:58:05.809 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:05 smithi171 ceph-mon[22959]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:05.809 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:05 smithi171 podman[56260]: 2024-08-29 14:58:05.674243279 +0000 UTC m=+0.680744416 container remove fc352c65c32f4eaf7cc7d6c94a3f8f6c58418e76d1562cd561c21c44dbad73ab (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, RELEASE=reef-fe43f1c, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-29T14:58:05.809 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:05 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-29T14:58:05.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:05 smithi027 ceph-mon[18892]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:05.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:05 smithi027 ceph-mon[29342]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:06.236 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:05 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-29T14:58:06.236 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:05 smithi171 systemd[1]: Stopped Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:58:06.236 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:05 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service: Consumed 10.968s CPU time. 2024-08-29T14:58:06.236 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 systemd[1]: Starting Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:58:06.495 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 podman[56389]: 2024-08-29 14:58:06.138193116 +0000 UTC m=+0.015591371 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T14:58:06.496 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 podman[56389]: 2024-08-29 14:58:06.250806649 +0000 UTC m=+0.128204890 container create a8bc5f23457c5ef9da18b01854871fb18ffe285901a4e89df466685839896ba3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-29T14:58:06.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 podman[56389]: 2024-08-29 14:58:06.495496611 +0000 UTC m=+0.372894852 container init a8bc5f23457c5ef9da18b01854871fb18ffe285901a4e89df466685839896ba3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_BRANCH=HEAD) 2024-08-29T14:58:06.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 podman[56389]: 2024-08-29 14:58:06.498450588 +0000 UTC m=+0.375848840 container start a8bc5f23457c5ef9da18b01854871fb18ffe285901a4e89df466685839896ba3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-29T14:58:06.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 ceph-mgr[56408]: -- 172.21.15.171:0/3077923929 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55da281e30e0 con 0x55da281c1000 2024-08-29T14:58:06.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 bash[56389]: a8bc5f23457c5ef9da18b01854871fb18ffe285901a4e89df466685839896ba3 2024-08-29T14:58:06.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:06 smithi171 systemd[1]: Started Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:58:07.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:58:07] "GET /metrics HTTP/1.1" 200 33744 "" "Prometheus/2.43.0" 2024-08-29T14:58:07.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[29342]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:07.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:07.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:07.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:07.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[18892]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:07.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:07.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:07.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:07 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:07 smithi171 ceph-mon[22959]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:07 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:08.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:07 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:08.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:07 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:09.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:09 smithi027 ceph-mon[18892]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:09.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:09 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:09.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:09 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:09.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:09 smithi027 ceph-mon[29342]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:09.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:09 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:09.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:09 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:09.823 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:09 smithi171 ceph-mon[22959]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:09.824 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:09 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:09.824 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:09 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[29342]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[18892]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:11 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.847 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-mon[22959]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:11.847 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.847 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.847 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:11.847 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:12.167 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:11] ENGINE Bus STARTING 2024-08-29T14:58:12.167 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: CherryPy Checker: 2024-08-29T14:58:12.167 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: The Application mounted at '' has an empty config. 2024-08-29T14:58:12.167 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:12.167 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:11] ENGINE Serving on http://:::9283 2024-08-29T14:58:12.167 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:11] ENGINE Bus STARTED 2024-08-29T14:58:12.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[18892]: Standby manager daemon x restarted 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[18892]: Standby manager daemon x started 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[29342]: Standby manager daemon x restarted 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[29342]: Standby manager daemon x started 2024-08-29T14:58:12.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:58:12.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:12.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:58:12.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:12 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:12.850 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:12 smithi171 ceph-mon[22959]: Standby manager daemon x restarted 2024-08-29T14:58:12.850 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:12 smithi171 ceph-mon[22959]: Standby manager daemon x started 2024-08-29T14:58:12.850 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:12 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:58:12.851 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:12 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:12.851 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:12 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:58:12.851 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:12 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:13 smithi027 ceph-mon[18892]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:13 smithi027 ceph-mon[18892]: mgrmap e22: y(active, since 3m), standbys: x 2024-08-29T14:58:13.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:13 smithi027 ceph-mon[29342]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:13 smithi027 ceph-mon[29342]: mgrmap e22: y(active, since 3m), standbys: x 2024-08-29T14:58:13.857 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:13 smithi171 ceph-mon[22959]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:13.857 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:13 smithi171 ceph-mon[22959]: mgrmap e22: y(active, since 3m), standbys: x 2024-08-29T14:58:15.004 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:14] ENGINE Bus STOPPING 2024-08-29T14:58:15.005 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:58:15.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:14.783+0000 7f9761446640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-29T14:58:15.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setuser ceph since I am not root 2024-08-29T14:58:15.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ignoring --setgroup ceph since I am not root 2024-08-29T14:58:15.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:14.940+0000 7f595224c040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-29T14:58:15.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:15.004+0000 7f595224c040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-29T14:58:15.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:15] ENGINE Bus STOPPED 2024-08-29T14:58:15.528 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:15.159+0000 7f595224c040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-29T14:58:15.528 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:15.228+0000 7f595224c040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-29T14:58:15.528 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:15.380+0000 7f595224c040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:58:15.528 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:15.526+0000 7f595224c040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-29T14:58:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: Activating manager daemon x 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:58:15.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: osdmap e88: 8 total, 8 up, 8 in 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: mgrmap e23: x(active, starting, since 0.011855s) 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:58:15.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:58:15.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:58:15.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:58:15.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:58:15.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[18892]: Manager daemon x is now available 2024-08-29T14:58:15.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:15.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: Activating manager daemon x 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:58:15.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: osdmap e88: 8 total, 8 up, 8 in 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: mgrmap e23: x(active, starting, since 0.011855s) 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:58:15.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-mon[29342]: Manager daemon x is now available 2024-08-29T14:58:15.843 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: Activating manager daemon x 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:58:15.844 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: osdmap e88: 8 total, 8 up, 8 in 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.14403 172.21.15.27:0/1636176785' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: mgrmap e23: x(active, starting, since 0.011855s) 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:58:15.845 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:58:15.846 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:58:15.846 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:58:15.846 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:58:15.846 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-mon[22959]: Manager daemon x is now available 2024-08-29T14:58:16.122 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:15.861+0000 7f595224c040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-29T14:58:16.122 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:15.926+0000 7f595224c040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-29T14:58:16.123 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:16.121+0000 7f595224c040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-29T14:58:16.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:15] ENGINE Bus STARTING 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.888+0000 7f209e2f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.888+0000 7f209e2f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.888+0000 7f209e2f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.888+0000 7f209e2f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.888+0000 7f209e2f7640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.889+0000 7f20a0afc640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.890+0000 7f20a0afc640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.890+0000 7f20a0afc640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.890+0000 7f20a0afc640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:15.890+0000 7f20a0afc640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:15] ENGINE Serving on http://:::9283 2024-08-29T14:58:16.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:15 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:15] ENGINE Bus STARTED 2024-08-29T14:58:17.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: Checking for cert/key for grafana.a 2024-08-29T14:58:17.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: Migrating grafana.a cert to cert store 2024-08-29T14:58:17.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: mgrmap e24: x(active, since 1.01245s) 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: Migrating grafana.a key to cert store 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:17.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: Deploying cephadm binary to smithi027 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[29342]: Deploying cephadm binary to smithi171 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: Checking for cert/key for grafana.a 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: Migrating grafana.a cert to cert store 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: mgrmap e24: x(active, since 1.01245s) 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:17.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: Migrating grafana.a key to cert store 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: Deploying cephadm binary to smithi027 2024-08-29T14:58:17.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:16 smithi027 ceph-mon[18892]: Deploying cephadm binary to smithi171 2024-08-29T14:58:17.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: Checking for cert/key for grafana.a 2024-08-29T14:58:17.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: Migrating grafana.a cert to cert store 2024-08-29T14:58:17.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: mgrmap e24: x(active, since 1.01245s) 2024-08-29T14:58:17.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: Migrating grafana.a key to cert store 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:17.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: Deploying cephadm binary to smithi027 2024-08-29T14:58:17.132 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:16 smithi171 ceph-mon[22959]: Deploying cephadm binary to smithi171 2024-08-29T14:58:17.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:17.049+0000 7f595224c040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-29T14:58:17.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:17.118+0000 7f595224c040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-29T14:58:17.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:17.128+0000 7f2084847640 -1 log_channel(cephadm) log [ERR] : [29/Aug/2024:14:58:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:58:17.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: Traceback (most recent call last): 2024-08-29T14:58:17.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: self._connections.run(self.expiration_interval) 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: self._run(expiration_interval) 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:58:17.518 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: s = self.context.wrap_socket( 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: return self.sslsocket_class._create( 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: self.do_handshake() 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: self._sslobj.do_handshake() 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:58:17.519 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:18.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:17.874+0000 7f595224c040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-29T14:58:18.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:17.997+0000 7f595224c040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[29342]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[29342]: mgrmap e25: x(active, since 2s) 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:16] ENGINE Bus STARTING 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:16] ENGINE Serving on http://172.21.15.171:8765 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:17] ENGINE Serving on https://172.21.15.171:7150 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:17] ENGINE Bus STARTED 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:58:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:58:18.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:58:18.071 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:58:18.071 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:58:18.071 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-08-29T14:58:18.071 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:58:18.071 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:58:18.071 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:58:18.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[18892]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[18892]: mgrmap e25: x(active, since 2s) 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:16] ENGINE Bus STARTING 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:16] ENGINE Serving on http://172.21.15.171:8765 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:17] ENGINE Serving on https://172.21.15.171:7150 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:17] ENGINE Bus STARTED 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:17 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:58:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:58:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:58:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-mon[22959]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-mon[22959]: mgrmap e25: x(active, since 2s) 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:16] ENGINE Bus STARTING 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:16] ENGINE Serving on http://172.21.15.171:8765 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:17] ENGINE Serving on https://172.21.15.171:7150 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:17] ENGINE Bus STARTED 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:17 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:17] ENGINE Error in HTTPServer.serve 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: Traceback (most recent call last): 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._run(expiration_interval) 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:58:18.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: s = self.context.wrap_socket( 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: return self.sslsocket_class._create( 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: self.do_handshake() 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._sslobj.do_handshake() 2024-08-29T14:58:18.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:58:18.462 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.156+0000 7f595224c040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-29T14:58:18.462 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.229+0000 7f595224c040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-29T14:58:18.463 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.388+0000 7f595224c040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-29T14:58:18.463 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.461+0000 7f595224c040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-29T14:58:18.717 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.526+0000 7f595224c040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-29T14:58:18.718 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.694+0000 7f595224c040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-29T14:58:18.993 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.763+0000 7f595224c040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-29T14:58:18.993 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.837+0000 7f595224c040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-29T14:58:18.993 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.925+0000 7f595224c040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-29T14:58:18.993 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:18.991+0000 7f595224c040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-29T14:58:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:19.058+0000 7f595224c040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-29T14:58:20.037 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-mon[29342]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:20.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-mon[29342]: mgrmap e26: x(active, since 4s) 2024-08-29T14:58:20.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-mon[18892]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:20.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-mon[18892]: mgrmap e26: x(active, since 4s) 2024-08-29T14:58:20.038 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:19.930+0000 7f595224c040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-29T14:58:20.038 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:58:19] ENGINE Bus STARTING 2024-08-29T14:58:20.038 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: CherryPy Checker: 2024-08-29T14:58:20.039 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: The Application mounted at '' has an empty config. 2024-08-29T14:58:20.039 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: 2024-08-29T14:58:20.039 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:58:20] ENGINE Serving on http://:::9283 2024-08-29T14:58:20.039 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: [29/Aug/2024:14:58:20] ENGINE Bus STARTED 2024-08-29T14:58:20.098 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:19 smithi171 ceph-mon[22959]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:20.098 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:19 smithi171 ceph-mon[22959]: mgrmap e26: x(active, since 4s) 2024-08-29T14:58:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:20 smithi171 ceph-mon[22959]: Standby manager daemon y started 2024-08-29T14:58:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:20 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-29T14:58:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:20 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:20 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-29T14:58:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:20 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:20 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:21.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:20 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:21.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[18892]: Standby manager daemon y started 2024-08-29T14:58:21.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-29T14:58:21.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:21.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-29T14:58:21.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:21.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:21.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:21.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[29342]: Standby manager daemon y started 2024-08-29T14:58:21.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-29T14:58:21.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:21.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-29T14:58:21.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/1115030693' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:21.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:21.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:20 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: mgrmap e27: x(active, since 6s), standbys: y 2024-08-29T14:58:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:58:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi027 to 2480M 2024-08-29T14:58:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:21 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: mgrmap e27: x(active, since 6s), standbys: y 2024-08-29T14:58:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:58:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi027 to 2480M 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: mgrmap e27: x(active, since 6s), standbys: y 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:58:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi027 to 2480M 2024-08-29T14:58:22.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:21 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.165 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:22 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.165 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:22 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.165 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:22 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.165 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:22 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:22 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:24.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:23 smithi171 ceph-mon[22959]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 10 op/s 2024-08-29T14:58:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:23 smithi027 ceph-mon[29342]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 10 op/s 2024-08-29T14:58:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:23 smithi027 ceph-mon[18892]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 10 op/s 2024-08-29T14:58:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-08-29T14:58:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:25 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-08-29T14:58:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 8 op/s 2024-08-29T14:58:26.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:26.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:25 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: Adjusting osd_memory_target on smithi171 to 2480M 2024-08-29T14:58:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:58:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:58:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:58:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:58:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:58:27.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:58:27.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:26 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: Adjusting osd_memory_target on smithi171 to 2480M 2024-08-29T14:58:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:58:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:58:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:58:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:58:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:58:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: Adjusting osd_memory_target on smithi171 to 2480M 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:58:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:58:27.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:26 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:27.335 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T14:58:27.917 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T14:58:27.917 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 7s ago 3m 24.9M - 0.25.0 c8568f914cd2 2e3fd226c647 2024-08-29T14:58:27.917 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (3m) 6s ago 3m 88.6M - 9.4.7 954c08fa6188 f705829982dc 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (2m) 7s ago 2m 15.9M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (2m) 6s ago 2m 14.0M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (21s) 6s ago 7m 497M - 19.3.0-4579-g1f4056d9 d17246d170ec a8bc5f23457c 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283,8765 running (9m) 7s ago 9m 299M - 18.2.4-774-gfe43f1c9 96384990183a b6d2336620e1 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (9m) 7s ago 9m 51.0M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (8m) 6s ago 8m 44.0M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (8m) 7s ago 8m 39.2M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (3m) 7s ago 3m 16.2M - 1.5.0 0da6a335fe13 8caf48017aa9 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (3m) 6s ago 3m 16.3M - 1.5.0 0da6a335fe13 91bd04afdd9c 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (7m) 7s ago 7m 50.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (7m) 7s ago 7m 51.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (6m) 7s ago 6m 56.2M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (6m) 7s ago 6m 45.1M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T14:58:27.918 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (6m) 6s ago 6m 53.5M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T14:58:27.919 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (5m) 6s ago 5m 46.1M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T14:58:27.919 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (5m) 6s ago 5m 44.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T14:58:27.919 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (4m) 6s ago 4m 48.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T14:58:27.919 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (2m) 6s ago 4m 52.4M - 2.43.0 a07b618ecd1d b057ad21b404 2024-08-29T14:58:27.919 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:58:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-29T14:58:28.266 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 1, 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 1 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T14:58:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T14:58:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T14:58:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 14, 2024-08-29T14:58:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 1 2024-08-29T14:58:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T14:58:28.268 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:58:28.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:58:28.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[29342]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[29342]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[29342]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[29342]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[18892]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[18892]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:58:28.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[18892]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:58:28.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:27 smithi027 ceph-mon[18892]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:28.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:27 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:58:28.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:27 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:58:28.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:27 smithi171 ceph-mon[22959]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 6 op/s 2024-08-29T14:58:28.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:27 smithi171 ceph-mon[22959]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:58:28.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:27 smithi171 ceph-mon[22959]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:58:28.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:27 smithi171 ceph-mon[22959]: from='client.15561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "1/2 daemons upgraded", 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T14:58:28.566 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:58:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:28 smithi171 ceph-mon[22959]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:28 smithi171 ceph-mon[22959]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:28 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3230240054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:58:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:28 smithi027 ceph-mon[18892]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:28 smithi027 ceph-mon[18892]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:28 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3230240054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:58:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:28 smithi027 ceph-mon[29342]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:28 smithi027 ceph-mon[29342]: from='client.15573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:28 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3230240054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:58:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:30 smithi171 ceph-mon[22959]: from='client.25472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:30 smithi171 ceph-mon[22959]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:30 smithi027 ceph-mon[18892]: from='client.25472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:30 smithi027 ceph-mon[18892]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:30.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:30 smithi027 ceph-mon[29342]: from='client.25472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:58:30.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:30 smithi027 ceph-mon[29342]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:31 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:31 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:31 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:32.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:32 smithi171 ceph-mon[22959]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:32 smithi027 ceph-mon[18892]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:32 smithi027 ceph-mon[29342]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:34.512 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:34.088Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T14:58:34.513 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:34.090Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T14:58:34.513 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:34.090Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T14:58:34.513 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:34.092Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T14:58:34.513 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:34.092Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T14:58:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:34 smithi171 ceph-mon[22959]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:34.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:34 smithi027 ceph-mon[18892]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:34.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:34 smithi027 ceph-mon[29342]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 6 op/s 2024-08-29T14:58:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:36 smithi171 ceph-mon[22959]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:36.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:36 smithi027 ceph-mon[18892]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:36 smithi027 ceph-mon[29342]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:38.000 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:37 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[19117]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:58:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-29T14:58:38.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:38 smithi171 ceph-mon[22959]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:38.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:38 smithi027 ceph-mon[29342]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:38 smithi027 ceph-mon[18892]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:40 smithi027 ceph-mon[18892]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:40.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:40 smithi027 ceph-mon[29342]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:40 smithi171 ceph-mon[22959]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:42.166 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 systemd[1]: Stopping Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:58:42.728 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-mon[22959]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:42.728 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-29T14:58:42.728 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-29T14:58:42.728 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-29T14:58:42.728 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.435Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.436Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.436Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.436Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[53138]: ts=2024-08-29T14:58:42.436Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-29T14:58:42.729 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 podman[64909]: 2024-08-29 14:58:42.446096896 +0000 UTC m=+0.114091901 container died b057ad21b4045311f746be3b041c3ddeef92c6d39dc6369e3bd40be5a36a3edf (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:58:42.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:42 smithi027 ceph-mon[18892]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:42 smithi027 ceph-mon[29342]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:43.017 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 podman[64909]: 2024-08-29 14:58:42.919868859 +0000 UTC m=+0.587863914 container cleanup b057ad21b4045311f746be3b041c3ddeef92c6d39dc6369e3bd40be5a36a3edf (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:58:43.017 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:42 smithi171 bash[64909]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a 2024-08-29T14:58:43.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 podman[64920]: 2024-08-29 14:58:43.196538648 +0000 UTC m=+0.752473317 container remove b057ad21b4045311f746be3b041c3ddeef92c6d39dc6369e3bd40be5a36a3edf (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:58:43.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-29T14:58:43.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 systemd[1]: Stopped Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:58:43.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.661s CPU time. 2024-08-29T14:58:43.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 systemd[1]: Starting Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:58:43.810 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 podman[65016]: 2024-08-29 14:58:43.45826688 +0000 UTC m=+0.017253451 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-29T14:58:43.810 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 podman[65016]: 2024-08-29 14:58:43.580851548 +0000 UTC m=+0.139838160 container create 2bad38ac27606468d5f0b3ae855b3825aa8c1f697d076f8680e242c10cbab9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:58:43.810 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 podman[65016]: 2024-08-29 14:58:43.782639286 +0000 UTC m=+0.341625855 container init 2bad38ac27606468d5f0b3ae855b3825aa8c1f697d076f8680e242c10cbab9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:58:43.810 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 podman[65016]: 2024-08-29 14:58:43.785340438 +0000 UTC m=+0.344327005 container start 2bad38ac27606468d5f0b3ae855b3825aa8c1f697d076f8680e242c10cbab9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:58:44.123 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:43] ENGINE Bus STOPPING 2024-08-29T14:58:44.123 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:58:44.123 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:44] ENGINE Bus STOPPED 2024-08-29T14:58:44.123 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:44] ENGINE Bus STARTING 2024-08-29T14:58:44.123 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:44] ENGINE Serving on http://:::9283 2024-08-29T14:58:44.123 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:58:44] ENGINE Bus STARTED 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.810Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.810Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.810Z caller=main.go:570 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi171 (none))" 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.810Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.810Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.811Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.812Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.813Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.813Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.815Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-29T14:58:44.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.815Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.22µs 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.815Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.816Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.823Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.824Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.824Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.19µs wal_replay_duration=8.755121ms wbl_replay_duration=116ns total_replay_duration=8.809865ms 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.825Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.825Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.825Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.844Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.328864ms db_storage=867ns remote_storage=1.168µs web_handler=445ns query_engine=530ns scrape=421.983µs scrape_sd=129.022µs notify=15.712µs notify_sd=30.44µs rules=18.346655ms tracing=5.647µs 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.845Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:58:43.845Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-29T14:58:44.125 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 bash[65016]: 2bad38ac27606468d5f0b3ae855b3825aa8c1f697d076f8680e242c10cbab9c1 2024-08-29T14:58:44.126 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:58:43 smithi171 systemd[1]: Started Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:58:44.623 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:44.623 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:58:44.624 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-08-29T14:58:44.625 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.625 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:44.625 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:44 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:44.898 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:44.898 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.898 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.898 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:58:44.898 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-08-29T14:58:44.899 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:44.899 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:44.899 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:44 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:58:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-08-29T14:58:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-29T14:58:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-29T14:58:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:45.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:45.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:45.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:58:45.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:45 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi171:9095"}]: dispatch 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:58:46.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:58:46.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:45 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:46.803 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[29342]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:46.804 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[29342]: Upgrade: Updating mgr.y 2024-08-29T14:58:46.804 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[29342]: Deploying daemon mgr.y on smithi027 2024-08-29T14:58:46.804 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:46.804 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[18892]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:46.804 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[18892]: Upgrade: Updating mgr.y 2024-08-29T14:58:46.804 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[18892]: Deploying daemon mgr.y on smithi027 2024-08-29T14:58:46.804 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:46 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:46 smithi171 ceph-mon[22959]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:46 smithi171 ceph-mon[22959]: Upgrade: Updating mgr.y 2024-08-29T14:58:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:46 smithi171 ceph-mon[22959]: Deploying daemon mgr.y on smithi027 2024-08-29T14:58:47.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:46 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:47.373 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:47 smithi027 systemd[1]: Stopping Ceph mgr.y for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:58:47.633 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:47 smithi027 ceph-mon[29342]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:47.633 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:47 smithi027 ceph-mon[18892]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:47.634 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:47 smithi027 podman[75364]: 2024-08-29 14:58:47.371683751 +0000 UTC m=+0.127368174 container died b6d2336620e1c9ceb45726b999f9eb7d9f0e18d2dd94316c043327cfe3371016 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-08-29T14:58:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:47 smithi171 ceph-mon[22959]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:48.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:47 smithi027 podman[75364]: 2024-08-29 14:58:47.782348381 +0000 UTC m=+0.538032842 container cleanup b6d2336620e1c9ceb45726b999f9eb7d9f0e18d2dd94316c043327cfe3371016 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-fe43f1c, GIT_BRANCH=HEAD, io.buildah.version=1.37.1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-29T14:58:48.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:47 smithi027 bash[75364]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y 2024-08-29T14:58:48.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 podman[75376]: 2024-08-29 14:58:48.134410791 +0000 UTC m=+0.762683349 container remove b6d2336620e1c9ceb45726b999f9eb7d9f0e18d2dd94316c043327cfe3371016 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, RELEASE=reef-fe43f1c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-29T14:58:48.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-08-29T14:58:48.479 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-08-29T14:58:48.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 systemd[1]: Stopped Ceph mgr.y for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:58:48.480 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service: Consumed 46.412s CPU time. 2024-08-29T14:58:48.811 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 systemd[1]: Starting Ceph mgr.y for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:58:48.811 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 podman[75505]: 2024-08-29 14:58:48.686051683 +0000 UTC m=+0.016419866 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T14:58:48.811 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:48 smithi027 podman[75505]: 2024-08-29 14:58:48.810469484 +0000 UTC m=+0.140837698 container create f766ab17b7e23ead1473deba26bb25300675dc92fa7dae5b45955b1b858ec196 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-08-29T14:58:49.061 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:49 smithi027 podman[75505]: 2024-08-29 14:58:49.046809202 +0000 UTC m=+0.377177372 container init f766ab17b7e23ead1473deba26bb25300675dc92fa7dae5b45955b1b858ec196 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.schema-version=1.0) 2024-08-29T14:58:49.061 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:49 smithi027 podman[75505]: 2024-08-29 14:58:49.050928973 +0000 UTC m=+0.381297144 container start f766ab17b7e23ead1473deba26bb25300675dc92fa7dae5b45955b1b858ec196 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-29T14:58:49.321 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:49 smithi027 ceph-mgr[75524]: -- 172.21.15.27:0/1905575506 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562cf04110e0 con 0x562cf03ee800 2024-08-29T14:58:49.321 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:49 smithi027 bash[75505]: f766ab17b7e23ead1473deba26bb25300675dc92fa7dae5b45955b1b858ec196 2024-08-29T14:58:49.321 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:49 smithi027 systemd[1]: Started Ceph mgr.y for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:58:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:50 smithi171 ceph-mon[22959]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:50 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:50 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:50 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:50.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[29342]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:50.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:50.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:50.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[18892]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:50 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:52.112 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:51 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:58:51] "GET /metrics HTTP/1.1" 200 33817 "" "Prometheus/2.43.0" 2024-08-29T14:58:52.495 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:52 smithi027 ceph-mon[29342]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:52.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:52 smithi027 ceph-mon[18892]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:52 smithi171 ceph-mon[22959]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:54.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[29342]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[18892]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:54.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:53 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:53 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:53 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:53 smithi171 ceph-mon[22959]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:53 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:53 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:53 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:53 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[18892]: Standby manager daemon y restarted 2024-08-29T14:58:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[18892]: Standby manager daemon y started 2024-08-29T14:58:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-29T14:58:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[29342]: Standby manager daemon y restarted 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[29342]: Standby manager daemon y started 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:54.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:54] ENGINE Bus STARTING 2024-08-29T14:58:54.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: CherryPy Checker: 2024-08-29T14:58:54.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: The Application mounted at '' has an empty config. 2024-08-29T14:58:54.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:54.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:54] ENGINE Serving on http://:::9283 2024-08-29T14:58:54.820 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:54 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:54] ENGINE Bus STARTED 2024-08-29T14:58:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:54 smithi171 ceph-mon[22959]: Standby manager daemon y restarted 2024-08-29T14:58:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:54 smithi171 ceph-mon[22959]: Standby manager daemon y started 2024-08-29T14:58:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:54 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-29T14:58:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:54 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:58:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:54 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-29T14:58:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:54 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:58:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:55 smithi027 ceph-mon[18892]: mgrmap e28: x(active, since 39s), standbys: y 2024-08-29T14:58:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:55 smithi027 ceph-mon[18892]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:56.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:55 smithi027 ceph-mon[29342]: mgrmap e28: x(active, since 39s), standbys: y 2024-08-29T14:58:56.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:55 smithi027 ceph-mon[29342]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:55 smithi171 ceph-mon[22959]: mgrmap e28: x(active, since 39s), standbys: y 2024-08-29T14:58:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:55 smithi171 ceph-mon[22959]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:57.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:56 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:56] ENGINE Bus STOPPING 2024-08-29T14:58:57.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:58:56.939+0000 7f21331c7640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-29T14:58:57.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: ignoring --setuser ceph since I am not root 2024-08-29T14:58:57.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: ignoring --setgroup ceph since I am not root 2024-08-29T14:58:57.268 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:58:56 smithi171 ceph-mgr[56408]: -- 172.21.15.171:0/1590020079 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fcf0f8b0e0 con 0x55fcf0f69400 2024-08-29T14:58:57.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T14:58:57.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:57] ENGINE Bus STOPPED 2024-08-29T14:58:57.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.713+0000 7f2673124640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.713+0000 7f2673124640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.713+0000 7f2673124640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.713+0000 7f2673124640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.713+0000 7f2673124640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.714+0000 7f267011e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.714+0000 7f267011e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.714+0000 7f267011e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.714+0000 7f267011e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:57.714+0000 7f267011e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T14:58:57.979 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:57] ENGINE Bus STARTING 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: Activating manager daemon y 2024-08-29T14:58:57.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: osdmap e89: 8 total, 8 up, 8 in 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.25186 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: mgrmap e29: y(active, starting, since 0.0112056s) 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:58:57.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:58:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: Manager daemon y is now available 2024-08-29T14:58:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: Activating manager daemon y 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: osdmap e89: 8 total, 8 up, 8 in 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.25186 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-29T14:58:57.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: mgrmap e29: y(active, starting, since 0.0112056s) 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:58:57.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: Manager daemon y is now available 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:57.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:57 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:58.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:58.128 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 172.21.15.171:0/3859095995' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: Activating manager daemon y 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: osdmap e89: 8 total, 8 up, 8 in 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.25186 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: mgrmap e29: y(active, starting, since 0.0112056s) 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T14:58:58.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T14:58:58.130 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: Manager daemon y is now available 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:58.131 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:57 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T14:58:58.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:58] ENGINE Serving on http://:::9283 2024-08-29T14:58:58.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:14:58:58] ENGINE Bus STARTED 2024-08-29T14:58:58.882 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T14:58:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:58 smithi171 ceph-mon[22959]: mgrmap e30: y(active, since 1.01112s) 2024-08-29T14:58:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:58 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:58 smithi171 ceph-mon[22959]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-mon[18892]: mgrmap e30: y(active, since 1.01112s) 2024-08-29T14:58:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:59.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-mon[18892]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:59.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-mon[29342]: mgrmap e30: y(active, since 1.01112s) 2024-08-29T14:58:59.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T14:58:59.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:58 smithi027 ceph-mon[29342]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:58:59.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:59.033+0000 7f2655eb0640 -1 log_channel(cephadm) log [ERR] : [29/Aug/2024:14:58:59] ENGINE Error in HTTPServer.serve 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: Traceback (most recent call last): 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self._connections.run(self.expiration_interval) 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self._run(expiration_interval) 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:58:59.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: s = self.context.wrap_socket( 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: return self.sslsocket_class._create( 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self.do_handshake() 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self._sslobj.do_handshake() 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:58:59.071 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T14:58:59.473 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T14:58:59.473 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 6s ago 4m 24.9M - 0.25.0 c8568f914cd2 2e3fd226c647 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (4m) 6s ago 4m 88.6M - 9.4.7 954c08fa6188 f705829982dc 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (3m) 6s ago 3m 16.2M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (3m) 6s ago 3m 14.2M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (53s) 6s ago 8m 509M - 19.3.0-4579-g1f4056d9 d17246d170ec a8bc5f23457c 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (10s) 6s ago 10m 72.0M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (10m) 6s ago 10m 50.6M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (8m) 6s ago 8m 45.3M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (8m) 6s ago 8m 38.0M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T14:58:59.474 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (3m) 6s ago 3m 16.1M - 1.5.0 0da6a335fe13 8caf48017aa9 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (3m) 6s ago 3m 16.5M - 1.5.0 0da6a335fe13 91bd04afdd9c 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (8m) 6s ago 8m 50.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (7m) 6s ago 7m 51.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (7m) 6s ago 7m 56.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (6m) 6s ago 6m 45.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (6m) 6s ago 6m 53.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (6m) 6s ago 6m 46.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (5m) 6s ago 5m 44.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T14:58:59.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (5m) 6s ago 5m 48.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T14:58:59.476 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (16s) 6s ago 4m 33.2M - 2.43.0 a07b618ecd1d 2bad38ac2760 2024-08-29T14:58:59.840 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 1 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:58:59.841 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T14:58:59.842 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 13, 2024-08-29T14:58:59.842 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 1 2024-08-29T14:58:59.842 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T14:58:59.842 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:59:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:58] ENGINE Bus STARTING 2024-08-29T14:59:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[29342]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[29342]: mgrmap e31: y(active, since 2s) 2024-08-29T14:59:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:59] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:59:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:59] ENGINE Error in HTTPServer.serve 2024-08-29T14:59:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:59:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/378776258' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:58] ENGINE Bus STARTING 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[18892]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[18892]: mgrmap e31: y(active, since 2s) 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:59] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:59] ENGINE Error in HTTPServer.serve 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-08-29T14:59:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:59:00.071 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T14:59:00.072 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:59:00.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:58:59 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/378776258' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:00.150 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:59:00.150 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T14:59:00.150 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T14:59:00.150 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-29T14:59:00.151 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-08-29T14:59:00.151 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-08-29T14:59:00.151 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T14:59:00.151 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-08-29T14:59:00.151 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-08-29T14:59:00.151 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T14:59:00.151 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:59:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:59 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:58] ENGINE Bus STARTING 2024-08-29T14:59:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:59 smithi171 ceph-mon[22959]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:59 smithi171 ceph-mon[22959]: mgrmap e31: y(active, since 2s) 2024-08-29T14:59:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:59 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:59] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T14:59:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:59 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:59] ENGINE Error in HTTPServer.serve 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: Traceback (most recent call last): 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._connections.run(self.expiration_interval) 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._run(expiration_interval) 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: s = self.context.wrap_socket( 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T14:59:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout: return self.sslsocket_class._create( 2024-08-29T14:59:00.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T14:59:00.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: self.do_handshake() 2024-08-29T14:59:00.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T14:59:00.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._sslobj.do_handshake() 2024-08-29T14:59:00.269 INFO:journalctl@ceph.mon.b.smithi171.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T14:59:00.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:58:59 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/378776258' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:01.230 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:00 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:59] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:59:01.230 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:00 smithi171 ceph-mon[22959]: [29/Aug/2024:14:58:59] ENGINE Bus STARTED 2024-08-29T14:59:01.230 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:00 smithi171 ceph-mon[22959]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:01.230 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:00 smithi171 ceph-mon[22959]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:01.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:59] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:59:01.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[18892]: [29/Aug/2024:14:58:59] ENGINE Bus STARTED 2024-08-29T14:59:01.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[18892]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:01.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[18892]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:01.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:59] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T14:59:01.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[29342]: [29/Aug/2024:14:58:59] ENGINE Bus STARTED 2024-08-29T14:59:01.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[29342]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:01.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:00 smithi027 ceph-mon[29342]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:02.070 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:01 smithi171 ceph-mon[22959]: from='client.25547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:02.070 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:01 smithi171 ceph-mon[22959]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:02.070 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:01 smithi171 ceph-mon[22959]: mgrmap e32: y(active, since 4s) 2024-08-29T14:59:02.070 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:01 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:02.071 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:01 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[29342]: from='client.25547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[29342]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[29342]: mgrmap e32: y(active, since 4s) 2024-08-29T14:59:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[18892]: from='client.25547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[18892]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[18892]: mgrmap e32: y(active, since 4s) 2024-08-29T14:59:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:01 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:02.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:59:02] ENGINE Bus STARTING 2024-08-29T14:59:02.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: CherryPy Checker: 2024-08-29T14:59:02.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: The Application mounted at '' has an empty config. 2024-08-29T14:59:02.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: 2024-08-29T14:59:02.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:59:02] ENGINE Serving on http://:::9283 2024-08-29T14:59:02.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[56404]: [29/Aug/2024:14:59:02] ENGINE Bus STARTED 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: Standby manager daemon x started 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: Standby manager daemon x started 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:02 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: Standby manager daemon x started 2024-08-29T14:59:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:59:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:59:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:59:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/3698193698' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:59:03.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:03.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:02 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:04.137 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[29342]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:04.137 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[29342]: mgrmap e33: y(active, since 6s), standbys: x 2024-08-29T14:59:04.137 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:59:04.137 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:04.137 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[18892]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:04.138 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[18892]: mgrmap e33: y(active, since 6s), standbys: x 2024-08-29T14:59:04.138 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:59:04.138 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:03 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:03 smithi171 ceph-mon[22959]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:03 smithi171 ceph-mon[22959]: mgrmap e33: y(active, since 6s), standbys: x 2024-08-29T14:59:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:03 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T14:59:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:03 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:05 smithi171 ceph-mon[22959]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-29T14:59:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:05 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:05 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:05 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:05 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[29342]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-29T14:59:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[18892]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-29T14:59:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:05 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:59:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:59:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:59:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:59:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:59:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:59:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:59:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:59:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:59:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:59:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T14:59:07.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:59:07.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T14:59:07.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:07.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:59:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:59:08.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[29342]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:59:08.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[29342]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:59:08.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[29342]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-29T14:59:08.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[18892]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:59:08.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[18892]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:59:08.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[18892]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:59:08.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[18892]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:59:08.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:07 smithi027 ceph-mon[18892]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-29T14:59:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:07 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:59:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:07 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T14:59:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:07 smithi171 ceph-mon[22959]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-29T14:59:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:07 smithi171 ceph-mon[22959]: Reconfiguring daemon prometheus.a on smithi171 2024-08-29T14:59:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:07 smithi171 ceph-mon[22959]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-29T14:59:10.231 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:09 smithi171 ceph-mon[22959]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-29T14:59:10.231 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:09 smithi171 systemd[1]: Stopping Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:10.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:09 smithi027 ceph-mon[18892]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-29T14:59:10.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:09 smithi027 ceph-mon[29342]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-29T14:59:10.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.230Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.231Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.231Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.231Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-29T14:59:10.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[65030]: ts=2024-08-29T14:59:10.231Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-29T14:59:10.519 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 podman[67470]: 2024-08-29 14:59:10.241048245 +0000 UTC m=+0.117996486 container died 2bad38ac27606468d5f0b3ae855b3825aa8c1f697d076f8680e242c10cbab9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:10.882 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 podman[67470]: 2024-08-29 14:59:10.605326285 +0000 UTC m=+0.482274525 container cleanup 2bad38ac27606468d5f0b3ae855b3825aa8c1f697d076f8680e242c10cbab9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:10.882 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 bash[67470]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a 2024-08-29T14:59:11.244 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:10 smithi171 podman[67482]: 2024-08-29 14:59:10.881518216 +0000 UTC m=+0.642548144 container remove 2bad38ac27606468d5f0b3ae855b3825aa8c1f697d076f8680e242c10cbab9c1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:11.245 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-29T14:59:11.245 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 systemd[1]: Stopped Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:11.245 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.035s CPU time. 2024-08-29T14:59:11.245 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 systemd[1]: Starting Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:11.496 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 podman[67574]: 2024-08-29 14:59:11.147628735 +0000 UTC m=+0.016896298 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-29T14:59:11.496 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 podman[67574]: 2024-08-29 14:59:11.282694036 +0000 UTC m=+0.151961661 container create 47ae71089b578401715bd76697a6872b0f99052ea69f2d37644679ed8e5c5b49 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:11.497 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 podman[67574]: 2024-08-29 14:59:11.493143146 +0000 UTC m=+0.362410718 container init 47ae71089b578401715bd76697a6872b0f99052ea69f2d37644679ed8e5c5b49 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:11.767 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 podman[67574]: 2024-08-29 14:59:11.496081291 +0000 UTC m=+0.365348864 container start 47ae71089b578401715bd76697a6872b0f99052ea69f2d37644679ed8e5c5b49 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.521Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.521Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.521Z caller=main.go:570 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi171 (none))" 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.521Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.521Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.522Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.523Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.523Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.523Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-29T14:59:11.768 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.525Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.525Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.836µs 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.525Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.527Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.534Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.536Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.536Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.536Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.611µs wal_replay_duration=11.322936ms wbl_replay_duration=138ns total_replay_duration=11.360576ms 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.538Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.538Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.538Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.557Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.445437ms db_storage=935ns remote_storage=1.095µs web_handler=388ns query_engine=635ns scrape=480.343µs scrape_sd=113.618µs notify=14.1µs notify_sd=18.08µs rules=18.457884ms tracing=10.94µs 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.557Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-29T14:59:11.769 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:11.557Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-29T14:59:11.770 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 bash[67574]: 47ae71089b578401715bd76697a6872b0f99052ea69f2d37644679ed8e5c5b49 2024-08-29T14:59:11.770 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:11 smithi171 systemd[1]: Started Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-mon[22959]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:59:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:11 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[18892]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:59:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[29342]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:59:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:11 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:12 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:59:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:12 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:13.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:12 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:59:13.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:12 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:13.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:12 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T14:59:13.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:12 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:13 smithi171 ceph-mon[22959]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:13 smithi171 ceph-mon[22959]: Upgrade: Updating mgr.x 2024-08-29T14:59:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:59:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:13 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:59:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:59:14.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:14.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:13 smithi171 ceph-mon[22959]: Deploying daemon mgr.x on smithi171 2024-08-29T14:59:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[29342]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[29342]: Upgrade: Updating mgr.x 2024-08-29T14:59:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:59:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:59:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:59:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[29342]: Deploying daemon mgr.x on smithi171 2024-08-29T14:59:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[18892]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[18892]: Upgrade: Updating mgr.x 2024-08-29T14:59:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:59:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-29T14:59:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-29T14:59:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:13 smithi027 ceph-mon[18892]: Deploying daemon mgr.x on smithi171 2024-08-29T14:59:15.018 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:14 smithi171 systemd[1]: Stopping Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:15.329 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 podman[67907]: 2024-08-29 14:59:15.050957319 +0000 UTC m=+0.152428304 container died a8bc5f23457c5ef9da18b01854871fb18ffe285901a4e89df466685839896ba3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d) 2024-08-29T14:59:15.688 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 podman[67907]: 2024-08-29 14:59:15.411453196 +0000 UTC m=+0.512924189 container cleanup a8bc5f23457c5ef9da18b01854871fb18ffe285901a4e89df466685839896ba3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-08-29T14:59:15.688 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 bash[67907]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x 2024-08-29T14:59:15.995 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 podman[67919]: 2024-08-29 14:59:15.68751136 +0000 UTC m=+0.633672122 container remove a8bc5f23457c5ef9da18b01854871fb18ffe285901a4e89df466685839896ba3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True) 2024-08-29T14:59:15.995 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-29T14:59:15.995 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-29T14:59:15.995 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 systemd[1]: Stopped Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:15.995 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:15 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service: Consumed 17.262s CPU time. 2024-08-29T14:59:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:15 smithi171 ceph-mon[22959]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:16.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 systemd[1]: Starting Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:16.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 podman[68051]: 2024-08-29 14:59:16.13852761 +0000 UTC m=+0.015978335 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T14:59:16.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 podman[68051]: 2024-08-29 14:59:16.238796011 +0000 UTC m=+0.116246728 container create 85b20cdc09553c648a2259b3a7d00b55b90fcf6653e89cc622e6f7c67c8a386d (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T14:59:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:15 smithi027 ceph-mon[18892]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:15 smithi027 ceph-mon[29342]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T14:59:16.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 podman[68051]: 2024-08-29 14:59:16.491496311 +0000 UTC m=+0.368947026 container init 85b20cdc09553c648a2259b3a7d00b55b90fcf6653e89cc622e6f7c67c8a386d (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-29T14:59:16.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 podman[68051]: 2024-08-29 14:59:16.494512758 +0000 UTC m=+0.371963475 container start 85b20cdc09553c648a2259b3a7d00b55b90fcf6653e89cc622e6f7c67c8a386d (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-08-29T14:59:16.767 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 ceph-mgr[68070]: -- 172.21.15.171:0/638611345 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f1260ef0e0 con 0x55f1260cd000 2024-08-29T14:59:16.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 bash[68051]: 85b20cdc09553c648a2259b3a7d00b55b90fcf6653e89cc622e6f7c67c8a386d 2024-08-29T14:59:16.768 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:16 smithi171 systemd[1]: Started Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:17.965 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:17 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:17.965 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:17 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:17.965 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:17 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:17.965 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:17 smithi171 ceph-mon[22959]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[29342]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:18.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:18.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:18.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:18.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-mon[18892]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:18.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:59:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:59:17] "GET /metrics HTTP/1.1" 200 33811 "" "Prometheus/2.43.0" 2024-08-29T14:59:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:20 smithi171 ceph-mon[22959]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:20.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:20 smithi027 ceph-mon[18892]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:20 smithi027 ceph-mon[29342]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: Standby manager daemon x restarted 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: Standby manager daemon x started 2024-08-29T14:59:22.120 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:22 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: [29/Aug/2024:14:59:21] ENGINE Bus STARTING 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: CherryPy Checker: 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: The Application mounted at '' has an empty config. 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: [29/Aug/2024:14:59:21] ENGINE Serving on http://:::9283 2024-08-29T14:59:22.121 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 14:59:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: [29/Aug/2024:14:59:21] ENGINE Bus STARTED 2024-08-29T14:59:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: Standby manager daemon x restarted 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: Standby manager daemon x started 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: Standby manager daemon x restarted 2024-08-29T14:59:22.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: Standby manager daemon x started 2024-08-29T14:59:22.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T14:59:22.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T14:59:22.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T14:59:22.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:22 smithi027 ceph-mon[18892]: from='mgr.? 172.21.15.171:0/731268012' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T14:59:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:23 smithi171 ceph-mon[22959]: mgrmap e34: y(active, since 25s), standbys: x 2024-08-29T14:59:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:23 smithi171 ceph-mon[22959]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:23 smithi027 ceph-mon[18892]: mgrmap e34: y(active, since 25s), standbys: x 2024-08-29T14:59:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:23 smithi027 ceph-mon[18892]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:23 smithi027 ceph-mon[29342]: mgrmap e34: y(active, since 25s), standbys: x 2024-08-29T14:59:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:23 smithi027 ceph-mon[29342]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all mgr 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all crash 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all rgw 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all rbd-mirror 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all ceph-exporter 2024-08-29T14:59:26.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all iscsi 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all nfs 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all nvmeof 2024-08-29T14:59:26.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all mgr 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-29T14:59:26.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all crash 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all rgw 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all rbd-mirror 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all ceph-exporter 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all iscsi 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all nfs 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all nvmeof 2024-08-29T14:59:26.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all mgr 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-29T14:59:26.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all crash 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all rgw 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all rbd-mirror 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all ceph-exporter 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all iscsi 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all nfs 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all nvmeof 2024-08-29T14:59:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:27.306 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:27 smithi027 systemd[1]: Stopping Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:27.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:27 smithi027 podman[83832]: 2024-08-29 14:59:27.305396259 +0000 UTC m=+0.111309678 container died 8caf48017aa973806a496e95a010e7d0bc30e350a28783ada13871f431bafa68 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:28.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:27 smithi027 podman[83832]: 2024-08-29 14:59:27.753406241 +0000 UTC m=+0.559319658 container cleanup 8caf48017aa973806a496e95a010e7d0bc30e350a28783ada13871f431bafa68 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:28.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:27 smithi027 bash[83832]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a 2024-08-29T14:59:28.010 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:59:27 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:59:27] "GET /metrics HTTP/1.1" 200 33811 "" "Prometheus/2.43.0" 2024-08-29T14:59:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:28 smithi171 ceph-mon[22959]: Upgrade: Updating node-exporter.a (1/2) 2024-08-29T14:59:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:28 smithi171 ceph-mon[22959]: Deploying daemon node-exporter.a on smithi027 2024-08-29T14:59:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:28 smithi171 ceph-mon[22959]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:28 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[29342]: Upgrade: Updating node-exporter.a (1/2) 2024-08-29T14:59:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[29342]: Deploying daemon node-exporter.a on smithi027 2024-08-29T14:59:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[29342]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[18892]: Upgrade: Updating node-exporter.a (1/2) 2024-08-29T14:59:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[18892]: Deploying daemon node-exporter.a on smithi027 2024-08-29T14:59:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[18892]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:28 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:28.319 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 podman[83843]: 2024-08-29 14:59:28.038640194 +0000 UTC m=+0.735187799 container remove 8caf48017aa973806a496e95a010e7d0bc30e350a28783ada13871f431bafa68 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:28.319 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-29T14:59:28.319 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-29T14:59:28.319 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 systemd[1]: Stopped Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:28.319 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.a.service: Consumed 2.192s CPU time. 2024-08-29T14:59:28.818 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 systemd[1]: Starting Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:28.818 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 bash[83972]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-29T14:59:29.318 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 bash[83972]: Getting image source signatures 2024-08-29T14:59:29.318 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 bash[83972]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-29T14:59:29.318 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 bash[83972]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-29T14:59:29.318 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:28 smithi027 bash[83972]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-29T14:59:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:30 smithi171 ceph-mon[22959]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:30.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:30 smithi027 ceph-mon[29342]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:30.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:30 smithi027 ceph-mon[18892]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:30.462 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T14:59:30.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:30 smithi027 bash[83972]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-29T14:59:30.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:30 smithi027 bash[83972]: Writing manifest to image destination 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (4m) 29s ago 5m 24.9M - 0.25.0 c8568f914cd2 2e3fd226c647 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (4m) 10s ago 4m 88.6M - 9.4.7 954c08fa6188 f705829982dc 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (3m) 29s ago 3m 16.5M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (3m) 10s ago 3m 14.6M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (14s) 10s ago 8m 64.2M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (42s) 29s ago 10m 490M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (10m) 29s ago 10m 52.8M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T14:59:31.047 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (9m) 10s ago 9m 49.9M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (9m) 29s ago 9m 42.3M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 29s ago 4m 15.7M - 1.5.0 0da6a335fe13 8caf48017aa9 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (4m) 10s ago 4m 16.5M - 1.5.0 0da6a335fe13 91bd04afdd9c 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (8m) 29s ago 8m 51.1M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (8m) 29s ago 8m 51.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (7m) 29s ago 7m 56.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (7m) 29s ago 7m 45.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (7m) 10s ago 7m 53.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (6m) 10s ago 6m 46.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (6m) 10s ago 6m 45.5M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (5m) 10s ago 5m 49.2M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T14:59:31.048 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (19s) 10s ago 5m 36.9M - 2.43.0 a07b618ecd1d 47ae71089b57 2024-08-29T14:59:31.068 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:30 smithi027 podman[83972]: 2024-08-29 14:59:30.807965359 +0000 UTC m=+2.315224935 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-29T14:59:31.068 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:30 smithi027 podman[83972]: 2024-08-29 14:59:30.947049778 +0000 UTC m=+2.454309352 container create e53886849ab5c6733f39c394787ad02242e49fd5a0e8394a79b396924225ccbd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:31.401 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:59:31.401 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T14:59:31.401 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T14:59:31.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:59:31.401 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 13, 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T14:59:31.402 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:59:31.403 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 podman[83972]: 2024-08-29 14:59:31.184528212 +0000 UTC m=+2.691787807 container init e53886849ab5c6733f39c394787ad02242e49fd5a0e8394a79b396924225ccbd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:31.403 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 podman[83972]: 2024-08-29 14:59:31.18835698 +0000 UTC m=+2.695616565 container start e53886849ab5c6733f39c394787ad02242e49fd5a0e8394a79b396924225ccbd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:31.403 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.191Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.191Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.192Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.192Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.192Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.192Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=arp 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-29T14:59:31.404 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=edac 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-29T14:59:31.405 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=os 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-29T14:59:31.406 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=stat 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=time 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=uname 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-29T14:59:31.407 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.193Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-29T14:59:31.408 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.194Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-29T14:59:31.408 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a[84115]: ts=2024-08-29T14:59:31.194Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-29T14:59:31.408 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 bash[83972]: e53886849ab5c6733f39c394787ad02242e49fd5a0e8394a79b396924225ccbd 2024-08-29T14:59:31.408 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 14:59:31 smithi027 systemd[1]: Started Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:31.703 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T14:59:31.703 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading node-exporter daemons", 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T14:59:31.704 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T14:59:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:32 smithi171 ceph-mon[22959]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:32 smithi171 ceph-mon[22959]: from='client.25559 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:32 smithi171 ceph-mon[22959]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:32 smithi171 ceph-mon[22959]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:32 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:32 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:32 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/3708401773' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[29342]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[29342]: from='client.25559 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[29342]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[29342]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/3708401773' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[18892]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[18892]: from='client.25559 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[18892]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[18892]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:32 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/3708401773' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:33 smithi171 ceph-mon[22959]: from='client.15684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:33.268 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:33 smithi171 systemd[1]: Stopping Ceph node-exporter.b for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:33.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:33 smithi027 ceph-mon[18892]: from='client.15684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:33.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:33 smithi027 ceph-mon[29342]: from='client.15684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T14:59:33.721 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:33 smithi171 podman[71601]: 2024-08-29 14:59:33.437931992 +0000 UTC m=+0.095557954 container died 91bd04afdd9c9a56bbce21914292846f0527dfdcfa61d3b1d36a3fa2478200cc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:59:34.017 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:33 smithi171 podman[71601]: 2024-08-29 14:59:33.81891098 +0000 UTC m=+0.476536934 container cleanup 91bd04afdd9c9a56bbce21914292846f0527dfdcfa61d3b1d36a3fa2478200cc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:59:34.017 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:33 smithi171 bash[71601]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b 2024-08-29T14:59:34.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:34 smithi027 ceph-mon[18892]: Upgrade: Updating node-exporter.b (2/2) 2024-08-29T14:59:34.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:34 smithi027 ceph-mon[18892]: Deploying daemon node-exporter.b on smithi171 2024-08-29T14:59:34.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:34 smithi027 ceph-mon[18892]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:34.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:34 smithi027 ceph-mon[29342]: Upgrade: Updating node-exporter.b (2/2) 2024-08-29T14:59:34.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:34 smithi027 ceph-mon[29342]: Deploying daemon node-exporter.b on smithi171 2024-08-29T14:59:34.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:34 smithi027 ceph-mon[29342]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:34.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:34 smithi171 ceph-mon[22959]: Upgrade: Updating node-exporter.b (2/2) 2024-08-29T14:59:34.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:34 smithi171 ceph-mon[22959]: Deploying daemon node-exporter.b on smithi171 2024-08-29T14:59:34.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:34 smithi171 ceph-mon[22959]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:34.434 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 podman[71613]: 2024-08-29 14:59:34.087018334 +0000 UTC m=+0.651738517 container remove 91bd04afdd9c9a56bbce21914292846f0527dfdcfa61d3b1d36a3fa2478200cc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:59:34.435 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-29T14:59:34.435 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-29T14:59:34.435 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 systemd[1]: Stopped Ceph node-exporter.b for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:34.435 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.b.service: Consumed 2.098s CPU time. 2024-08-29T14:59:34.435 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 systemd[1]: Starting Ceph node-exporter.b for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:34.767 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 bash[71740]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-29T14:59:35.267 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 bash[71740]: Getting image source signatures 2024-08-29T14:59:35.267 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 bash[71740]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-29T14:59:35.267 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 bash[71740]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-29T14:59:35.267 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:34 smithi171 bash[71740]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-29T14:59:36.300 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:36 smithi171 ceph-mon[22959]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:36 smithi027 ceph-mon[18892]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:36 smithi027 ceph-mon[29342]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:36.751 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:36 smithi171 bash[71740]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-29T14:59:36.752 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:36 smithi171 bash[71740]: Writing manifest to image destination 2024-08-29T14:59:37.017 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:36 smithi171 podman[71740]: 2024-08-29 14:59:36.689599534 +0000 UTC m=+2.161831553 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-29T14:59:37.018 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:36 smithi171 podman[71740]: 2024-08-29 14:59:36.846542302 +0000 UTC m=+2.318774393 container create 46afc3b5440459faa591cf8c229ac73dd2f4ad45ec5b27aa0194a78db2df1c4c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 podman[71740]: 2024-08-29 14:59:37.048797187 +0000 UTC m=+2.521029206 container init 46afc3b5440459faa591cf8c229ac73dd2f4ad45ec5b27aa0194a78db2df1c4c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 podman[71740]: 2024-08-29 14:59:37.051908453 +0000 UTC m=+2.524140472 container start 46afc3b5440459faa591cf8c229ac73dd2f4ad45ec5b27aa0194a78db2df1c4c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.055Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.055Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.055Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.055Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.056Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.056Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.056Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-29T14:59:37.518 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.056Z caller=node_exporter.go:117 level=info collector=arp 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=edac 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-29T14:59:37.519 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-29T14:59:37.520 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=os 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=stat 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-29T14:59:37.521 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=time 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=uname 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.057Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b[71816]: ts=2024-08-29T14:59:37.058Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 bash[71740]: 46afc3b5440459faa591cf8c229ac73dd2f4ad45ec5b27aa0194a78db2df1c4c 2024-08-29T14:59:37.522 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 14:59:37 smithi171 systemd[1]: Started Ceph node-exporter.b for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:38.026 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:59:37 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:59:37] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.43.0" 2024-08-29T14:59:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[29342]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[18892]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:38 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.488 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:38 smithi171 ceph-mon[22959]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:38.488 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:38 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.488 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:38 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.488 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:38 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:38.488 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:38 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:38.488 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:38 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:40.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:40 smithi027 ceph-mon[18892]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:40.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:40 smithi027 ceph-mon[29342]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:40 smithi171 ceph-mon[22959]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.361 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:42.361 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.361 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.361 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.361 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.361 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.361 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.362 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:42.362 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:43 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:43 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:43.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:43 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:44.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:44 smithi027 ceph-mon[29342]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:44.290 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:44 smithi027 ceph-mon[18892]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:44.340 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:44 smithi171 ceph-mon[22959]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:46 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:46.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T14:59:46.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:46.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:46 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T14:59:48.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 14:59:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:14:59:47] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.43.0" 2024-08-29T14:59:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:48 smithi171 ceph-mon[22959]: Upgrade: Updating prometheus.a 2024-08-29T14:59:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:48 smithi171 ceph-mon[22959]: Deploying daemon prometheus.a on smithi171 2024-08-29T14:59:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:48 smithi171 ceph-mon[22959]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:48 smithi027 ceph-mon[18892]: Upgrade: Updating prometheus.a 2024-08-29T14:59:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:48 smithi027 ceph-mon[18892]: Deploying daemon prometheus.a on smithi171 2024-08-29T14:59:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:48 smithi027 ceph-mon[18892]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:48 smithi027 ceph-mon[29342]: Upgrade: Updating prometheus.a 2024-08-29T14:59:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:48 smithi027 ceph-mon[29342]: Deploying daemon prometheus.a on smithi171 2024-08-29T14:59:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:48 smithi027 ceph-mon[29342]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:50 smithi171 ceph-mon[22959]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:50 smithi027 ceph-mon[18892]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:50 smithi027 ceph-mon[29342]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:52 smithi171 ceph-mon[22959]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:52 smithi027 ceph-mon[18892]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:52 smithi027 ceph-mon[29342]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:54.119 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:54 smithi171 ceph-mon[22959]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:54 smithi027 ceph-mon[18892]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:54 smithi027 ceph-mon[29342]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:56 smithi027 ceph-mon[18892]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:56 smithi027 ceph-mon[29342]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:56.471 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:56 smithi171 ceph-mon[22959]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:57.111 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:56 smithi171 systemd[1]: Stopping Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:57.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.109Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-29T14:59:57.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-29T14:59:57.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-29T14:59:57.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[67588]: ts=2024-08-29T14:59:57.110Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-29T14:59:57.518 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 podman[76053]: 2024-08-29 14:59:57.120216211 +0000 UTC m=+0.138344392 container died 47ae71089b578401715bd76697a6872b0f99052ea69f2d37644679ed8e5c5b49 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:57.878 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 podman[76053]: 2024-08-29 14:59:57.64726786 +0000 UTC m=+0.665396085 container cleanup 47ae71089b578401715bd76697a6872b0f99052ea69f2d37644679ed8e5c5b49 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:57.878 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 bash[76053]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a 2024-08-29T14:59:58.123 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-mon[22959]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:58.124 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:58.124 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:57 smithi171 podman[76065]: 2024-08-29 14:59:57.866261554 +0000 UTC m=+0.747704925 container remove 47ae71089b578401715bd76697a6872b0f99052ea69f2d37644679ed8e5c5b49 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:58 smithi027 ceph-mon[18892]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:58 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:58.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:58 smithi027 ceph-mon[29342]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T14:59:58.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:58 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T14:59:58.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-29T14:59:58.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 systemd[1]: Stopped Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T14:59:58.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.104s CPU time. 2024-08-29T14:59:58.517 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 systemd[1]: Starting Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T14:59:58.820 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 podman[76198]: 2024-08-29 14:59:58.440113396 +0000 UTC m=+0.016975079 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-08-29T14:59:58.820 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 podman[76198]: 2024-08-29 14:59:58.576222771 +0000 UTC m=+0.153084449 container create b0e73daa625e9cce05ef1ecc82bc8de739d30539b6dfed3708bd0ccf79711f32 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:58.821 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 podman[76198]: 2024-08-29 14:59:58.820002976 +0000 UTC m=+0.396864655 container init b0e73daa625e9cce05ef1ecc82bc8de739d30539b6dfed3708bd0ccf79711f32 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 podman[76198]: 2024-08-29 14:59:58.823050411 +0000 UTC m=+0.399912095 container start b0e73daa625e9cce05ef1ecc82bc8de739d30539b6dfed3708bd0ccf79711f32 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.842Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.842Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.842Z caller=main.go:623 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi171 (none))" 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.842Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.842Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.844Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.845Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.847Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-08-29T14:59:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.847Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.848Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.848Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.609µs 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.848Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.850Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.856Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.856Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.857Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.857Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.857Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.567µs wal_replay_duration=9.313988ms wbl_replay_duration=87ns total_replay_duration=9.358457ms 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.860Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.860Z caller=main.go:1153 level=info msg="TSDB started" 2024-08-29T14:59:59.269 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.860Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-29T14:59:59.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.876Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.818187ms db_storage=1.009µs remote_storage=1.036µs web_handler=310ns query_engine=525ns scrape=384.957µs scrape_sd=89.839µs notify=17.909µs notify_sd=15.905µs rules=14.969802ms tracing=7.315µs 2024-08-29T14:59:59.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.876Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-08-29T14:59:59.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T14:59:58.876Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-08-29T14:59:59.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 bash[76198]: b0e73daa625e9cce05ef1ecc82bc8de739d30539b6dfed3708bd0ccf79711f32 2024-08-29T14:59:59.270 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 14:59:58 smithi171 systemd[1]: Started Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:00:00.090 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:59 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.090 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:59 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.090 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:59 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:00.090 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:59 smithi171 ceph-mon[22959]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:00.090 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:59 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.090 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 14:59:59 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[18892]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[29342]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 14:59:59 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:00 smithi171 ceph-mon[22959]: overall HEALTH_OK 2024-08-29T15:00:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:00 smithi027 ceph-mon[18892]: overall HEALTH_OK 2024-08-29T15:00:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:00 smithi027 ceph-mon[29342]: overall HEALTH_OK 2024-08-29T15:00:02.032 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:00:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:01 smithi171 ceph-mon[22959]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:01 smithi027 ceph-mon[29342]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:01 smithi027 ceph-mon[18892]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (4m) 21s ago 5m 24.9M - 0.25.0 c8568f914cd2 2e3fd226c647 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (5m) 21s ago 5m 86.1M - 9.4.7 954c08fa6188 f705829982dc 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (4m) 21s ago 4m 16.8M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (4m) 21s ago 4m 14.7M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (46s) 21s ago 9m 451M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (73s) 21s ago 11m 509M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (11m) 21s ago 11m 54.1M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T15:00:02.613 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (9m) 21s ago 9m 51.2M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (9m) 21s ago 9m 46.2M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (31s) 21s ago 4m 8409k - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (25s) 21s ago 4m 3343k - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (9m) 21s ago 9m 51.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (8m) 21s ago 8m 51.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (8m) 21s ago 8m 56.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (7m) 21s ago 7m 45.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (7m) 21s ago 7m 54.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (7m) 21s ago 7m 46.5M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (6m) 21s ago 6m 45.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (6m) 21s ago 6m 49.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:00:02.614 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 starting - - - - 2024-08-29T15:00:02.964 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:00:02.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:00:02.964 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T15:00:02.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:02.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:00:02.964 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:00:02.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 13, 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:00:02.965 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:00:03.260 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading prometheus daemons", 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:00:03.261 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:00:03.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:02 smithi027 ceph-mon[18892]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:03.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:02 smithi027 ceph-mon[29342]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:02 smithi171 ceph-mon[22959]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:03 smithi171 ceph-mon[22959]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:03 smithi171 ceph-mon[22959]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:03 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/4143608742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:03 smithi171 ceph-mon[22959]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[18892]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[18892]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/4143608742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[18892]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[29342]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[29342]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/4143608742' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:03 smithi027 ceph-mon[29342]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:04 smithi171 ceph-mon[22959]: from='client.25607 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:04 smithi027 ceph-mon[18892]: from='client.25607 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:04 smithi027 ceph-mon[29342]: from='client.25607 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:06.272 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:06 smithi171 ceph-mon[22959]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:06.272 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.273 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.273 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.273 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:06 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[18892]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[29342]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:06 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:08 smithi027 ceph-mon[18892]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:08 smithi027 ceph-mon[29342]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:08.486 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:08 smithi171 ceph-mon[22959]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:00:09] "GET /metrics HTTP/1.1" 200 33810 "" "Prometheus/2.51.0" 2024-08-29T15:00:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[18892]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:09 smithi027 ceph-mon[29342]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:10.143 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.144 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.145 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.145 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.145 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.145 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.145 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.145 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:10.145 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:09 smithi171 ceph-mon[22959]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:12 smithi027 ceph-mon[29342]: Upgrade: Updating alertmanager.a 2024-08-29T15:00:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:12 smithi027 ceph-mon[29342]: Deploying daemon alertmanager.a on smithi027 2024-08-29T15:00:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:12 smithi027 ceph-mon[29342]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:12 smithi027 ceph-mon[18892]: Upgrade: Updating alertmanager.a 2024-08-29T15:00:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:12 smithi027 ceph-mon[18892]: Deploying daemon alertmanager.a on smithi027 2024-08-29T15:00:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:12 smithi027 ceph-mon[18892]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:12 smithi171 ceph-mon[22959]: Upgrade: Updating alertmanager.a 2024-08-29T15:00:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:12 smithi171 ceph-mon[22959]: Deploying daemon alertmanager.a on smithi027 2024-08-29T15:00:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:12 smithi171 ceph-mon[22959]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:13.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:13.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:13.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:14 smithi027 ceph-mon[18892]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:14 smithi027 ceph-mon[29342]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:14 smithi171 ceph-mon[22959]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:16.158 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:16 smithi027 ceph-mon[29342]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:16.159 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:16 smithi027 ceph-mon[18892]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:16 smithi171 ceph-mon[22959]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:18.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:18 smithi027 ceph-mon[18892]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:18.140 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:18 smithi027 ceph-mon[29342]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:18.140 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:17 smithi027 systemd[1]: Stopping Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:00:18.466 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[62981]: ts=2024-08-29T15:00:18.138Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-29T15:00:18.466 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:18 smithi027 podman[88518]: 2024-08-29 15:00:18.179523029 +0000 UTC m=+0.147171646 container died 2e3fd226c647821b371b4fa73e6ccf8f963ca285fd883615c8b59fcab5e71022 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:00:18.535 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:18 smithi171 ceph-mon[22959]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:18.818 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:18 smithi027 podman[88518]: 2024-08-29 15:00:18.681144737 +0000 UTC m=+0.648793393 container cleanup 2e3fd226c647821b371b4fa73e6ccf8f963ca285fd883615c8b59fcab5e71022 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:00:18.818 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:18 smithi027 bash[88518]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-29T15:00:19.157 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:18 smithi027 podman[88530]: 2024-08-29 15:00:18.90753318 +0000 UTC m=+0.718742771 container remove 2e3fd226c647821b371b4fa73e6ccf8f963ca285fd883615c8b59fcab5e71022 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:00:19.157 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:18 smithi027 podman[88530]: 2024-08-29 15:00:18.991528651 +0000 UTC m=+0.802738253 volume remove bba05364115ee27b29647b71454078ad22fc5f1065998066cda9ef39b3b72790 2024-08-29T15:00:19.157 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:00:19] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.51.0" 2024-08-29T15:00:19.505 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:19 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-29T15:00:19.505 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:19 smithi027 systemd[1]: Stopped Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:00:19.505 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:19 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@alertmanager.a.service: Consumed 1.186s CPU time. 2024-08-29T15:00:19.505 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:19 smithi027 systemd[1]: Starting Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:00:19.814 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:19 smithi027 podman[88662]: 2024-08-29 15:00:19.496629033 +0000 UTC m=+0.022511724 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-08-29T15:00:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-mon[18892]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-mon[29342]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:20.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:19 smithi027 podman[88662]: 2024-08-29 15:00:19.813817102 +0000 UTC m=+0.339699784 volume create 77a52f9a4ea478a51208bb4a310620c74e2b3b7947bff3c77e1cf1d9568b1404 2024-08-29T15:00:20.068 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:19 smithi027 podman[88662]: 2024-08-29 15:00:19.889409413 +0000 UTC m=+0.415292092 container create 4ac5fe286fabd1fa04328fcd588accc8ffc9edbf3a0f16a31b3a7cb0108c3090 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:00:20.319 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:20 smithi171 ceph-mon[22959]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:20.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 podman[88662]: 2024-08-29 15:00:20.175616648 +0000 UTC m=+0.701499335 container init 4ac5fe286fabd1fa04328fcd588accc8ffc9edbf3a0f16a31b3a7cb0108c3090 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:00:20.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 podman[88662]: 2024-08-29 15:00:20.178836013 +0000 UTC m=+0.704718695 container start 4ac5fe286fabd1fa04328fcd588accc8ffc9edbf3a0f16a31b3a7cb0108c3090 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:00:20.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.189Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.189Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.190Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.192Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.213Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.214Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.216Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:20.216Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 bash[88662]: 4ac5fe286fabd1fa04328fcd588accc8ffc9edbf3a0f16a31b3a7cb0108c3090 2024-08-29T15:00:20.569 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:20 smithi027 systemd[1]: Started Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:00:21.588 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:21.588 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:21.588 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:21.588 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[29342]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:21.589 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:21.589 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:21.589 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:21.589 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:21 smithi027 ceph-mon[18892]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:21.635 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:21 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:21.636 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:21 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:21.636 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:21 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:21.636 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:21 smithi171 ceph-mon[22959]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:22.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:22 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:22.192Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000389672s 2024-08-29T15:00:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:24 smithi171 ceph-mon[22959]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:24 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:24 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:24 smithi027 ceph-mon[18892]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:24 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:24 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:24 smithi027 ceph-mon[29342]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:24 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:24 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[18892]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:25.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:25 smithi027 ceph-mon[29342]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:25 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:25 smithi171 ceph-mon[22959]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:28 smithi027 ceph-mon[29342]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:28 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:28 smithi027 ceph-mon[18892]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:28 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:28 smithi171 ceph-mon[22959]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:28 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:29 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:00:29] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.51.0" 2024-08-29T15:00:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:29.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:29 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:30 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T15:00:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:30 smithi171 ceph-mon[22959]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:30 smithi171 ceph-mon[22959]: mgrmap e35: y(active, since 92s), standbys: x 2024-08-29T15:00:30.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:00:30 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:00:30.194Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002477159s 2024-08-29T15:00:30.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:30 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T15:00:30.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:30 smithi027 ceph-mon[18892]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:30.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:30 smithi027 ceph-mon[18892]: mgrmap e35: y(active, since 92s), standbys: x 2024-08-29T15:00:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:30 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-29T15:00:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:30 smithi027 ceph-mon[29342]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:30 smithi027 ceph-mon[29342]: mgrmap e35: y(active, since 92s), standbys: x 2024-08-29T15:00:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:31 smithi171 ceph-mon[22959]: Upgrade: Updating grafana.a 2024-08-29T15:00:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:31 smithi171 ceph-mon[22959]: Deploying daemon grafana.a on smithi171 2024-08-29T15:00:31.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:31 smithi027 ceph-mon[29342]: Upgrade: Updating grafana.a 2024-08-29T15:00:31.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:31 smithi027 ceph-mon[29342]: Deploying daemon grafana.a on smithi171 2024-08-29T15:00:31.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:31 smithi027 ceph-mon[18892]: Upgrade: Updating grafana.a 2024-08-29T15:00:31.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:31 smithi027 ceph-mon[18892]: Deploying daemon grafana.a on smithi171 2024-08-29T15:00:32.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:32 smithi027 ceph-mon[29342]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:32.356 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:32 smithi027 ceph-mon[18892]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:32 smithi171 ceph-mon[22959]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:33.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:33 smithi027 ceph-mon[18892]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:33.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:33 smithi027 ceph-mon[29342]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:33 smithi171 ceph-mon[22959]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:33.578 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (14s) 10s ago 6m 17.4M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (5m) 9s ago 5m 87.6M - 9.4.7 954c08fa6188 f705829982dc 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (4m) 10s ago 4m 17.1M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (4m) 9s ago 4m 14.9M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (78s) 9s ago 9m 452M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (105s) 10s ago 11m 512M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (11m) 10s ago 11m 53.7M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (10m) 9s ago 10m 48.2M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T15:00:34.169 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (10m) 10s ago 10m 42.0M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (63s) 10s ago 5m 9697k - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (57s) 9s ago 5m 11.5M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (9m) 10s ago 9m 51.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (9m) 10s ago 9m 52.1M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (8m) 10s ago 8m 56.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (8m) 10s ago 8m 46.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (8m) 9s ago 8m 54.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (7m) 9s ago 7m 46.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (7m) 9s ago 7m 45.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (6m) 9s ago 6m 49.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:00:34.170 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (35s) 9s ago 6m 35.5M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:00:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:34 smithi171 ceph-mon[22959]: from='client.25613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:34 smithi171 ceph-mon[22959]: from='client.25619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:34.518 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:00:34.519 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:00:34.520 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 13, 2024-08-29T15:00:34.520 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:00:34.520 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:00:34.520 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:00:34.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:34 smithi027 ceph-mon[29342]: from='client.25613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:34.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:34 smithi027 ceph-mon[29342]: from='client.25619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:34.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:34 smithi027 ceph-mon[18892]: from='client.25613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:34.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:34 smithi027 ceph-mon[18892]: from='client.25619 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:34.816 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:00:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:00:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading grafana daemons", 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:00:34.817 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:00:35.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:35 smithi171 ceph-mon[22959]: from='client.15732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:35.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:35 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/582922385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:35.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:35 smithi171 ceph-mon[22959]: from='client.25637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:35.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:35 smithi171 ceph-mon[22959]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[18892]: from='client.15732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/582922385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[18892]: from='client.25637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[18892]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[29342]: from='client.15732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/582922385' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[29342]: from='client.25637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:00:35.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:35 smithi027 ceph-mon[29342]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:38 smithi171 ceph-mon[22959]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:38 smithi027 ceph-mon[18892]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:38 smithi027 ceph-mon[29342]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:39.293 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:00:39] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.51.0" 2024-08-29T15:00:40.057 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:40 smithi171 ceph-mon[22959]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:40.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:40 smithi027 ceph-mon[29342]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:40.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:40 smithi027 ceph-mon[18892]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:00:40.353 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:40 smithi171 systemd[1]: Stopping Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:00:40.733 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:40 smithi171 podman[80669]: 2024-08-29 15:00:40.351783481 +0000 UTC m=+0.117165317 container died f705829982dcfa5f65815e176b733adcca3b24d40eec444d50b3170ef8a46e65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, architecture=x86_64, distribution-scope=public) 2024-08-29T15:00:41.017 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:40 smithi171 podman[80669]: 2024-08-29 15:00:40.732791028 +0000 UTC m=+0.498172863 container cleanup f705829982dcfa5f65815e176b733adcca3b24d40eec444d50b3170ef8a46e65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, release=239.1651231664, io.buildah.version=1.32.0, version=8.5, maintainer=Paul Cuzner , io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, name=ubi8, build-date=2022-04-29T11:48:11.055897) 2024-08-29T15:00:41.017 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:40 smithi171 bash[80669]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a 2024-08-29T15:00:41.017 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:40 smithi171 podman[80681]: 2024-08-29 15:00:40.967240826 +0000 UTC m=+0.613784548 container remove f705829982dcfa5f65815e176b733adcca3b24d40eec444d50b3170ef8a46e65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, com.redhat.component=ubi8-container, name=ubi8, version=8.5, vcs-type=git, release=239.1651231664, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.openshift.tags=base rhel8) 2024-08-29T15:00:41.017 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:40 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-08-29T15:00:41.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-08-29T15:00:41.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 systemd[1]: Stopped Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:00:41.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@grafana.a.service: Consumed 7.448s CPU time. 2024-08-29T15:00:41.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 systemd[1]: Starting Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:00:41.838 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 podman[80809]: 2024-08-29 15:00:41.420961209 +0000 UTC m=+0.016709454 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-08-29T15:00:41.838 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 podman[80809]: 2024-08-29 15:00:41.543562516 +0000 UTC m=+0.139310795 container create 99f2571ddaab8f90df9586a9a708d8d258bae46bfb214f0683167bbe01e14e57 (image=quay.io/ceph/grafana:10.4.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-29T15:00:41.838 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 podman[80809]: 2024-08-29 15:00:41.762563566 +0000 UTC m=+0.358311801 container init 99f2571ddaab8f90df9586a9a708d8d258bae46bfb214f0683167bbe01e14e57 (image=quay.io/ceph/grafana:10.4.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-29T15:00:41.838 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 podman[80809]: 2024-08-29 15:00:41.765546613 +0000 UTC m=+0.361294852 container start 99f2571ddaab8f90df9586a9a708d8d258bae46bfb214f0683167bbe01e14e57 (image=quay.io/ceph/grafana:10.4.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-29T15:00:42.098 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-mon[22959]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:42.098 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:42.098 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:42.098 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 bash[80809]: 99f2571ddaab8f90df9586a9a708d8d258bae46bfb214f0683167bbe01e14e57 2024-08-29T15:00:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 systemd[1]: Started Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:00:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.893815627Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-08-29T15:00:41Z 2024-08-29T15:00:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894088244Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-29T15:00:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894099388Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-29T15:00:42.099 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894104534Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894109125Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894113324Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894117584Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894122064Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.89412647Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.89413105Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.89413527Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894139443Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894143774Z level=info msg=Target target=[all] 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.89415242Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-29T15:00:42.100 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894156656Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894160764Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894164924Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.894169127Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=settings t=2024-08-29T15:00:41.89417335Z level=info msg="App mode production" 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=sqlstore t=2024-08-29T15:00:41.894574788Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=sqlstore t=2024-08-29T15:00:41.894601064Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.895391718Z level=info msg="Starting DB migrations" 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.907150485Z level=info msg="Executing migration" id="Add uid column to user" 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.912745651Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=5.591714ms 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.913204945Z level=info msg="Executing migration" id="Update uid column values for users" 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.913368101Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=161.887µs 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.913784277Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-08-29T15:00:42.101 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.914544504Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=761.468µs 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.916537673Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.92157684Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.038476ms 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.922191875Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.927480967Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.287664ms 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.927980136Z level=info msg="Executing migration" id="Add column uid in team" 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.933073647Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.092793ms 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.933472766Z level=info msg="Executing migration" id="Update uid column values in team" 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.933612206Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=140.008µs 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.936014567Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.936810682Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=796.432µs 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.938673259Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-08-29T15:00:42.102 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.93938857Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=715.087µs 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.941062475Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.946497715Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.433572ms 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.946949019Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.951959225Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=5.009255ms 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.952358991Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.952411868Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=52.472µs 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.954939221Z level=info msg="Executing migration" id="permission kind migration" 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.960018427Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.078051ms 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.960403649Z level=info msg="Executing migration" id="permission attribute migration" 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.965588865Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.183981ms 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.965974643Z level=info msg="Executing migration" id="permission identifier migration" 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.971537031Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.561737ms 2024-08-29T15:00:42.103 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.971961983Z level=info msg="Executing migration" id="add permission identifier index" 2024-08-29T15:00:42.104 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.972790938Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=828.617µs 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.974689097Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.975463649Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=775.613µs 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.975824395Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.976563343Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=738.665µs 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.976934465Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.976958591Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=23.86µs 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.977690831Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-08-29T15:00:42.105 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.978391696Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=700.151µs 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.978751051Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.979429217Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=678.045µs 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.979782014Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.995229346Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=15.444947ms 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.99569989Z level=info msg="Executing migration" id="create correlation v2" 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.996457074Z level=info msg="Migration successfully executed" id="create correlation v2" duration=756.561µs 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.996834027Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.997531685Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=697.474µs 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.997891903Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.998570093Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=677.984µs 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.998941007Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-08-29T15:00:42.106 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.999618317Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=677.261µs 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:41.999990194Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.000167787Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=175.865µs 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.000718967Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.001310482Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=591.197µs 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.001685158Z level=info msg="Executing migration" id="add provisioning column" 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.006942118Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.254242ms 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.007398772Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.007549279Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=150.089µs 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.007961219Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.008010067Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=48.279µs 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.008782934Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.01429943Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.513442ms 2024-08-29T15:00:42.107 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.014714758Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.014922303Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=207.578µs 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.015419149Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.015542201Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=121.672µs 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.015959513Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.016878293Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=918.03µs 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.017255676Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.017958884Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=702.636µs 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.018324541Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.019117146Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=792.307µs 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.019503215Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.019986161Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=482.753µs 2024-08-29T15:00:42.108 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.020345994Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.020544456Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=199.629µs 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.021119855Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.021806547Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=686.706µs 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.022150197Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.022857378Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=706.78µs 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.023206402Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.023899095Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=692.521µs 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.024252262Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.024959873Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=708.687µs 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.025311977Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.026004398Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=692.246µs 2024-08-29T15:00:42.109 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.026368602Z level=info msg="Executing migration" id="create anon_device table" 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.026948428Z level=info msg="Migration successfully executed" id="create anon_device table" duration=579.418µs 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.027334993Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.028053689Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=720.063µs 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.028413224Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.029119017Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=705.38µs 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.029483052Z level=info msg="Executing migration" id="create signing_key table" 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.030078676Z level=info msg="Migration successfully executed" id="create signing_key table" duration=595.688µs 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.030450107Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.031137061Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=686.782µs 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.031490512Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.032172373Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=681.852µs 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.032514996Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-08-29T15:00:42.110 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.032945206Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=430.866µs 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.03337137Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.038660575Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.288021ms 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.039063135Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.040847871Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.78548ms 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.041867337Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.042795238Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=927.408µs 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.044504567Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.045233828Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=729.264µs 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.045559438Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.046260542Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=701.016µs 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.048241973Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-08-29T15:00:42.111 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.049108501Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=866.131µs 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.050966295Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.051759399Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=792.276µs 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.053756064Z level=info msg="Executing migration" id="create sso_setting table" 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.054501622Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=745.324µs 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.056387433Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.057070063Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=682.597µs 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.058937685Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.059345344Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=407.798µs 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.061611879Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.061667641Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=57.538µs 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.064255511Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-08-29T15:00:42.112 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.069693701Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.437825ms 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.070079583Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.075440861Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.361404ms 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.075823655Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.076054058Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=230.315µs 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=migrator t=2024-08-29T15:00:42.078467856Z level=info msg="migrations completed" performed=65 skipped=482 duration=171.367401ms 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=sqlstore t=2024-08-29T15:00:42.079205461Z level=info msg="Created default organization" 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=secrets t=2024-08-29T15:00:42.08134698Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-29T15:00:42.113 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugin.store t=2024-08-29T15:00:42.097809508Z level=info msg="Loading plugins..." 2024-08-29T15:00:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[18892]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[29342]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:42 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:42.429 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=local.finder t=2024-08-29T15:00:42.140627015Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-08-29T15:00:42.429 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugin.store t=2024-08-29T15:00:42.140646087Z level=info msg="Plugins loaded" count=55 duration=42.83758ms 2024-08-29T15:00:42.429 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=query_data t=2024-08-29T15:00:42.142912895Z level=info msg="Query Service initialization" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=live.push_http t=2024-08-29T15:00:42.145358912Z level=info msg="Live Push Gateway initialization" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ngalert.migration t=2024-08-29T15:00:42.146658595Z level=info msg=Starting 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ngalert.state.manager t=2024-08-29T15:00:42.154173724Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=infra.usagestats.collector t=2024-08-29T15:00:42.15562982Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=provisioning.datasources t=2024-08-29T15:00:42.158204905Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=provisioning.datasources t=2024-08-29T15:00:42.158473329Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=provisioning.alerting t=2024-08-29T15:00:42.196311666Z level=info msg="starting to provision alerting" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=provisioning.alerting t=2024-08-29T15:00:42.196329445Z level=info msg="finished to provision alerting" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafanaStorageLogger t=2024-08-29T15:00:42.196569607Z level=info msg="Storage starting" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ngalert.multiorg.alertmanager t=2024-08-29T15:00:42.196725213Z level=info msg="Starting MultiOrg Alertmanager" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ngalert.state.manager t=2024-08-29T15:00:42.196721378Z level=info msg="Warming state cache for startup" 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=http.server t=2024-08-29T15:00:42.19896257Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-08-29T15:00:42.430 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=http.server t=2024-08-29T15:00:42.199240591Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ngalert.state.manager t=2024-08-29T15:00:42.24151798Z level=info msg="State cache has been initialized" states=0 duration=44.794808ms 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ngalert.scheduler t=2024-08-29T15:00:42.241547359Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ticker t=2024-08-29T15:00:42.241593448Z level=info msg=starting first_tick=2024-08-29T15:00:50Z 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=provisioning.dashboard t=2024-08-29T15:00:42.241855143Z level=info msg="starting to provision dashboards" 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana.update.checker t=2024-08-29T15:00:42.283045714Z level=info msg="Update check succeeded" duration=86.478317ms 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugins.update.checker t=2024-08-29T15:00:42.288979803Z level=info msg="Update check succeeded" duration=92.202665ms 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=sqlstore.transactions t=2024-08-29T15:00:42.340135244Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana-apiserver t=2024-08-29T15:00:42.341237877Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana-apiserver t=2024-08-29T15:00:42.34160633Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-08-29T15:00:42.431 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:00:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=provisioning.dashboard t=2024-08-29T15:00:42.428470209Z level=info msg="finished to provision dashboards" 2024-08-29T15:00:43.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:43 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:43 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:43.401 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:43 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:44.259 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:44 smithi171 ceph-mon[22959]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:44 smithi027 ceph-mon[18892]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:44 smithi027 ceph-mon[29342]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:46 smithi027 ceph-mon[29342]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:46 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:46 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:46 smithi027 ceph-mon[18892]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:46 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:46 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:46 smithi171 ceph-mon[22959]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:46 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:46 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:47 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:47 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:47 smithi171 ceph-mon[22959]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:47 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:47 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:47 smithi027 ceph-mon[18892]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:47 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:47 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:47 smithi027 ceph-mon[29342]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:00:49] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.51.0" 2024-08-29T15:00:50.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:00:49] ENGINE Bus STOPPING 2024-08-29T15:00:50.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:00:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T15:00:50.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:00:50] ENGINE Bus STOPPED 2024-08-29T15:00:50.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:00:50] ENGINE Bus STARTING 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:00:50.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:00:50.074 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:00:50.077 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:00:50.078 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.079 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.080 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:00:50.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:50.082 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:50.082 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:50.082 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.082 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:00:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:00:50.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:00:50.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:00:50.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:00:50.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:50.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:50.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:50.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:50.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:50 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:00:50.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:00:50] ENGINE Serving on http://:::9283 2024-08-29T15:00:50.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:00:50] ENGINE Bus STARTED 2024-08-29T15:00:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T15:00:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-08-29T15:00:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: Upgrade: Finalizing container_image settings 2024-08-29T15:00:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: Upgrade: Complete! 2024-08-29T15:00:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:00:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: Upgrade: Finalizing container_image settings 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: Upgrade: Complete! 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:51 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-29T15:00:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi171:3000"}]: dispatch 2024-08-29T15:00:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: Upgrade: Finalizing container_image settings 2024-08-29T15:00:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: Upgrade: Complete! 2024-08-29T15:00:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:00:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:00:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:00:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:00:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:51 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:52 smithi027 ceph-mon[18892]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:52 smithi027 ceph-mon[29342]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:52 smithi171 ceph-mon[22959]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:53 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:53 smithi171 ceph-mon[22959]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:53 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:53 smithi027 ceph-mon[18892]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:54.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:53 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:00:54.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:53 smithi027 ceph-mon[29342]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:56 smithi027 ceph-mon[18892]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:56 smithi027 ceph-mon[29342]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:56 smithi171 ceph-mon[22959]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:58 smithi027 ceph-mon[18892]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:00:58 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:58 smithi027 ceph-mon[29342]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:00:58 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:58 smithi171 ceph-mon[22959]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:00:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:00:58 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:00:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:00:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:00:59] "GET /metrics HTTP/1.1" 200 33808 "" "Prometheus/2.51.0" 2024-08-29T15:01:00.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:00 smithi027 ceph-mon[29342]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:00 smithi027 ceph-mon[18892]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:00 smithi171 ceph-mon[22959]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:02.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:02 smithi171 ceph-mon[22959]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:02 smithi027 ceph-mon[18892]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:02 smithi027 ceph-mon[29342]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:04 smithi027 ceph-mon[18892]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:04 smithi027 ceph-mon[29342]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:04 smithi171 ceph-mon[22959]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:05.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'echo wait for minority of mons to upgrade' 2024-08-29T15:01:06.151 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:01:06.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:06 smithi027 ceph-mon[29342]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:06.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:06 smithi027 ceph-mon[18892]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:06 smithi171 ceph-mon[22959]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:07.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:07 smithi027 ceph-mon[18892]: from='client.25643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:07.298 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:07 smithi027 ceph-mon[29342]: from='client.25643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:07 smithi171 ceph-mon[22959]: from='client.25643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:07.976 INFO:teuthology.orchestra.run.smithi027.stdout:wait for minority of mons to upgrade 2024-08-29T15:01:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:08 smithi027 ceph-mon[18892]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:08 smithi027 ceph-mon[29342]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:08 smithi171 ceph-mon[22959]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:08.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-08-29T15:01:09.027 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:01:09.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:01:09] "GET /metrics HTTP/1.1" 200 33808 "" "Prometheus/2.51.0" 2024-08-29T15:01:10.114 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:10 smithi027 ceph-mon[29342]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:10.115 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:10 smithi027 ceph-mon[18892]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:10 smithi171 ceph-mon[22959]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:12 smithi027 ceph-mon[18892]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:12 smithi027 ceph-mon[18892]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:12 smithi027 ceph-mon[29342]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:12 smithi027 ceph-mon[29342]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:12 smithi171 ceph-mon[22959]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:12 smithi171 ceph-mon[22959]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:12.535 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:01:13.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-29T15:01:13.391 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:01:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:01:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[18892]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:01:13.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:13.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:13 smithi027 ceph-mon[29342]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:01:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:01:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:13 smithi171 ceph-mon[22959]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:16.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:16 smithi027 ceph-mon[29342]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:16.236 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:16 smithi027 ceph-mon[18892]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:16 smithi171 ceph-mon[22959]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:16.553 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (57s) 53s ago 6m 17.4M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (35s) 31s ago 6m 41.5M - c8b91775d855 99f2571ddaab 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (5m) 53s ago 5m 17.1M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (5m) 31s ago 5m 15.0M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (2m) 31s ago 10m 452M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (2m) 53s ago 12m 512M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (12m) 53s ago 12m 53.7M 2048M 18.2.4-774-gfe43f1c9 96384990183a 517d2f5042c9 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (11m) 31s ago 11m 48.8M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (10m) 53s ago 10m 42.0M 2048M 18.2.4-774-gfe43f1c9 96384990183a f7785aec7256 2024-08-29T15:01:17.148 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (106s) 53s ago 6m 9697k - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (100s) 31s ago 6m 10.7M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (10m) 53s ago 10m 51.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (9m) 53s ago 9m 52.1M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (9m) 53s ago 9m 56.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (9m) 53s ago 9m 46.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (8m) 31s ago 8m 54.5M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (8m) 31s ago 8m 46.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (8m) 31s ago 8m 45.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (7m) 31s ago 7m 49.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:01:17.149 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (78s) 31s ago 7m 37.2M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:01:17.502 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:17.502 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:01:17.502 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: Upgrade: Setting container_image for all mgr 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[29342]: Upgrade: It appears safe to stop mon.a 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:01:17.503 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:01:17.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:17.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:17.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: Upgrade: Setting container_image for all mgr 2024-08-29T15:01:17.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:17.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:01:17.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-29T15:01:17.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:17 smithi027 ceph-mon[18892]: Upgrade: It appears safe to stop mon.a 2024-08-29T15:01:17.504 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:01:17.504 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:01:17.504 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 3 2024-08-29T15:01:17.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 13, 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:01:17.505 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:01:17.506 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:01:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:01:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:01:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all mgr 2024-08-29T15:01:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:01:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-29T15:01:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:17 smithi171 ceph-mon[22959]: Upgrade: It appears safe to stop mon.a 2024-08-29T15:01:17.810 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:01:17.810 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:01:17.810 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:01:17.810 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-08-29T15:01:17.810 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T15:01:17.811 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/3 daemons upgraded", 2024-08-29T15:01:17.811 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-08-29T15:01:17.811 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:01:17.811 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: from='client.25655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: from='client.25661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: from='client.? 172.21.15.27:0/1513404532' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:01:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[18892]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: from='client.25655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: from='client.25661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: from='client.? 172.21.15.27:0/1513404532' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:01:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:18 smithi027 ceph-mon[29342]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: from='client.25655 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: from='client.25661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:18.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/1513404532' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:18.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='mgr.y' 2024-08-29T15:01:18.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:01:18.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:01:18.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:18 smithi171 ceph-mon[22959]: from='mgr.15591 172.21.15.27:0/4008138494' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[18892]: from='client.25667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[18892]: Upgrade: Updating mon.a 2024-08-29T15:01:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[18892]: Deploying daemon mon.a on smithi027 2024-08-29T15:01:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[18892]: from='client.25679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:19.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:01:19] "GET /metrics HTTP/1.1" 200 33819 "" "Prometheus/2.51.0" 2024-08-29T15:01:19.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[29342]: from='client.25667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:19.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[29342]: Upgrade: Updating mon.a 2024-08-29T15:01:19.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[29342]: Deploying daemon mon.a on smithi027 2024-08-29T15:01:19.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-mon[29342]: from='client.25679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:19 smithi171 ceph-mon[22959]: from='client.25667 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:19 smithi171 ceph-mon[22959]: Upgrade: Updating mon.a 2024-08-29T15:01:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:19 smithi171 ceph-mon[22959]: Deploying daemon mon.a on smithi027 2024-08-29T15:01:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:19 smithi171 ceph-mon[22959]: from='client.25679 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:19.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 systemd[1]: Stopping Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:01:19.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[18888]: 2024-08-29T15:01:19.604+0000 7f2aa348f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:01:19.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[18888]: 2024-08-29T15:01:19.604+0000 7f2aa348f640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-08-29T15:01:20.154 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:19 smithi027 podman[93761]: 2024-08-29 15:01:19.871737324 +0000 UTC m=+0.353008304 container died 517d2f5042c9097eb28678c5173890bb128f85ae39c7122b89803201d0e12591 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.1, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-29T15:01:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:20 smithi027 podman[93761]: 2024-08-29 15:01:20.331532835 +0000 UTC m=+0.812803857 container cleanup 517d2f5042c9097eb28678c5173890bb128f85ae39c7122b89803201d0e12591 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, org.label-schema.license=GPLv2) 2024-08-29T15:01:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:20 smithi027 bash[93761]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a 2024-08-29T15:01:20.968 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:20 smithi027 podman[93774]: 2024-08-29 15:01:20.616436402 +0000 UTC m=+0.739530639 container remove 517d2f5042c9097eb28678c5173890bb128f85ae39c7122b89803201d0e12591 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.1, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:01:20.968 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:20 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-29T15:01:20.968 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:20 smithi027 systemd[1]: Stopped Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:01:20.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:20 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service: Consumed 14.573s CPU time. 2024-08-29T15:01:20.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:20 smithi027 systemd[1]: Starting Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:01:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 podman[93903]: 2024-08-29 15:01:21.080958083 +0000 UTC m=+0.016849175 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:01:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 podman[93903]: 2024-08-29 15:01:21.200915582 +0000 UTC m=+0.136806662 container create 1ad77d7a522185b89efc5670afff46beacfb3567443cad40629c75249de4aaf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-29T15:01:21.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 podman[93903]: 2024-08-29 15:01:21.478664562 +0000 UTC m=+0.414555639 container init 1ad77d7a522185b89efc5670afff46beacfb3567443cad40629c75249de4aaf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-29T15:01:21.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 podman[93903]: 2024-08-29 15:01:21.482081651 +0000 UTC m=+0.417972731 container start 1ad77d7a522185b89efc5670afff46beacfb3567443cad40629c75249de4aaf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-29T15:01:21.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: set uid:gid to 167:167 (ceph:ceph) 2024-08-29T15:01:21.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev), process ceph-mon, pid 2 2024-08-29T15:01:21.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: pidfile_write: ignore empty --pid-file 2024-08-29T15:01:21.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: load: jerasure load: lrc 2024-08-29T15:01:21.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: RocksDB version: 7.9.2 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Git sha 0 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Compile date 2024-08-28 21:32:03 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: DB SUMMARY 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: DB Session ID: 1QUVPUGJHJJTPQ4HQLKR 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: CURRENT file: CURRENT 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: IDENTITY file: IDENTITY 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1511 Bytes 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 1681027 ; 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.error_if_exists: 0 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.create_if_missing: 0 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.paranoid_checks: 1 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-29T15:01:21.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-29T15:01:21.598 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-29T15:01:21.598 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.env: 0x55e17441b160 2024-08-29T15:01:21.598 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.fs: PosixFileSystem 2024-08-29T15:01:21.598 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.info_log: 0x55e175415880 2024-08-29T15:01:21.598 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_file_opening_threads: 16 2024-08-29T15:01:21.599 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.statistics: (nil) 2024-08-29T15:01:21.599 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.use_fsync: 0 2024-08-29T15:01:21.599 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_log_file_size: 0 2024-08-29T15:01:21.599 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.keep_log_file_num: 1000 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.recycle_log_file_num: 0 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.allow_fallocate: 1 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.allow_mmap_reads: 0 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.allow_mmap_writes: 0 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.use_direct_reads: 0 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.create_missing_column_families: 0 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.db_log_dir: 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.wal_dir: 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-29T15:01:21.600 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.advise_random_on_open: 1 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.db_write_buffer_size: 0 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.write_buffer_manager: 0x55e175418960 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-29T15:01:21.601 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.rate_limiter: (nil) 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.wal_recovery_mode: 2 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.enable_thread_tracking: 0 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.enable_pipelined_write: 0 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.unordered_write: 0 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-29T15:01:21.602 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.row_cache: None 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.wal_filter: None 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.allow_ingest_behind: 0 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.two_write_queues: 0 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.manual_wal_flush: 0 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.wal_compression: 0 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.atomic_flush: 0 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-29T15:01:21.603 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.log_readahead_size: 0 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.best_efforts_recovery: 0 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.allow_data_in_errors: 0 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.db_host_id: __hostname__ 2024-08-29T15:01:21.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_background_jobs: 2 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_background_compactions: -1 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_subcompactions: 1 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_total_wal_size: 0 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-29T15:01:21.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_open_files: -1 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bytes_per_sync: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_readahead_size: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_background_flushes: -1 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Compression algorithms supported: 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kZSTD supported: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kXpressCompression supported: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kBZip2Compression supported: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kLZ4Compression supported: 1 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kZlibCompression supported: 1 2024-08-29T15:01:21.606 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kLZ4HCCompression supported: 1 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: kSnappyCompression supported: 1 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.merge_operator: 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_filter: None 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_filter_factory: None 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.sst_partitioner_factory: None 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e1754154e0) 2024-08-29T15:01:21.607 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_type: 0 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_index_type: 0 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_shortening: 1 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: checksum: 4 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: no_block_cache: 0 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache: 0x55e175408850 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_options: 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: capacity : 536870912 2024-08-29T15:01:21.608 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_shard_bits : 4 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: strict_capacity_limit : 0 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_compressed: (nil) 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: persistent_cache: (nil) 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size: 4096 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size_deviation: 10 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_restart_interval: 16 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_block_restart_interval: 1 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: metadata_block_size: 4096 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: partition_filters: 0 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: use_delta_encoding: 1 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: filter_policy: bloomfilter 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: whole_key_filtering: 1 2024-08-29T15:01:21.609 INFO:journalctl@ceph.mon.a.smithi027.stdout: verify_compression: 0 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: format_version: 5 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_index_compression: 1 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_align: 0 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: max_auto_readahead_size: 262144 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: prepopulate_block_cache: 0 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: initial_auto_readahead_size: 8192 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.write_buffer_size: 33554432 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_write_buffer_number: 2 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression: NoCompression 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression: Disabled 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.prefix_extractor: nullptr 2024-08-29T15:01:21.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.num_levels: 7 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-29T15:01:21.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.level: 32767 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.strategy: 0 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.enabled: false 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.target_file_size_base: 67108864 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-29T15:01:21.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.arena_block_size: 1048576 2024-08-29T15:01:21.613 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.disable_auto_compactions: 0 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-29T15:01:21.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.inplace_update_support: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.bloom_locality: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.max_successive_merges: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.paranoid_file_checks: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.force_consistency_checks: 1 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.report_bg_io_stats: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.ttl: 2592000 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-29T15:01:21.615 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.enable_blob_files: false 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.min_blob_size: 0 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.blob_file_size: 268435456 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.blob_file_starting_level: 0 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12353, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-08-29T15:01:21.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 20dc9dcb-0469-47e4-947b-3d131c636d86 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943681509784, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943681515614, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 1513894, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12354, "largest_seqno": 13657, "table_properties": {"data_size": 1507639, "index_size": 3678, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1541, "raw_key_size": 14915, "raw_average_key_size": 24, "raw_value_size": 1494548, "raw_average_value_size": 2450, "num_data_blocks": 168, "num_entries": 610, "num_filter_entries": 610, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1724943681, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "20dc9dcb-0469-47e4-947b-3d131c636d86", "db_session_id": "1QUVPUGJHJJTPQ4HQLKR", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943681515729, "job": 1, "event": "recovery_finished"} 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e175438e00 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: DB pointer 0x55e175538000 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** DB Stats ** 2024-08-29T15:01:21.617 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: L0 1/0 1.44 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 494.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: L6 1/0 7.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Sum 2/0 9.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 494.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-29T15:01:21.618 INFO:journalctl@ceph.mon.a.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 494.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 494.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-29T15:01:21.619 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative compaction: 0.00 GB write, 136.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval compaction: 0.00 GB write, 136.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache BinnedLRUCache@0x55e175408850#2 capacity: 512.00 MB usage: 5.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: starting mon.a rank 0 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???) e3 preinit fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???).mds e9 new map 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???).mds e9 print_map 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: e9 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-29T15:01:21.620 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: legacy client fscid: 1 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: Filesystem 'foo' (1) 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: fs_name foo 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: epoch 8 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: created 2024-08-29T14:55:32.405244+0000 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: modified 2024-08-29T14:55:49.392761+0000 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: tableserver 0 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: root 0 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: session_timeout 60 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: session_autoclose 300 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: max_file_size 1099511627776 2024-08-29T15:01:21.621 INFO:journalctl@ceph.mon.a.smithi027.stdout: max_xattr_size 65536 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: required_client_features {} 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: last_failure 0 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: last_failure_osd_epoch 0 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: max_mds 1 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: in 0 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: up {0=14619} 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: failed 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: damaged 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: stopped 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_pools [4] 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: metadata_pool 3 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: inline_data disabled 2024-08-29T15:01:21.622 INFO:journalctl@ceph.mon.a.smithi027.stdout: balancer 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: bal_rank_mask -1 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: standby_count_wanted 1 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: qdb_cluster leader: 0 members: 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: [mds.foo.smithi027.afjjno{0:14619} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] compat {c=[1],r=[1],i=[7ff]}] 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: Standby daemons: 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout: [mds.foo.smithi171.grbaeh{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] compat {c=[1],r=[1],i=[7ff]}] 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 ceph-mon[93921]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-08-29T15:01:21.623 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 bash[93903]: 1ad77d7a522185b89efc5670afff46beacfb3567443cad40629c75249de4aaf6 2024-08-29T15:01:21.624 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:21 smithi027 systemd[1]: Started Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:01:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: mon.a calling monitor election 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: monmap epoch 3 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: last_changed 2024-08-29T14:50:19.801400+0000 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: min_mon_release 18 (reef) 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: election_strategy: 1 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:01:23.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: osdmap e89: 8 total, 8 up, 8 in 2024-08-29T15:01:23.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-29T15:01:23.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: overall HEALTH_OK 2024-08-29T15:01:23.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: from='mgr.15591 ' entity='' 2024-08-29T15:01:23.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:22 smithi171 ceph-mon[22959]: mgrmap e36: y(active, since 2m), standbys: x 2024-08-29T15:01:23.269 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: ignoring --setuser ceph since I am not root 2024-08-29T15:01:23.269 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: ignoring --setgroup ceph since I am not root 2024-08-29T15:01:23.269 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:23 smithi171 ceph-mgr[68070]: -- 172.21.15.171:0/1695845156 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dcea7590e0 con 0x55dcea736800 2024-08-29T15:01:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: mon.a calling monitor election 2024-08-29T15:01:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: monmap epoch 3 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: last_changed 2024-08-29T14:50:19.801400+0000 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: min_mon_release 18 (reef) 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: election_strategy: 1 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: osdmap e89: 8 total, 8 up, 8 in 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: overall HEALTH_OK 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: from='mgr.15591 ' entity='' 2024-08-29T15:01:23.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[93921]: mgrmap e36: y(active, since 2m), standbys: x 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:23 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ignoring --setuser ceph since I am not root 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:23 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ignoring --setgroup ceph since I am not root 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:23 smithi027 ceph-mgr[75524]: -- 172.21.15.27:0/1505805555 <== mon.2 v2:172.21.15.27:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x55d934019520 con 0x55d933fec800 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:23 smithi027 ceph-mgr[75524]: -- 172.21.15.27:0/1505805555 <== mon.2 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d93400f0e0 con 0x55d933fec800 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: mon.a calling monitor election 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:01:23.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: monmap epoch 3 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: last_changed 2024-08-29T14:50:19.801400+0000 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: min_mon_release 18 (reef) 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: election_strategy: 1 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: osdmap e89: 8 total, 8 up, 8 in 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: overall HEALTH_OK 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: from='mgr.15591 ' entity='' 2024-08-29T15:01:23.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:22 smithi027 ceph-mon[29342]: mgrmap e36: y(active, since 2m), standbys: x 2024-08-29T15:01:28.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: [29/Aug/2024:15:01:27] ENGINE Bus STARTING 2024-08-29T15:01:28.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: CherryPy Checker: 2024-08-29T15:01:28.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: The Application mounted at '' has an empty config. 2024-08-29T15:01:28.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: 2024-08-29T15:01:28.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: [29/Aug/2024:15:01:28] ENGINE Serving on http://:::9283 2024-08-29T15:01:28.267 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x[68066]: [29/Aug/2024:15:01:28] ENGINE Bus STARTED 2024-08-29T15:01:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-mon[22959]: Standby manager daemon x restarted 2024-08-29T15:01:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-mon[22959]: Standby manager daemon x started 2024-08-29T15:01:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T15:01:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T15:01:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T15:01:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:28 smithi171 ceph-mon[22959]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[93921]: Standby manager daemon x restarted 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[93921]: Standby manager daemon x started 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[93921]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[93921]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[93921]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[93921]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[29342]: Standby manager daemon x restarted 2024-08-29T15:01:28.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[29342]: Standby manager daemon x started 2024-08-29T15:01:28.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-29T15:01:28.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-29T15:01:28.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-29T15:01:28.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-mon[29342]: from='mgr.? 172.21.15.171:0/387852395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-29T15:01:29.043 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.704+0000 7f7dbae4d640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.043 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.704+0000 7f7dbae4d640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.704+0000 7f7dbae4d640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.704+0000 7f7dbae4d640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.704+0000 7f7dbae4d640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.706+0000 7f7db8648640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.706+0000 7f7db8648640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.706+0000 7f7db8648640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.706+0000 7f7db8648640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.044 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:28.706+0000 7f7db8648640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-29T15:01:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: mgrmap e37: y(active, since 2m), standbys: x 2024-08-29T15:01:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: Active manager daemon y restarted 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: Activating manager daemon y 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: mgrmap e38: y(active, starting, since 0.00727571s), standbys: x 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T15:01:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: Manager daemon y is now available 2024-08-29T15:01:29.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:29.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:29.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:29.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T15:01:29.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-29T15:01:29.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T15:01:29.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:29 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-29T15:01:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:29.208+0000 7f7dea97a640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-29T15:01:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:29] ENGINE Bus STARTING 2024-08-29T15:01:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: CherryPy Checker: 2024-08-29T15:01:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: The Application mounted at '' has an empty config. 2024-08-29T15:01:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: mgrmap e37: y(active, since 2m), standbys: x 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: Active manager daemon y restarted 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: Activating manager daemon y 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: mgrmap e38: y(active, starting, since 0.00727571s), standbys: x 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T15:01:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T15:01:29.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: Manager daemon y is now available 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-29T15:01:29.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: mgrmap e37: y(active, since 2m), standbys: x 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: Active manager daemon y restarted 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: Activating manager daemon y 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: mgrmap e38: y(active, starting, since 0.00727571s), standbys: x 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.afjjno"}]: dispatch 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi171.grbaeh"}]: dispatch 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-29T15:01:29.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: Manager daemon y is now available 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:29.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:29.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-29T15:01:29.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-29T15:01:29.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-29T15:01:29.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-29T15:01:29.693 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:29] ENGINE Serving on http://:::9283 2024-08-29T15:01:29.693 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:29] ENGINE Bus STARTED 2024-08-29T15:01:30.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:29.738+0000 7f7d9e49a640 -1 log_channel(cephadm) log [ERR] : [29/Aug/2024:15:01:29] ENGINE Error in HTTPServer.serve 2024-08-29T15:01:30.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: Traceback (most recent call last): 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self._connections.run(self.expiration_interval) 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self._run(expiration_interval) 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: s = self.context.wrap_socket( 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: return self.sslsocket_class._create( 2024-08-29T15:01:30.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T15:01:30.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self.do_handshake() 2024-08-29T15:01:30.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T15:01:30.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: self._sslobj.do_handshake() 2024-08-29T15:01:30.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T15:01:30.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:30 smithi171 ceph-mon[22959]: mgrmap e39: y(active, since 1.01228s), standbys: x 2024-08-29T15:01:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:30 smithi171 ceph-mon[22959]: [29/Aug/2024:15:01:29] ENGINE Bus STARTING 2024-08-29T15:01:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:30 smithi171 ceph-mon[22959]: [29/Aug/2024:15:01:29] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T15:01:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:30 smithi171 ceph-mon[22959]: [29/Aug/2024:15:01:29] ENGINE Error in HTTPServer.serve 2024-08-29T15:01:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout: Traceback (most recent call last): 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._connections.run(self.expiration_interval) 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._run(expiration_interval) 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: s = self.context.wrap_socket( 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: return self.sslsocket_class._create( 2024-08-29T15:01:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T15:01:30.519 INFO:journalctl@ceph.mon.b.smithi171.stdout: self.do_handshake() 2024-08-29T15:01:30.519 INFO:journalctl@ceph.mon.b.smithi171.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T15:01:30.519 INFO:journalctl@ceph.mon.b.smithi171.stdout: self._sslobj.do_handshake() 2024-08-29T15:01:30.519 INFO:journalctl@ceph.mon.b.smithi171.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T15:01:30.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:30 smithi171 ceph-mon[22959]: [29/Aug/2024:15:01:29] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T15:01:30.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:30 smithi171 ceph-mon[22959]: [29/Aug/2024:15:01:29] ENGINE Bus STARTED 2024-08-29T15:01:30.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[29342]: mgrmap e39: y(active, since 1.01228s), standbys: x 2024-08-29T15:01:30.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[29342]: [29/Aug/2024:15:01:29] ENGINE Bus STARTING 2024-08-29T15:01:30.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[29342]: [29/Aug/2024:15:01:29] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T15:01:30.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[29342]: [29/Aug/2024:15:01:29] ENGINE Error in HTTPServer.serve 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T15:01:30.559 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[29342]: [29/Aug/2024:15:01:29] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[29342]: [29/Aug/2024:15:01:29] ENGINE Bus STARTED 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[93921]: mgrmap e39: y(active, since 1.01228s), standbys: x 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[93921]: [29/Aug/2024:15:01:29] ENGINE Bus STARTING 2024-08-29T15:01:30.560 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[93921]: [29/Aug/2024:15:01:29] ENGINE Serving on https://172.21.15.27:7150 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[93921]: [29/Aug/2024:15:01:29] ENGINE Error in HTTPServer.serve 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-08-29T15:01:30.561 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[93921]: [29/Aug/2024:15:01:29] ENGINE Serving on http://172.21.15.27:8765 2024-08-29T15:01:30.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:30 smithi027 ceph-mon[93921]: [29/Aug/2024:15:01:29] ENGINE Bus STARTED 2024-08-29T15:01:31.473 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:31 smithi171 ceph-mon[22959]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:31.473 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:31 smithi171 ceph-mon[22959]: mgrmap e40: y(active, since 2s), standbys: x 2024-08-29T15:01:31.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:31 smithi027 ceph-mon[29342]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:31.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:31 smithi027 ceph-mon[29342]: mgrmap e40: y(active, since 2s), standbys: x 2024-08-29T15:01:31.532 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:31 smithi027 ceph-mon[93921]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:31.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:31 smithi027 ceph-mon[93921]: mgrmap e40: y(active, since 2s), standbys: x 2024-08-29T15:01:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: mgrmap e41: y(active, since 4s), standbys: x 2024-08-29T15:01:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: mgrmap e41: y(active, since 4s), standbys: x 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: mgrmap e41: y(active, since 4s), standbys: x 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:33.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:34.929 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:34 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:34.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:34 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:34.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:34 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:34.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:34 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:34 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:34 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:35.993 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:35 smithi027 ceph-mon[29342]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:35.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:35 smithi027 ceph-mon[93921]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:35 smithi171 ceph-mon[22959]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-29T15:01:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:37.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T15:01:37.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:37 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T15:01:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-29T15:01:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[93921]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.conf 2024-08-29T15:01:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:37 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.conf 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.838 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[29342]: Upgrade: It appears safe to stop mon.c 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T15:01:38.839 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-29T15:01:38.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:38 smithi027 ceph-mon[93921]: Upgrade: It appears safe to stop mon.c 2024-08-29T15:01:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T15:01:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-29T15:01:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: Updating smithi171:/etc/ceph/ceph.client.admin.keyring 2024-08-29T15:01:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: Updating smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: Updating smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-29T15:01:39.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:38 smithi171 ceph-mon[22959]: Upgrade: It appears safe to stop mon.c 2024-08-29T15:01:39.293 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:01:39] "GET /metrics HTTP/1.1" 200 31126 "" "Prometheus/2.51.0" 2024-08-29T15:01:40.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[29342]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-29T15:01:40.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[29342]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:40.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:01:40.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:01:40.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[29342]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:40.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[93921]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-29T15:01:40.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:40.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:01:40.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:01:40.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:39 smithi171 ceph-mon[22959]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-08-29T15:01:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:39 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:39 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:01:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:39 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:01:40.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:39 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:41.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:40 smithi027 ceph-mon[93921]: Upgrade: Updating mon.c 2024-08-29T15:01:41.084 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:40 smithi027 ceph-mon[93921]: Deploying daemon mon.c on smithi027 2024-08-29T15:01:41.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:40 smithi027 ceph-mon[29342]: Upgrade: Updating mon.c 2024-08-29T15:01:41.084 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:40 smithi027 ceph-mon[29342]: Deploying daemon mon.c on smithi027 2024-08-29T15:01:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:40 smithi171 ceph-mon[22959]: Upgrade: Updating mon.c 2024-08-29T15:01:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:40 smithi171 ceph-mon[22959]: Deploying daemon mon.c on smithi027 2024-08-29T15:01:41.470 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:41 smithi027 systemd[1]: Stopping Ceph mon.c for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:01:41.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c[29338]: 2024-08-29T15:01:41.467+0000 7f0911b60640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:01:41.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c[29338]: 2024-08-29T15:01:41.467+0000 7f0911b60640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-08-29T15:01:41.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:41 smithi027 podman[98798]: 2024-08-29 15:01:41.590664973 +0000 UTC m=+0.246103353 container died f7785aec7256bbbfa602bd49c67d79e1ea78aba38fb3e94d5d95f2b55b33e795 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, RELEASE=reef-fe43f1c, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-29T15:01:41.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:41] ENGINE Bus STOPPING 2024-08-29T15:01:41.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T15:01:41.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:41] ENGINE Bus STOPPED 2024-08-29T15:01:41.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:41] ENGINE Bus STARTING 2024-08-29T15:01:41.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:41] ENGINE Serving on http://:::9283 2024-08-29T15:01:41.819 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:41 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:01:41] ENGINE Bus STARTED 2024-08-29T15:01:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:41 smithi027 podman[98798]: 2024-08-29 15:01:41.926214475 +0000 UTC m=+0.581652838 container cleanup f7785aec7256bbbfa602bd49c67d79e1ea78aba38fb3e94d5d95f2b55b33e795 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.1, GIT_BRANCH=HEAD) 2024-08-29T15:01:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:41 smithi027 bash[98798]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c 2024-08-29T15:01:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 podman[98822]: 2024-08-29 15:01:42.102752597 +0000 UTC m=+0.506997128 container remove f7785aec7256bbbfa602bd49c67d79e1ea78aba38fb3e94d5d95f2b55b33e795 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, GIT_BRANCH=HEAD) 2024-08-29T15:01:42.773 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.c.service: Deactivated successfully. 2024-08-29T15:01:42.773 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 systemd[1]: Stopped Ceph mon.c for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:01:42.773 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.c.service: Consumed 8.082s CPU time. 2024-08-29T15:01:42.773 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 systemd[1]: Starting Ceph mon.c for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:01:43.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 podman[98951]: 2024-08-29 15:01:42.675766504 +0000 UTC m=+0.016432202 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:01:43.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 podman[98951]: 2024-08-29 15:01:42.779293671 +0000 UTC m=+0.119959418 container create 78f3772b32b4604923da25350d8abc84dbb958fe1e8b2a9f1fad68ea3800e4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:01:43.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 podman[98951]: 2024-08-29 15:01:42.965002778 +0000 UTC m=+0.305668470 container init 78f3772b32b4604923da25350d8abc84dbb958fe1e8b2a9f1fad68ea3800e4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True) 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 podman[98951]: 2024-08-29 15:01:42.968421112 +0000 UTC m=+0.309086790 container start 78f3772b32b4604923da25350d8abc84dbb958fe1e8b2a9f1fad68ea3800e4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, org.label-schema.build-date=20240731, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: set uid:gid to 167:167 (ceph:ceph) 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev), process ceph-mon, pid 2 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: pidfile_write: ignore empty --pid-file 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: load: jerasure load: lrc 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: RocksDB version: 7.9.2 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Git sha 0 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Compile date 2024-08-28 21:32:03 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: DB SUMMARY 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: DB Session ID: JM4PZWWMWOZRJV684FXB 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: CURRENT file: CURRENT 2024-08-29T15:01:43.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: IDENTITY file: IDENTITY 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1404 Bytes 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 5607675 ; 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.error_if_exists: 0 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.create_if_missing: 0 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.paranoid_checks: 1 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-29T15:01:43.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.env: 0x55f82ba87160 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.fs: PosixFileSystem 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.info_log: 0x55f82d44b880 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_file_opening_threads: 16 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.statistics: (nil) 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.use_fsync: 0 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_log_file_size: 0 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.keep_log_file_num: 1000 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.recycle_log_file_num: 0 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.allow_fallocate: 1 2024-08-29T15:01:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.allow_mmap_reads: 0 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.allow_mmap_writes: 0 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.use_direct_reads: 0 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.create_missing_column_families: 0 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.db_log_dir: 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.wal_dir: 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-29T15:01:43.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.advise_random_on_open: 1 2024-08-29T15:01:43.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.db_write_buffer_size: 0 2024-08-29T15:01:43.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.write_buffer_manager: 0x55f82d44e960 2024-08-29T15:01:43.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-29T15:01:43.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-29T15:01:43.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-29T15:01:43.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.rate_limiter: (nil) 2024-08-29T15:01:43.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-29T15:01:43.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.wal_recovery_mode: 2 2024-08-29T15:01:43.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.enable_thread_tracking: 0 2024-08-29T15:01:43.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.enable_pipelined_write: 0 2024-08-29T15:01:43.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.unordered_write: 0 2024-08-29T15:01:43.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-29T15:01:43.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.row_cache: None 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.wal_filter: None 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.allow_ingest_behind: 0 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.two_write_queues: 0 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.manual_wal_flush: 0 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.wal_compression: 0 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.atomic_flush: 0 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-29T15:01:43.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.log_readahead_size: 0 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.best_efforts_recovery: 0 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.allow_data_in_errors: 0 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.db_host_id: __hostname__ 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_background_jobs: 2 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_background_compactions: -1 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_subcompactions: 1 2024-08-29T15:01:43.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_total_wal_size: 0 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_open_files: -1 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bytes_per_sync: 0 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_readahead_size: 0 2024-08-29T15:01:43.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_background_flushes: -1 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Compression algorithms supported: 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kZSTD supported: 0 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kXpressCompression supported: 0 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kBZip2Compression supported: 0 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kLZ4Compression supported: 1 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kZlibCompression supported: 1 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kLZ4HCCompression supported: 1 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: kSnappyCompression supported: 1 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-29T15:01:43.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.merge_operator: 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_filter: None 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_filter_factory: None 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.sst_partitioner_factory: None 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f82d44b4e0) 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_type: 0 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_index_type: 0 2024-08-29T15:01:43.052 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_shortening: 1 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: checksum: 4 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: no_block_cache: 0 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache: 0x55f82d43e850 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_options: 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: capacity : 536870912 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_shard_bits : 4 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: strict_capacity_limit : 0 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_compressed: (nil) 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: persistent_cache: (nil) 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size: 4096 2024-08-29T15:01:43.053 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size_deviation: 10 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_restart_interval: 16 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_block_restart_interval: 1 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: metadata_block_size: 4096 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: partition_filters: 0 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: use_delta_encoding: 1 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: filter_policy: bloomfilter 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: whole_key_filtering: 1 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: verify_compression: 0 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: format_version: 5 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_index_compression: 1 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_align: 0 2024-08-29T15:01:43.054 INFO:journalctl@ceph.mon.c.smithi027.stdout: max_auto_readahead_size: 262144 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout: prepopulate_block_cache: 0 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout: initial_auto_readahead_size: 8192 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.write_buffer_size: 33554432 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_write_buffer_number: 2 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression: NoCompression 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression: Disabled 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.prefix_extractor: nullptr 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.num_levels: 7 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-29T15:01:43.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.level: 32767 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.strategy: 0 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-29T15:01:43.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.enabled: false 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.target_file_size_base: 67108864 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-29T15:01:43.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.arena_block_size: 1048576 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-29T15:01:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.disable_auto_compactions: 0 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.inplace_update_support: 0 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-29T15:01:43.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.bloom_locality: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.max_successive_merges: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.paranoid_file_checks: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.force_consistency_checks: 1 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.report_bg_io_stats: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.ttl: 2592000 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-29T15:01:43.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.enable_blob_files: false 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.min_blob_size: 0 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.blob_file_size: 268435456 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.blob_file_starting_level: 0 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12953, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-29T15:01:43.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: db5d8f7d-b07b-4fd8-90bb-a6d33af7132d 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943702995918, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:42 smithi027 ceph-mon[98970]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943703008759, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3468315, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12959, "largest_seqno": 14877, "table_properties": {"data_size": 3459916, "index_size": 5241, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2117, "raw_key_size": 19838, "raw_average_key_size": 24, "raw_value_size": 3442292, "raw_average_value_size": 4192, "num_data_blocks": 238, "num_entries": 821, "num_filter_entries": 821, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1724943703, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "db5d8f7d-b07b-4fd8-90bb-a6d33af7132d", "db_session_id": "JM4PZWWMWOZRJV684FXB", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943703008906, "job": 1, "event": "recovery_finished"} 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f82d46ee00 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: DB pointer 0x55f82d56e000 2024-08-29T15:01:43.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** DB Stats ** 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T15:01:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: L0 1/0 3.31 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 700.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: L6 1/0 7.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: Sum 2/0 11.21 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 700.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 700.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 700.8 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-29T15:01:43.064 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative compaction: 0.00 GB write, 190.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval compaction: 0.00 GB write, 190.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache BinnedLRUCache@0x55f82d43e850#2 capacity: 512.00 MB usage: 8.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.065 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: starting mon.c rank 2 at public addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] at bind addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???) e3 preinit fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???).mds e9 new map 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???).mds e9 print_map 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: e9 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: legacy client fscid: 1 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: Filesystem 'foo' (1) 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: fs_name foo 2024-08-29T15:01:43.066 INFO:journalctl@ceph.mon.c.smithi027.stdout: epoch 8 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: created 2024-08-29T14:55:32.405244+0000 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: modified 2024-08-29T14:55:49.392761+0000 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: tableserver 0 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: root 0 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: session_timeout 60 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: session_autoclose 300 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: max_file_size 1099511627776 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: max_xattr_size 65536 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: required_client_features {} 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: last_failure 0 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: last_failure_osd_epoch 0 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T15:01:43.067 INFO:journalctl@ceph.mon.c.smithi027.stdout: max_mds 1 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: in 0 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: up {0=14619} 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: failed 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: damaged 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: stopped 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_pools [4] 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: metadata_pool 3 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: inline_data disabled 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: balancer 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: bal_rank_mask -1 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: standby_count_wanted 1 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: qdb_cluster leader: 0 members: 2024-08-29T15:01:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout: [mds.foo.smithi027.afjjno{0:14619} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] compat {c=[1],r=[1],i=[7ff]}] 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: Standby daemons: 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout: [mds.foo.smithi171.grbaeh{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] compat {c=[1],r=[1],i=[7ff]}] 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 bash[98951]: 78f3772b32b4604923da25350d8abc84dbb958fe1e8b2a9f1fad68ea3800e4ed 2024-08-29T15:01:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 systemd[1]: Started Ceph mon.c for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:01:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-29T15:01:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T15:01:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T15:01:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T15:01:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[93921]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-29T15:01:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T15:01:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T15:01:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T15:01:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:43 smithi171 ceph-mon[22959]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-08-29T15:01:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:43 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-29T15:01:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:43 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-29T15:01:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:43 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-29T15:01:43.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:43 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: mon.c calling monitor election 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: mon.a calling monitor election 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: monmap epoch 3 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: last_changed 2024-08-29T14:50:19.801400+0000 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: min_mon_release 18 (reef) 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: election_strategy: 1 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: mgrmap e41: y(active, since 15s), standbys: x 2024-08-29T15:01:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: overall HEALTH_OK 2024-08-29T15:01:44.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:44.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:44 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: mon.c calling monitor election 2024-08-29T15:01:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: mon.a calling monitor election 2024-08-29T15:01:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:01:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: monmap epoch 3 2024-08-29T15:01:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: last_changed 2024-08-29T14:50:19.801400+0000 2024-08-29T15:01:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:01:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: min_mon_release 18 (reef) 2024-08-29T15:01:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: election_strategy: 1 2024-08-29T15:01:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:01:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:01:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:01:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:01:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:01:44.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: mgrmap e41: y(active, since 15s), standbys: x 2024-08-29T15:01:44.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: overall HEALTH_OK 2024-08-29T15:01:44.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:44.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:44.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-29T15:01:44.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:44.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: mon.c calling monitor election 2024-08-29T15:01:44.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: mon.a calling monitor election 2024-08-29T15:01:44.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: monmap epoch 3 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: last_changed 2024-08-29T14:50:19.801400+0000 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: min_mon_release 18 (reef) 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: election_strategy: 1 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:01:44.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: mgrmap e41: y(active, since 15s), standbys: x 2024-08-29T15:01:44.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: overall HEALTH_OK 2024-08-29T15:01:44.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:44.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:44.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:45.500 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:45 smithi027 ceph-mon[98970]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:45.500 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:45 smithi027 ceph-mon[93921]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:45 smithi171 ceph-mon[22959]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:47.708 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[93921]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[98970]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:47 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:47 smithi171 ceph-mon[22959]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-08-29T15:01:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:47 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:47 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:47 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:47 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:48.129 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:01:48.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:48 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:01:48.021+0000 7f7dea97a640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-08-29T15:01:48.735 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (89s) 2s ago 7m 24.2M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (67s) 15s ago 7m 74.4M - c8b91775d855 99f2571ddaab 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (6m) 2s ago 6m 18.4M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (6m) 15s ago 6m 15.3M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (2m) 15s ago 11m 451M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (3m) 2s ago 12m 501M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (27s) 2s ago 13m 40.1M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 1ad77d7a5221 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (11m) 15s ago 11m 55.5M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6s) 2s ago 11m 25.2M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 78f3772b32b4 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 2s ago 6m 12.2M - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (2m) 15s ago 6m 11.2M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (10m) 2s ago 10m 52.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (10m) 2s ago 10m 52.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:01:48.736 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (10m) 2s ago 10m 57.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:01:48.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (9m) 2s ago 9m 46.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:01:48.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (9m) 15s ago 9m 54.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:01:48.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (8m) 15s ago 8m 47.2M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:01:48.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (8m) 15s ago 8m 46.5M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:01:48.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (8m) 15s ago 8m 50.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:01:48.737 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (110s) 15s ago 7m 43.6M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:01:49.013 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:01:49] "GET /metrics HTTP/1.1" 200 33746 "" "Prometheus/2.51.0" 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 1, 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:01:49.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 11, 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 4 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:01:49.093 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/3 daemons upgraded", 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:01:49.397 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:01:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[93921]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[93921]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[93921]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[93921]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/648636238' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:01:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[98970]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[98970]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[98970]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[98970]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/648636238' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:01:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:50 smithi171 ceph-mon[22959]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:50 smithi171 ceph-mon[22959]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:50 smithi171 ceph-mon[22959]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:50 smithi171 ceph-mon[22959]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:50 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/648636238' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:01:51.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:51.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:01:51.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:01:51.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:01:51.328 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.330 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:01:51.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.334 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:51 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-29T15:01:51.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:01:51.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:01:51.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.523 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:01:51.524 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:01:51.525 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:51 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all grafana 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all loki 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all promtail 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Finalizing container_image settings 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[93921]: Upgrade: Complete! 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:01:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:01:52.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:01:52.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:01:52.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all grafana 2024-08-29T15:01:52.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all loki 2024-08-29T15:01:52.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all promtail 2024-08-29T15:01:52.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Finalizing container_image settings 2024-08-29T15:01:52.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:52 smithi027 ceph-mon[98970]: Upgrade: Complete! 2024-08-29T15:01:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:01:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:01:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:01:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:01:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all grafana 2024-08-29T15:01:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all loki 2024-08-29T15:01:52.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all promtail 2024-08-29T15:01:52.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Finalizing container_image settings 2024-08-29T15:01:52.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:52 smithi171 ceph-mon[22959]: Upgrade: Complete! 2024-08-29T15:01:53.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:53 smithi027 ceph-mon[93921]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:53.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:53 smithi027 ceph-mon[98970]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:53.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:53 smithi171 ceph-mon[22959]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:54.687 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:54 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:55.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:54 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:55.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:54 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:01:56.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:55 smithi171 ceph-mon[22959]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:55 smithi027 ceph-mon[93921]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:55 smithi027 ceph-mon[98970]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:57 smithi171 ceph-mon[22959]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:57 smithi027 ceph-mon[93921]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:58.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:57 smithi027 ceph-mon[98970]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:01:59.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:59.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:58 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:01:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:01:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:01:59] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.51.0" 2024-08-29T15:02:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:01:59 smithi171 ceph-mon[22959]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:01:59 smithi027 ceph-mon[93921]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:01:59 smithi027 ceph-mon[98970]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:01 smithi171 ceph-mon[22959]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:01 smithi027 ceph-mon[93921]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:01 smithi027 ceph-mon[98970]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:03 smithi171 ceph-mon[22959]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:03 smithi027 ceph-mon[93921]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:03 smithi027 ceph-mon[98970]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:04.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:02:04 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=infra.usagestats t=2024-08-29T15:02:04.203323719Z level=info msg="Usage stats are ready to report" 2024-08-29T15:02:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:05 smithi171 ceph-mon[22959]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:05 smithi027 ceph-mon[93921]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:06.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:05 smithi027 ceph-mon[98970]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:07 smithi171 ceph-mon[22959]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:08.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:07 smithi027 ceph-mon[93921]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:07 smithi027 ceph-mon[98970]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:02:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:02:09] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.51.0" 2024-08-29T15:02:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:09 smithi171 ceph-mon[22959]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:09 smithi027 ceph-mon[93921]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:10.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:09 smithi027 ceph-mon[98970]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:11 smithi171 ceph-mon[22959]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:12.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:11 smithi027 ceph-mon[93921]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:12.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:11 smithi027 ceph-mon[98970]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:13 smithi171 ceph-mon[22959]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:13 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:13 smithi027 ceph-mon[93921]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:13 smithi027 ceph-mon[98970]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:15 smithi171 ceph-mon[22959]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:16.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:15 smithi027 ceph-mon[93921]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:15 smithi027 ceph-mon[98970]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:17 smithi171 ceph-mon[22959]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:17 smithi027 ceph-mon[93921]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:17 smithi027 ceph-mon[98970]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:02:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:02:19] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.51.0" 2024-08-29T15:02:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:19 smithi171 ceph-mon[22959]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:20.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:19 smithi027 ceph-mon[98970]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:20.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:19 smithi027 ceph-mon[93921]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:20.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'sleep 60' 2024-08-29T15:02:20.669 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:02:20.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:20 smithi027 ceph-mon[98970]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:02:20.961 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:20 smithi027 ceph-mon[93921]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:02:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:20 smithi171 ceph-mon[22959]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:02:21.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:21 smithi027 ceph-mon[93921]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:21.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:21 smithi027 ceph-mon[98970]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:21 smithi171 ceph-mon[22959]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:23.851 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:23 smithi027 ceph-mon[98970]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:23.851 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:23 smithi027 ceph-mon[93921]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:23 smithi171 ceph-mon[22959]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:25 smithi171 ceph-mon[22959]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:25 smithi027 ceph-mon[93921]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:26.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:25 smithi027 ceph-mon[98970]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:27 smithi171 ceph-mon[22959]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:27 smithi027 ceph-mon[93921]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:27 smithi027 ceph-mon[98970]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:29.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:29.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:28 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:02:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:02:29] "GET /metrics HTTP/1.1" 200 33810 "" "Prometheus/2.51.0" 2024-08-29T15:02:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:29 smithi171 ceph-mon[22959]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:29 smithi027 ceph-mon[98970]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:29 smithi027 ceph-mon[93921]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:31 smithi171 ceph-mon[22959]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:31 smithi027 ceph-mon[98970]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:31 smithi027 ceph-mon[93921]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:33 smithi171 ceph-mon[22959]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:33 smithi027 ceph-mon[93921]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:33 smithi027 ceph-mon[98970]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:35 smithi171 ceph-mon[22959]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:35 smithi027 ceph-mon[93921]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:36.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:35 smithi027 ceph-mon[98970]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:37 smithi171 ceph-mon[22959]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:37 smithi027 ceph-mon[93921]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:37 smithi027 ceph-mon[98970]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:02:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:02:39] "GET /metrics HTTP/1.1" 200 33810 "" "Prometheus/2.51.0" 2024-08-29T15:02:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:39 smithi171 ceph-mon[22959]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:39 smithi027 ceph-mon[93921]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:39 smithi027 ceph-mon[98970]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:41 smithi171 ceph-mon[22959]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:41 smithi027 ceph-mon[93921]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:41 smithi027 ceph-mon[98970]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:43 smithi171 ceph-mon[22959]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:43 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:43 smithi027 ceph-mon[93921]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:43 smithi027 ceph-mon[98970]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:45 smithi171 ceph-mon[22959]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:45 smithi027 ceph-mon[93921]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:45 smithi027 ceph-mon[98970]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:47 smithi171 ceph-mon[22959]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:47 smithi027 ceph-mon[93921]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:47 smithi027 ceph-mon[98970]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:02:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:02:49] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-08-29T15:02:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:49 smithi171 ceph-mon[22959]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:49 smithi027 ceph-mon[93921]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:50.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:49 smithi027 ceph-mon[98970]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:51.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:50 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:02:51.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:50 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:02:51.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:50 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:02:52.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:51 smithi171 ceph-mon[22959]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:51 smithi027 ceph-mon[93921]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:51 smithi027 ceph-mon[98970]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:53 smithi171 ceph-mon[22959]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:53 smithi027 ceph-mon[93921]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:54.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:53 smithi027 ceph-mon[98970]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:02:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:02:55.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:55 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:02:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:02:55.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:55 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:02:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:57 smithi171 ceph-mon[22959]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:57 smithi027 ceph-mon[93921]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:57 smithi027 ceph-mon[98970]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:59 smithi027 ceph-mon[98970]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:02:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:59 smithi027 ceph-mon[93921]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:02:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:02:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:02:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:02:59] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.51.0" 2024-08-29T15:02:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:59 smithi171 ceph-mon[22959]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:02:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:02:59 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:03:01.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:01 smithi171 ceph-mon[22959]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:01.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:01 smithi171 ceph-mon[22959]: mgrmap e42: y(active, since 92s), standbys: x 2024-08-29T15:03:01.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:01 smithi027 ceph-mon[93921]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:01.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:01 smithi027 ceph-mon[93921]: mgrmap e42: y(active, since 92s), standbys: x 2024-08-29T15:03:01.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:01 smithi027 ceph-mon[98970]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:01.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:01 smithi027 ceph-mon[98970]: mgrmap e42: y(active, since 92s), standbys: x 2024-08-29T15:03:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:03 smithi171 ceph-mon[22959]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:03 smithi027 ceph-mon[93921]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:03.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:03 smithi027 ceph-mon[98970]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:05 smithi171 ceph-mon[22959]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:05.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:05 smithi027 ceph-mon[93921]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:05.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:05 smithi027 ceph-mon[98970]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:07 smithi171 ceph-mon[22959]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:07.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:07 smithi027 ceph-mon[93921]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:07.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:07 smithi027 ceph-mon[98970]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:09.265 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:03:09] "GET /metrics HTTP/1.1" 200 33809 "" "Prometheus/2.51.0" 2024-08-29T15:03:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:09 smithi171 ceph-mon[22959]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:09 smithi027 ceph-mon[93921]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:09.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:09 smithi027 ceph-mon[98970]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:11.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:11 smithi027 ceph-mon[93921]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:11 smithi027 ceph-mon[98970]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:11 smithi171 ceph-mon[22959]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-29T15:03:13.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:13 smithi027 ceph-mon[93921]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:13.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:13 smithi027 ceph-mon[98970]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:13 smithi171 ceph-mon[22959]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:03:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:03:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:14 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:03:15.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:15 smithi027 ceph-mon[98970]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:15.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:15 smithi027 ceph-mon[93921]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:15 smithi171 ceph-mon[22959]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:17 smithi027 ceph-mon[93921]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:17.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:17 smithi027 ceph-mon[98970]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:17 smithi171 ceph-mon[22959]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:19.286 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:03:19] "GET /metrics HTTP/1.1" 200 33806 "" "Prometheus/2.51.0" 2024-08-29T15:03:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:19 smithi027 ceph-mon[93921]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:19.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:19 smithi027 ceph-mon[98970]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:19 smithi171 ceph-mon[22959]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:21.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:21 smithi027 ceph-mon[93921]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:21 smithi027 ceph-mon[98970]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:21 smithi171 ceph-mon[22959]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:23.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'echo wait for majority of mons to upgrade' 2024-08-29T15:03:23.275 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:03:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:23 smithi027 ceph-mon[93921]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:23 smithi027 ceph-mon[98970]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:23.569 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:23 smithi171 ceph-mon[22959]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:24.856 INFO:teuthology.orchestra.run.smithi027.stdout:wait for majority of mons to upgrade 2024-08-29T15:03:25.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:25 smithi027 ceph-mon[93921]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:25.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:25 smithi027 ceph-mon[98970]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:25.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-08-29T15:03:25.727 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:03:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:25 smithi171 ceph-mon[22959]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:27.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:27 smithi027 ceph-mon[98970]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:27 smithi027 ceph-mon[93921]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:27 smithi171 ceph-mon[22959]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:28.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:28 smithi027 ceph-mon[93921]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:28 smithi027 ceph-mon[98970]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:28 smithi171 ceph-mon[22959]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:29.305 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:03:29] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-08-29T15:03:29.370 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:03:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:29 smithi027 ceph-mon[98970]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:03:29.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:29 smithi027 ceph-mon[93921]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:29.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:03:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:29 smithi171 ceph-mon[22959]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:29 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-29T15:03:30.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-29T15:03:30.225 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:03:30.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[93921]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:03:30.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:30.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:03:30.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[93921]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[98970]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:30.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:30 smithi027 ceph-mon[98970]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:03:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:30 smithi171 ceph-mon[22959]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:03:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:30 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:30 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:03:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:30 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:03:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:30 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-29T15:03:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:30 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:30 smithi171 ceph-mon[22959]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:03:31.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:31 smithi027 ceph-mon[93921]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:31.723 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:31 smithi027 ceph-mon[98970]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:31 smithi171 ceph-mon[22959]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:33.132 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:03:33.721 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:03:33.721 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 39s ago 9m 24.2M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:03:33.721 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (2m) 38s ago 8m 75.9M - c8b91775d855 99f2571ddaab 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (7m) 39s ago 7m 18.8M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (7m) 38s ago 7m 15.6M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (4m) 38s ago 12m 453M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (4m) 39s ago 14m 507M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (2m) 39s ago 14m 44.0M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 1ad77d7a5221 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (13m) 38s ago 13m 57.4M 2048M 18.2.4-774-gfe43f1c9 96384990183a ddc2b74b65d7 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (111s) 39s ago 13m 35.0M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 78f3772b32b4 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 39s ago 8m 11.9M - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (3m) 38s ago 8m 11.3M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (12m) 39s ago 12m 52.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (12m) 39s ago 12m 53.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (11m) 39s ago 11m 57.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:03:33.722 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (11m) 39s ago 11m 47.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:03:33.723 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (11m) 38s ago 11m 55.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:03:33.723 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (10m) 38s ago 10m 47.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:03:33.723 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (10m) 38s ago 10m 46.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:03:33.723 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (9m) 38s ago 9m 50.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:03:33.723 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (3m) 38s ago 9m 43.6M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:03:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:03:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:03:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:03:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:03:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: Upgrade: Setting container_image for all mgr 2024-08-29T15:03:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:03:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-29T15:03:34.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:33 smithi171 ceph-mon[22959]: Upgrade: It appears safe to stop mon.b 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all mgr 2024-08-29T15:03:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[98970]: Upgrade: It appears safe to stop mon.b 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all mgr 2024-08-29T15:03:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:34.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-29T15:03:34.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-29T15:03:34.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:33 smithi027 ceph-mon[93921]: Upgrade: It appears safe to stop mon.b 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 1, 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:03:34.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 11, 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 4 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:03:34.079 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/3 daemons upgraded", 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:03:34.379 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='client.? 172.21.15.27:0/2468909913' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:03:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:34 smithi171 ceph-mon[22959]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:03:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2468909913' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2468909913' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='mgr.34104 ' entity='mgr.y' 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-29T15:03:35.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-29T15:03:35.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:34 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-29T15:03:35.921 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:35 smithi171 ceph-mon[22959]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:35.921 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:35 smithi171 ceph-mon[22959]: Upgrade: Updating mon.b 2024-08-29T15:03:35.922 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:35 smithi171 ceph-mon[22959]: Deploying daemon mon.b on smithi171 2024-08-29T15:03:35.922 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:35 smithi171 ceph-mon[22959]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[93921]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[93921]: Upgrade: Updating mon.b 2024-08-29T15:03:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[93921]: Deploying daemon mon.b on smithi171 2024-08-29T15:03:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[93921]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:36.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[98970]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[98970]: Upgrade: Updating mon.b 2024-08-29T15:03:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[98970]: Deploying daemon mon.b on smithi171 2024-08-29T15:03:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:35 smithi027 ceph-mon[98970]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:03:36.186 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:35 smithi171 systemd[1]: Stopping Ceph mon.b for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:03:36.186 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b[22955]: 2024-08-29T15:03:36.164+0000 7f966673d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:03:36.186 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b[22955]: 2024-08-29T15:03:36.164+0000 7f966673d640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-08-29T15:03:36.466 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:36 smithi171 podman[89872]: 2024-08-29 15:03:36.185361234 +0000 UTC m=+0.120676242 container died ddc2b74b65d7d5b6a79099f17f14c6be2b93712e200a161b24c033efe397dd80 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, io.buildah.version=1.37.1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=reef-fe43f1c) 2024-08-29T15:03:36.523 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:36 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:03:36] ENGINE Bus STOPPING 2024-08-29T15:03:36.523 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:36 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:03:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T15:03:36.523 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:36 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:03:36] ENGINE Bus STOPPED 2024-08-29T15:03:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:36 smithi171 podman[89872]: 2024-08-29 15:03:36.564410553 +0000 UTC m=+0.499725554 container cleanup ddc2b74b65d7d5b6a79099f17f14c6be2b93712e200a161b24c033efe397dd80 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-29T15:03:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:36 smithi171 bash[89872]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b 2024-08-29T15:03:36.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:36 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:03:36] ENGINE Bus STARTING 2024-08-29T15:03:36.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:36 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:03:36] ENGINE Serving on http://:::9283 2024-08-29T15:03:36.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:36 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:03:36] ENGINE Bus STARTED 2024-08-29T15:03:37.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:36 smithi171 podman[89884]: 2024-08-29 15:03:36.832777777 +0000 UTC m=+0.647190940 container remove ddc2b74b65d7d5b6a79099f17f14c6be2b93712e200a161b24c033efe397dd80 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-08-29T15:03:37.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-29T15:03:37.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 systemd[1]: Stopped Ceph mon.b for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:03:37.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.b.service: Consumed 9.921s CPU time. 2024-08-29T15:03:37.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 systemd[1]: Starting Ceph mon.b for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:03:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 podman[90012]: 2024-08-29 15:03:37.314915125 +0000 UTC m=+0.016378347 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:03:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 podman[90012]: 2024-08-29 15:03:37.425808942 +0000 UTC m=+0.127272203 container create e4fb79954e664eba897c32842e8ee136a53dbdfd876669e42455bbaf0aca0714 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 podman[90012]: 2024-08-29 15:03:37.653295588 +0000 UTC m=+0.354758801 container init e4fb79954e664eba897c32842e8ee136a53dbdfd876669e42455bbaf0aca0714 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 podman[90012]: 2024-08-29 15:03:37.656633685 +0000 UTC m=+0.358096900 container start e4fb79954e664eba897c32842e8ee136a53dbdfd876669e42455bbaf0aca0714 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: set uid:gid to 167:167 (ceph:ceph) 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev), process ceph-mon, pid 2 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: pidfile_write: ignore empty --pid-file 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: load: jerasure load: lrc 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: RocksDB version: 7.9.2 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Git sha 0 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Compile date 2024-08-28 21:32:03 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: DB SUMMARY 2024-08-29T15:03:38.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: DB Session ID: F9XRN3RK2JUSLKVPY4LR 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: CURRENT file: CURRENT 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: IDENTITY file: IDENTITY 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1404 Bytes 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 7801913 ; 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.error_if_exists: 0 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.create_if_missing: 0 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.paranoid_checks: 1 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.env: 0x55f2c6995160 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.fs: PosixFileSystem 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.info_log: 0x55f2c81f7880 2024-08-29T15:03:38.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_file_opening_threads: 16 2024-08-29T15:03:38.020 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.statistics: (nil) 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.use_fsync: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_log_file_size: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.keep_log_file_num: 1000 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.recycle_log_file_num: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.allow_fallocate: 1 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.allow_mmap_reads: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.allow_mmap_writes: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.use_direct_reads: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.create_missing_column_families: 0 2024-08-29T15:03:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.db_log_dir: 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.wal_dir: 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.advise_random_on_open: 1 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.db_write_buffer_size: 0 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.write_buffer_manager: 0x55f2c81fa960 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-29T15:03:38.022 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.rate_limiter: (nil) 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.wal_recovery_mode: 2 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.enable_thread_tracking: 0 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.enable_pipelined_write: 0 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.unordered_write: 0 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.row_cache: None 2024-08-29T15:03:38.023 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.wal_filter: None 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.allow_ingest_behind: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.two_write_queues: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.manual_wal_flush: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.wal_compression: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.atomic_flush: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.log_readahead_size: 0 2024-08-29T15:03:38.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.best_efforts_recovery: 0 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.allow_data_in_errors: 0 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.db_host_id: __hostname__ 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_background_jobs: 2 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_background_compactions: -1 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_subcompactions: 1 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_total_wal_size: 0 2024-08-29T15:03:38.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_open_files: -1 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bytes_per_sync: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_readahead_size: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_background_flushes: -1 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Compression algorithms supported: 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kZSTD supported: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kXpressCompression supported: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kBZip2Compression supported: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-29T15:03:38.026 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kLZ4Compression supported: 1 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kZlibCompression supported: 1 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kLZ4HCCompression supported: 1 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: kSnappyCompression supported: 1 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.merge_operator: 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_filter: None 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_filter_factory: None 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.sst_partitioner_factory: None 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-29T15:03:38.027 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f2c81f74e0) 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: cache_index_and_filter_blocks: 1 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: pin_top_level_index_and_filter: 1 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_type: 0 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_block_index_type: 0 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_shortening: 1 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: checksum: 4 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: no_block_cache: 0 2024-08-29T15:03:38.028 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache: 0x55f2c81ea850 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_name: BinnedLRUCache 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_options: 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: capacity : 536870912 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: num_shard_bits : 4 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: strict_capacity_limit : 0 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: high_pri_pool_ratio: 0.000 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_cache_compressed: (nil) 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: persistent_cache: (nil) 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_size: 4096 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_size_deviation: 10 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_restart_interval: 16 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: index_block_restart_interval: 1 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: metadata_block_size: 4096 2024-08-29T15:03:38.029 INFO:journalctl@ceph.mon.b.smithi171.stdout: partition_filters: 0 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: use_delta_encoding: 1 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: filter_policy: bloomfilter 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: whole_key_filtering: 1 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: verify_compression: 0 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: read_amp_bytes_per_bit: 0 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: format_version: 5 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: enable_index_compression: 1 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: block_align: 0 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_auto_readahead_size: 262144 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: prepopulate_block_cache: 0 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: initial_auto_readahead_size: 8192 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout: num_file_reads_for_auto_readahead: 2 2024-08-29T15:03:38.030 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.write_buffer_size: 33554432 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_write_buffer_number: 2 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression: NoCompression 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression: Disabled 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.prefix_extractor: nullptr 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.num_levels: 7 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-29T15:03:38.031 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.level: 32767 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.strategy: 0 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-29T15:03:38.032 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.enabled: false 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.target_file_size_base: 67108864 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-29T15:03:38.033 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-29T15:03:38.034 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-29T15:03:38.034 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.arena_block_size: 1048576 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.disable_auto_compactions: 0 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-29T15:03:38.035 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.inplace_update_support: 0 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-29T15:03:38.036 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.bloom_locality: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.max_successive_merges: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.paranoid_file_checks: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.force_consistency_checks: 1 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.report_bg_io_stats: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.ttl: 2592000 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.enable_blob_files: false 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.min_blob_size: 0 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.blob_file_size: 268435456 2024-08-29T15:03:38.037 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.blob_file_starting_level: 0 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 13030, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1d91bae4-121e-494f-a6c0-8971c1cbb194 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943817683451, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943817703286, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4822151, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13035, "largest_seqno": 16721, "table_properties": {"data_size": 4807886, "index_size": 9058, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 37360, "raw_average_key_size": 22, "raw_value_size": 4775135, "raw_average_value_size": 2890, "num_data_blocks": 406, "num_entries": 1652, "num_filter_entries": 1652, "num_deletions": 261, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1724943817, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1d91bae4-121e-494f-a6c0-8971c1cbb194", "db_session_id": "F9XRN3RK2JUSLKVPY4LR", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: EVENT_LOG_v1 {"time_micros": 1724943817703429, "job": 1, "event": "recovery_finished"} 2024-08-29T15:03:38.038 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f2c821ae00 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: DB pointer 0x55f2c831a000 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** DB Stats ** 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-29T15:03:38.039 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: L0 1/0 4.60 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: L6 1/0 7.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: Sum 2/0 12.50 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 542.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** Compaction Stats [default] ** 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 542.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-29T15:03:38.040 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: Cumulative compaction: 0.00 GB write, 193.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: Interval compaction: 0.00 GB write, 193.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: Block cache BinnedLRUCache@0x55f2c81ea850#2 capacity: 512.00 MB usage: 13.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-08-29T15:03:38.041 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: starting mon.b rank 1 at public addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] at bind addrs [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???) e3 preinit fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???).mds e9 new map 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???).mds e9 print_map 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: e9 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: legacy client fscid: 1 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: Filesystem 'foo' (1) 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: fs_name foo 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: epoch 8 2024-08-29T15:03:38.042 INFO:journalctl@ceph.mon.b.smithi171.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: created 2024-08-29T14:55:32.405244+0000 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: modified 2024-08-29T14:55:49.392761+0000 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: tableserver 0 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: root 0 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: session_timeout 60 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: session_autoclose 300 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_file_size 1099511627776 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_xattr_size 65536 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: required_client_features {} 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: last_failure 0 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: last_failure_osd_epoch 0 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: max_mds 1 2024-08-29T15:03:38.043 INFO:journalctl@ceph.mon.b.smithi171.stdout: in 0 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: up {0=14619} 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: failed 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: damaged 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: stopped 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: data_pools [4] 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: metadata_pool 3 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: inline_data disabled 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: balancer 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: bal_rank_mask -1 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: standby_count_wanted 1 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: qdb_cluster leader: 0 members: 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: [mds.foo.smithi027.afjjno{0:14619} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.27:6834/1598995072,v1:172.21.15.27:6835/1598995072] compat {c=[1],r=[1],i=[7ff]}] 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.044 INFO:journalctl@ceph.mon.b.smithi171.stdout: Standby daemons: 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout: 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout: [mds.foo.smithi171.grbaeh{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.171:6832/1791433656,v1:172.21.15.171:6833/1791433656] compat {c=[1],r=[1],i=[7ff]}] 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 ceph-mon[90031]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 bash[90012]: e4fb79954e664eba897c32842e8ee136a53dbdfd876669e42455bbaf0aca0714 2024-08-29T15:03:38.045 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:37 smithi171 systemd[1]: Started Ceph mon.b for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:03:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:03:39] "GET /metrics HTTP/1.1" 200 33805 "" "Prometheus/2.51.0" 2024-08-29T15:03:39.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: mgrmap e42: y(active, since 2m), standbys: x 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: overall HEALTH_OK 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: mon.b calling monitor election 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: mon.a calling monitor election 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: mon.c calling monitor election 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:03:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: monmap epoch 4 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: last_changed 2024-08-29T15:03:38.167884+0000 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: min_mon_release 19 (squid) 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: election_strategy: 1 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: mgrmap e42: y(active, since 2m), standbys: x 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: overall HEALTH_OK 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[98970]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: mgrmap e42: y(active, since 2m), standbys: x 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: overall HEALTH_OK 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: mon.b calling monitor election 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: mon.a calling monitor election 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: mon.c calling monitor election 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-29T15:03:39.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: monmap epoch 4 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: last_changed 2024-08-29T15:03:38.167884+0000 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: min_mon_release 19 (squid) 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: election_strategy: 1 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: mgrmap e42: y(active, since 2m), standbys: x 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: overall HEALTH_OK 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.322 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:03:39.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:39 smithi027 ceph-mon[93921]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:39.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:03:39.510 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: mgrmap e42: y(active, since 2m), standbys: x 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: overall HEALTH_OK 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: mon.b calling monitor election 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: mon.a calling monitor election 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: mon.c calling monitor election 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: monmap epoch 4 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: last_changed 2024-08-29T15:03:38.167884+0000 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: created 2024-08-29T14:48:41.684545+0000 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: min_mon_release 19 (squid) 2024-08-29T15:03:39.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: election_strategy: 1 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: 1: [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] mon.b 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: 2: [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon.c 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: fsmap foo:1 {0=foo.smithi027.afjjno=up:active} 1 up:standby 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: osdmap e90: 8 total, 8 up, 8 in 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: mgrmap e42: y(active, since 2m), standbys: x 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: overall HEALTH_OK 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:03:39.512 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:39 smithi171 ceph-mon[90031]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:41 smithi171 ceph-mon[90031]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:41.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:41 smithi027 ceph-mon[93921]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:41.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:41 smithi027 ceph-mon[98970]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:43.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:42 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:03:42.716+0000 7f7dea97a640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-08-29T15:03:43.617 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:43 smithi171 ceph-mon[90031]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:43.618 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.618 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.618 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.618 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[98970]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[93921]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:43.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:43.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:44.609 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:03:44.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:03:44.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:03:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:45 smithi171 ceph-mon[90031]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:45 smithi027 ceph-mon[93921]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:45.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:45 smithi027 ceph-mon[98970]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: Reconfiguring mon.a (monmap changed)... 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:47 smithi171 ceph-mon[90031]: Reconfiguring daemon mon.a on smithi027 2024-08-29T15:03:47.548 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: Reconfiguring mon.a (monmap changed)... 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[98970]: Reconfiguring daemon mon.a on smithi027 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: Reconfiguring mon.a (monmap changed)... 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:47.550 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:47 smithi027 ceph-mon[93921]: Reconfiguring daemon mon.a on smithi027 2024-08-29T15:03:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[98970]: Reconfiguring mgr.y (monmap changed)... 2024-08-29T15:03:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-29T15:03:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-29T15:03:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[98970]: Reconfiguring daemon mgr.y on smithi027 2024-08-29T15:03:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[93921]: Reconfiguring mgr.y (monmap changed)... 2024-08-29T15:03:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-29T15:03:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-29T15:03:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:48 smithi027 ceph-mon[93921]: Reconfiguring daemon mgr.y on smithi027 2024-08-29T15:03:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:48 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:48 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:48 smithi171 ceph-mon[90031]: Reconfiguring mgr.y (monmap changed)... 2024-08-29T15:03:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:48 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-29T15:03:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:48 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-29T15:03:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:48 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:48 smithi171 ceph-mon[90031]: Reconfiguring daemon mgr.y on smithi027 2024-08-29T15:03:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:03:49] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.51.0" 2024-08-29T15:03:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: Reconfiguring mon.c (monmap changed)... 2024-08-29T15:03:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:50.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:50.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:50.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:49 smithi171 ceph-mon[90031]: Reconfiguring daemon mon.c on smithi027 2024-08-29T15:03:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: Reconfiguring mon.c (monmap changed)... 2024-08-29T15:03:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[98970]: Reconfiguring daemon mon.c on smithi027 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: Reconfiguring mon.c (monmap changed)... 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:49 smithi027 ceph-mon[93921]: Reconfiguring daemon mon.c on smithi027 2024-08-29T15:03:51.678 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[98970]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[98970]: Reconfiguring osd.0 (monmap changed)... 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.0 on smithi027 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[93921]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[93921]: Reconfiguring osd.0 (monmap changed)... 2024-08-29T15:03:51.679 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-29T15:03:51.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:51.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:51 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.0 on smithi027 2024-08-29T15:03:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:51 smithi171 ceph-mon[90031]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:51 smithi171 ceph-mon[90031]: Reconfiguring osd.0 (monmap changed)... 2024-08-29T15:03:51.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-29T15:03:51.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:51.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:51 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.0 on smithi027 2024-08-29T15:03:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[93921]: Reconfiguring osd.1 (monmap changed)... 2024-08-29T15:03:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.1 on smithi027 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[98970]: Reconfiguring osd.1 (monmap changed)... 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:53.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:52 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.1 on smithi027 2024-08-29T15:03:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:52 smithi171 ceph-mon[90031]: Reconfiguring osd.1 (monmap changed)... 2024-08-29T15:03:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-29T15:03:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:52 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.1 on smithi027 2024-08-29T15:03:53.922 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[98970]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:53.923 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.923 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.923 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-29T15:03:53.923 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:53.924 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[93921]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:53.924 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.924 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:53.925 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-29T15:03:53.925 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:53 smithi171 ceph-mon[90031]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-29T15:03:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[98970]: Reconfiguring osd.2 (monmap changed)... 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.2 on smithi027 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[93921]: Reconfiguring osd.2 (monmap changed)... 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.2 on smithi027 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:55.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:55.067 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-29T15:03:55.067 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:54 smithi171 ceph-mon[90031]: Reconfiguring osd.2 (monmap changed)... 2024-08-29T15:03:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:54 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.2 on smithi027 2024-08-29T15:03:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-29T15:03:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:55 smithi027 ceph-mon[93921]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:55 smithi027 ceph-mon[93921]: Reconfiguring osd.3 (monmap changed)... 2024-08-29T15:03:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:55 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.3 on smithi027 2024-08-29T15:03:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:55 smithi027 ceph-mon[98970]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:55 smithi027 ceph-mon[98970]: Reconfiguring osd.3 (monmap changed)... 2024-08-29T15:03:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:55 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.3 on smithi027 2024-08-29T15:03:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:55 smithi171 ceph-mon[90031]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:55 smithi171 ceph-mon[90031]: Reconfiguring osd.3 (monmap changed)... 2024-08-29T15:03:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:55 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.3 on smithi027 2024-08-29T15:03:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:57 smithi171 ceph-mon[90031]: Reconfiguring mds.foo.smithi027.afjjno (monmap changed)... 2024-08-29T15:03:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-29T15:03:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:57.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:57 smithi171 ceph-mon[90031]: Reconfiguring daemon mds.foo.smithi027.afjjno on smithi027 2024-08-29T15:03:57.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:57 smithi171 ceph-mon[90031]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[93921]: Reconfiguring mds.foo.smithi027.afjjno (monmap changed)... 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[93921]: Reconfiguring daemon mds.foo.smithi027.afjjno on smithi027 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[93921]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[98970]: Reconfiguring mds.foo.smithi027.afjjno (monmap changed)... 2024-08-29T15:03:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.afjjno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-29T15:03:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[98970]: Reconfiguring daemon mds.foo.smithi027.afjjno on smithi027 2024-08-29T15:03:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:57 smithi027 ceph-mon[98970]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:58.995 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:58.996 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:58.996 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: Reconfiguring mon.b (monmap changed)... 2024-08-29T15:03:58.996 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:58.996 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:58.996 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:58.996 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: Reconfiguring daemon mon.b on smithi171 2024-08-29T15:03:58.996 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:03:59.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:59.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: Reconfiguring mon.b (monmap changed)... 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: Reconfiguring daemon mon.b on smithi171 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:59.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: Reconfiguring mon.b (monmap changed)... 2024-08-29T15:03:59.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-29T15:03:59.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-29T15:03:59.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:03:59.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: Reconfiguring daemon mon.b on smithi171 2024-08-29T15:03:59.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:03:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:03:59] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.51.0" 2024-08-29T15:03:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:59 smithi171 ceph-mon[90031]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:03:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:03:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-29T15:03:59.749 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-29T15:03:59.750 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:03:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[93921]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[98970]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-29T15:04:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:03:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:00.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:00 smithi171 ceph-mon[90031]: Reconfiguring mgr.x (monmap changed)... 2024-08-29T15:04:00.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:00 smithi171 ceph-mon[90031]: Reconfiguring daemon mgr.x on smithi171 2024-08-29T15:04:00.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:00.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:00.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-29T15:04:00.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[98970]: Reconfiguring mgr.x (monmap changed)... 2024-08-29T15:04:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[98970]: Reconfiguring daemon mgr.x on smithi171 2024-08-29T15:04:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-29T15:04:01.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[93921]: Reconfiguring mgr.x (monmap changed)... 2024-08-29T15:04:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[93921]: Reconfiguring daemon mgr.x on smithi171 2024-08-29T15:04:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-29T15:04:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:01.810 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:01 smithi171 ceph-mon[90031]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:01.810 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:01 smithi171 ceph-mon[90031]: Reconfiguring osd.4 (monmap changed)... 2024-08-29T15:04:01.810 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:01 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.4 on smithi171 2024-08-29T15:04:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:01 smithi027 ceph-mon[98970]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:01 smithi027 ceph-mon[98970]: Reconfiguring osd.4 (monmap changed)... 2024-08-29T15:04:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:01 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.4 on smithi171 2024-08-29T15:04:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:01 smithi027 ceph-mon[93921]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:01 smithi027 ceph-mon[93921]: Reconfiguring osd.4 (monmap changed)... 2024-08-29T15:04:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:01 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.4 on smithi171 2024-08-29T15:04:03.203 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.203 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.203 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:02 smithi171 ceph-mon[90031]: Reconfiguring osd.5 (monmap changed)... 2024-08-29T15:04:03.204 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-29T15:04:03.204 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:03.204 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:02 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.5 on smithi171 2024-08-29T15:04:03.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[98970]: Reconfiguring osd.5 (monmap changed)... 2024-08-29T15:04:03.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-29T15:04:03.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:03.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.5 on smithi171 2024-08-29T15:04:03.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[93921]: Reconfiguring osd.5 (monmap changed)... 2024-08-29T15:04:03.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-29T15:04:03.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:03.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:02 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.5 on smithi171 2024-08-29T15:04:03.990 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:03 smithi171 ceph-mon[90031]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:03.990 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.990 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:03.990 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-29T15:04:03.990 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[98970]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-29T15:04:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[93921]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-29T15:04:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:04.708 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:04:05.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:04 smithi027 ceph-mon[98970]: Reconfiguring osd.6 (monmap changed)... 2024-08-29T15:04:05.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:04 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.6 on smithi171 2024-08-29T15:04:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:04 smithi027 ceph-mon[93921]: Reconfiguring osd.6 (monmap changed)... 2024-08-29T15:04:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:04 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.6 on smithi171 2024-08-29T15:04:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:04 smithi171 ceph-mon[90031]: Reconfiguring osd.6 (monmap changed)... 2024-08-29T15:04:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:04 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.6 on smithi171 2024-08-29T15:04:05.296 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:04:05.296 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 71s ago 9m 24.2M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:04:05.296 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (3m) 22s ago 9m 77.4M - c8b91775d855 99f2571ddaab 2024-08-29T15:04:05.296 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (8m) 71s ago 8m 18.8M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:04:05.296 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (8m) 22s ago 8m 15.9M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (4m) 22s ago 13m 454M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (5m) 71s ago 15m 507M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (2m) 71s ago 15m 44.0M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 1ad77d7a5221 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (27s) 22s ago 13m 27.9M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec e4fb79954e66 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (2m) 71s ago 13m 35.0M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 78f3772b32b4 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 71s ago 9m 11.9M - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (4m) 22s ago 8m 11.1M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (13m) 71s ago 13m 52.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (12m) 71s ago 12m 53.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (12m) 71s ago 12m 57.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (11m) 71s ago 11m 47.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (11m) 22s ago 11m 56.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:04:05.297 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (11m) 22s ago 11m 48.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:04:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (10m) 22s ago 10m 47.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:04:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (10m) 22s ago 10m 51.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:04:05.298 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (4m) 22s ago 9m 43.7M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:04:05.652 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:04:05.652 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 3 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 10, 2024-08-29T15:04:05.653 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 5 2024-08-29T15:04:05.654 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:04:05.654 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:04:05.954 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:04:05.954 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:04:05.954 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:04:05.954 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-08-29T15:04:05.954 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-08-29T15:04:05.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mon" 2024-08-29T15:04:05.955 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:04:05.955 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "3/3 daemons upgraded", 2024-08-29T15:04:05.955 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-08-29T15:04:05.955 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:04:05.955 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: Reconfiguring osd.7 (monmap changed)... 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: Reconfiguring daemon osd.7 on smithi171 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:06.177 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4053581618' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: Reconfiguring osd.7 (monmap changed)... 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: Reconfiguring daemon osd.7 on smithi171 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4053581618' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: Reconfiguring osd.7 (monmap changed)... 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: Reconfiguring daemon osd.7 on smithi171 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4053581618' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:06 smithi171 ceph-mon[90031]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:06 smithi171 ceph-mon[90031]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-29T15:04:07.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[93921]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[93921]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-29T15:04:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[98970]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[98970]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi171.grbaeh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-29T15:04:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: Reconfiguring mds.foo.smithi171.grbaeh (monmap changed)... 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: Reconfiguring daemon mds.foo.smithi171.grbaeh on smithi171 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:04:08.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:04:08.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:04:08.271 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: Reconfiguring mds.foo.smithi171.grbaeh (monmap changed)... 2024-08-29T15:04:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: Reconfiguring daemon mds.foo.smithi171.grbaeh on smithi171 2024-08-29T15:04:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: Reconfiguring mds.foo.smithi171.grbaeh (monmap changed)... 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: Reconfiguring daemon mds.foo.smithi171.grbaeh on smithi171 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:08.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:04:08.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:08.324 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:04:08.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:04:08.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:04:08.326 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:09.249 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:04:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:04:09] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.51.0" 2024-08-29T15:04:09.249 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all grafana 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all loki 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all promtail 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Finalizing container_image settings 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: Upgrade: Complete! 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:04:09.250 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all grafana 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all loki 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all promtail 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Finalizing container_image settings 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: Upgrade: Complete! 2024-08-29T15:04:09.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:08 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all grafana 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all loki 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all promtail 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Finalizing container_image settings 2024-08-29T15:04:09.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: Upgrade: Complete! 2024-08-29T15:04:09.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:08 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:10.199 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:09 smithi027 ceph-mon[98970]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:10.200 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:09 smithi027 ceph-mon[93921]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:09 smithi171 ceph-mon[90031]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:04:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:04:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:04:12.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:04:12.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:04:12.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:04:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:13 smithi171 ceph-mon[90031]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:13 smithi027 ceph-mon[93921]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:13 smithi027 ceph-mon[98970]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:15 smithi171 ceph-mon[90031]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:15 smithi027 ceph-mon[93921]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:15 smithi027 ceph-mon[98970]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:17 smithi171 ceph-mon[90031]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:17 smithi027 ceph-mon[93921]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:18.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:17 smithi027 ceph-mon[98970]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:04:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:04:19] "GET /metrics HTTP/1.1" 200 33817 "" "Prometheus/2.51.0" 2024-08-29T15:04:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:19 smithi171 ceph-mon[90031]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:20.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:19 smithi027 ceph-mon[93921]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:19 smithi027 ceph-mon[98970]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:21 smithi171 ceph-mon[90031]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:21 smithi027 ceph-mon[93921]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:21 smithi027 ceph-mon[98970]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:23 smithi171 ceph-mon[90031]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:23 smithi027 ceph-mon[93921]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:23 smithi027 ceph-mon[98970]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:25 smithi171 ceph-mon[90031]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:26.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:25 smithi027 ceph-mon[93921]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:25 smithi027 ceph-mon[98970]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:27 smithi171 ceph-mon[90031]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:27 smithi027 ceph-mon[93921]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:27 smithi027 ceph-mon[98970]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:04:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:04:29] "GET /metrics HTTP/1.1" 200 33810 "" "Prometheus/2.51.0" 2024-08-29T15:04:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:29 smithi171 ceph-mon[90031]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:30.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:29 smithi027 ceph-mon[93921]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:30.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:29 smithi027 ceph-mon[98970]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:31 smithi171 ceph-mon[90031]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:32.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:31 smithi027 ceph-mon[93921]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:31 smithi027 ceph-mon[98970]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:33 smithi171 ceph-mon[90031]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:34.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:33 smithi027 ceph-mon[93921]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:34.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:33 smithi027 ceph-mon[98970]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:35 smithi171 ceph-mon[90031]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:36.282 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:35 smithi027 ceph-mon[98970]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:36.282 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:35 smithi027 ceph-mon[93921]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:36.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'sleep 60' 2024-08-29T15:04:37.157 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:04:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:37 smithi171 ceph-mon[90031]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:37 smithi171 ceph-mon[90031]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:37 smithi027 ceph-mon[93921]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:37 smithi027 ceph-mon[93921]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:37 smithi027 ceph-mon[98970]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:38.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:37 smithi027 ceph-mon[98970]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:04:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:04:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:04:39] "GET /metrics HTTP/1.1" 200 33810 "" "Prometheus/2.51.0" 2024-08-29T15:04:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:39 smithi171 ceph-mon[90031]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:40.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:39 smithi027 ceph-mon[93921]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:40.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:39 smithi027 ceph-mon[98970]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:41 smithi171 ceph-mon[90031]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:41 smithi027 ceph-mon[93921]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:41 smithi027 ceph-mon[98970]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:43 smithi171 ceph-mon[90031]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:43 smithi027 ceph-mon[93921]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:43 smithi027 ceph-mon[98970]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:45 smithi171 ceph-mon[90031]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:45 smithi027 ceph-mon[93921]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:45 smithi027 ceph-mon[98970]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:48 smithi171 ceph-mon[90031]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:48 smithi027 ceph-mon[93921]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:48 smithi027 ceph-mon[98970]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:04:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:04:49] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.51.0" 2024-08-29T15:04:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:50 smithi171 ceph-mon[90031]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:50 smithi027 ceph-mon[93921]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:50 smithi027 ceph-mon[98970]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:52 smithi171 ceph-mon[90031]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:52 smithi027 ceph-mon[93921]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:52 smithi027 ceph-mon[98970]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:54.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:54 smithi171 ceph-mon[90031]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:54 smithi027 ceph-mon[93921]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:54 smithi027 ceph-mon[98970]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:56 smithi171 ceph-mon[90031]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:56 smithi027 ceph-mon[93921]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:56 smithi027 ceph-mon[98970]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:58 smithi171 ceph-mon[90031]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:58 smithi027 ceph-mon[93921]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:58 smithi027 ceph-mon[98970]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:04:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:04:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:04:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:04:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:04:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:04:59] "GET /metrics HTTP/1.1" 200 33815 "" "Prometheus/2.51.0" 2024-08-29T15:04:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:04:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:00 smithi027 ceph-mon[93921]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:00.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:00 smithi027 ceph-mon[98970]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:00 smithi171 ceph-mon[90031]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:02 smithi027 ceph-mon[98970]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:02 smithi027 ceph-mon[93921]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:02 smithi171 ceph-mon[90031]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:04 smithi027 ceph-mon[98970]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:04 smithi027 ceph-mon[93921]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:04 smithi171 ceph-mon[90031]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:06 smithi027 ceph-mon[93921]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:06 smithi027 ceph-mon[98970]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:06 smithi171 ceph-mon[90031]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:08 smithi027 ceph-mon[93921]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:08 smithi027 ceph-mon[98970]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:08 smithi171 ceph-mon[90031]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:05:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:05:09] "GET /metrics HTTP/1.1" 200 33815 "" "Prometheus/2.51.0" 2024-08-29T15:05:10.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:10 smithi027 ceph-mon[93921]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:10.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:10 smithi027 ceph-mon[98970]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:10 smithi171 ceph-mon[90031]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:12 smithi027 ceph-mon[98970]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:12 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:12 smithi027 ceph-mon[93921]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:12 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:12 smithi171 ceph-mon[90031]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:12 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:14.289 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:14 smithi171 ceph-mon[90031]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:14.289 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:14.314 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:14 smithi027 ceph-mon[98970]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:14.314 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:14.315 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:14 smithi027 ceph-mon[93921]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:14.315 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[98970]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[93921]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:16.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:16 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:16 smithi171 ceph-mon[90031]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:16 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:16 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:16 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:16 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:17.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:17.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:17.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:17.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:17.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:17.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:18 smithi027 ceph-mon[93921]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:18.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:18 smithi027 ceph-mon[98970]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:18 smithi171 ceph-mon[90031]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:19 smithi027 ceph-mon[93921]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:19.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:19 smithi027 ceph-mon[98970]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:19.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:05:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:05:19] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.51.0" 2024-08-29T15:05:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:19 smithi171 ceph-mon[90031]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:21.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:21 smithi027 ceph-mon[93921]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:21 smithi027 ceph-mon[98970]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:21 smithi171 ceph-mon[90031]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:23 smithi171 ceph-mon[90031]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:23 smithi027 ceph-mon[93921]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:23 smithi027 ceph-mon[98970]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:25 smithi171 ceph-mon[90031]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:25.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:25 smithi027 ceph-mon[93921]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:25 smithi027 ceph-mon[98970]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:27 smithi171 ceph-mon[90031]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:27 smithi027 ceph-mon[93921]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:27 smithi027 ceph-mon[98970]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:05:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:05:29] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.51.0" 2024-08-29T15:05:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:29 smithi171 ceph-mon[90031]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:29 smithi027 ceph-mon[93921]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:29 smithi027 ceph-mon[98970]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:31 smithi171 ceph-mon[90031]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:31 smithi027 ceph-mon[93921]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:31 smithi027 ceph-mon[98970]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:33 smithi171 ceph-mon[90031]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:33.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:33 smithi027 ceph-mon[93921]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:33.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:33 smithi027 ceph-mon[98970]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:35 smithi171 ceph-mon[90031]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:35.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:35 smithi027 ceph-mon[93921]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:35 smithi027 ceph-mon[98970]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:37 smithi171 ceph-mon[90031]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:37.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:37 smithi027 ceph-mon[93921]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:37.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:37 smithi027 ceph-mon[98970]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:05:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:05:39] "GET /metrics HTTP/1.1" 200 33816 "" "Prometheus/2.51.0" 2024-08-29T15:05:39.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'echo wait for all mons to upgrade' 2024-08-29T15:05:39.564 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:05:39.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:39 smithi027 ceph-mon[98970]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:39.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:39 smithi027 ceph-mon[93921]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:39 smithi171 ceph-mon[90031]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:41.142 INFO:teuthology.orchestra.run.smithi027.stdout:wait for all mons to upgrade 2024-08-29T15:05:41.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:41 smithi027 ceph-mon[98970]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:41.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:41 smithi027 ceph-mon[93921]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:41 smithi171 ceph-mon[90031]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:41.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-08-29T15:05:42.054 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:05:43.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:43 smithi027 ceph-mon[98970]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:43.653 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:43 smithi027 ceph-mon[93921]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:43 smithi171 ceph-mon[90031]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:44.752 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:44.752 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:44 smithi027 ceph-mon[93921]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:05:44.752 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:44.753 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:44 smithi027 ceph-mon[98970]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:05:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:05:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:44 smithi171 ceph-mon[90031]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:05:45.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:45 smithi027 ceph-mon[93921]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:45.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:45 smithi027 ceph-mon[98970]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:45 smithi171 ceph-mon[90031]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:45.814 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:05:46.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-29T15:05:46.657 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:05:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[93921]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:05:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[93921]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:05:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[98970]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:05:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:46 smithi027 ceph-mon[98970]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:05:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:46 smithi171 ceph-mon[90031]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:05:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:47.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:46 smithi171 ceph-mon[90031]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:05:48.091 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:47 smithi027 ceph-mon[93921]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:48.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:47 smithi027 ceph-mon[98970]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:47 smithi171 ceph-mon[90031]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:49.226 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:05:49] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.51.0" 2024-08-29T15:05:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all mgr 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all mon 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all crash 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all rgw 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all rbd-mirror 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all ceph-exporter 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all iscsi 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all nfs 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all mgr 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all mon 2024-08-29T15:05:49.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all crash 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all rgw 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all rbd-mirror 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all ceph-exporter 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all iscsi 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all nfs 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all grafana 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all loki 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all promtail 2024-08-29T15:05:49.575 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Finalizing container_image settings 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-29T15:05:49.576 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all grafana 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all loki 2024-08-29T15:05:49.577 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all promtail 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Finalizing container_image settings 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-29T15:05:49.578 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: Upgrade: Complete! 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.580 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: Upgrade: Complete! 2024-08-29T15:05:49.582 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:05:49.582 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:05:49.582 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:49.582 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:49.582 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:49.582 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.582 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all mgr 2024-08-29T15:05:49.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all mon 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all crash 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all rgw 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all rbd-mirror 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all ceph-exporter 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all iscsi 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all nfs 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all node-exporter 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all prometheus 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all alertmanager 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all grafana 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all loki 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all promtail 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Finalizing container_image settings 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-29T15:05:49.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: Upgrade: Complete! 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:05:49.774 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:05:49.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:49.775 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:05:50.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'sleep 60' 2024-08-29T15:05:50.493 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:05:50.724 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:50 smithi027 ceph-mon[98970]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:05:50.725 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:50 smithi027 ceph-mon[93921]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:05:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:50 smithi171 ceph-mon[90031]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:05:51.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:51 smithi027 ceph-mon[98970]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:51.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:51 smithi027 ceph-mon[93921]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:51 smithi171 ceph-mon[90031]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:53.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:53 smithi027 ceph-mon[93921]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:53.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:53 smithi027 ceph-mon[98970]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:53 smithi171 ceph-mon[90031]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:56.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:55 smithi171 ceph-mon[90031]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:55 smithi027 ceph-mon[93921]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:55 smithi027 ceph-mon[98970]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:57 smithi171 ceph-mon[90031]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:57 smithi027 ceph-mon[93921]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:58.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:57 smithi027 ceph-mon[98970]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:05:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:05:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:05:59] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.51.0" 2024-08-29T15:06:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:59 smithi171 ceph-mon[90031]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:05:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:59 smithi027 ceph-mon[93921]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:05:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:59 smithi027 ceph-mon[98970]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:05:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:01 smithi171 ceph-mon[90031]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:01 smithi027 ceph-mon[93921]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:01 smithi027 ceph-mon[98970]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:03 smithi171 ceph-mon[90031]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:03 smithi027 ceph-mon[93921]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:03 smithi027 ceph-mon[98970]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:05 smithi171 ceph-mon[90031]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:06.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:05 smithi027 ceph-mon[98970]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:05 smithi027 ceph-mon[93921]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:07 smithi171 ceph-mon[90031]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:08.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:07 smithi027 ceph-mon[93921]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:07 smithi027 ceph-mon[98970]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:06:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:06:09] "GET /metrics HTTP/1.1" 200 33814 "" "Prometheus/2.51.0" 2024-08-29T15:06:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:09 smithi171 ceph-mon[90031]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:09 smithi027 ceph-mon[93921]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:10.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:09 smithi027 ceph-mon[98970]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:11 smithi171 ceph-mon[90031]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:12.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:11 smithi027 ceph-mon[93921]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:11 smithi027 ceph-mon[98970]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:13 smithi171 ceph-mon[90031]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:13 smithi027 ceph-mon[93921]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:13 smithi027 ceph-mon[98970]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:15 smithi171 ceph-mon[90031]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:16.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:15 smithi027 ceph-mon[93921]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:15 smithi027 ceph-mon[98970]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:17 smithi171 ceph-mon[90031]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:17 smithi027 ceph-mon[93921]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:17 smithi027 ceph-mon[98970]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:06:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:06:19] "GET /metrics HTTP/1.1" 200 33743 "" "Prometheus/2.51.0" 2024-08-29T15:06:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:19 smithi171 ceph-mon[90031]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:19 smithi027 ceph-mon[93921]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:19 smithi027 ceph-mon[98970]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:21 smithi171 ceph-mon[90031]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:21 smithi027 ceph-mon[98970]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:21 smithi027 ceph-mon[93921]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:23 smithi171 ceph-mon[90031]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:23 smithi027 ceph-mon[98970]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:23 smithi027 ceph-mon[93921]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:25 smithi171 ceph-mon[90031]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:25 smithi027 ceph-mon[93921]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:26.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:25 smithi027 ceph-mon[98970]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:27 smithi171 ceph-mon[90031]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:27 smithi027 ceph-mon[98970]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:27 smithi027 ceph-mon[93921]: pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:06:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:06:29] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.51.0" 2024-08-29T15:06:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:29 smithi171 ceph-mon[90031]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:29 smithi027 ceph-mon[98970]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:29 smithi027 ceph-mon[93921]: pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:31 smithi171 ceph-mon[90031]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:31 smithi027 ceph-mon[93921]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:31 smithi027 ceph-mon[98970]: pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:33 smithi171 ceph-mon[90031]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:33 smithi027 ceph-mon[93921]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:33 smithi027 ceph-mon[98970]: pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:35 smithi171 ceph-mon[90031]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:35 smithi027 ceph-mon[93921]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:36.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:35 smithi027 ceph-mon[98970]: pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:37 smithi171 ceph-mon[90031]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:37 smithi027 ceph-mon[93921]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:37 smithi027 ceph-mon[98970]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:06:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:06:39] "GET /metrics HTTP/1.1" 200 33812 "" "Prometheus/2.51.0" 2024-08-29T15:06:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:39 smithi171 ceph-mon[90031]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:39 smithi027 ceph-mon[93921]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:39 smithi027 ceph-mon[98970]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:41 smithi171 ceph-mon[90031]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:41 smithi027 ceph-mon[98970]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:41 smithi027 ceph-mon[93921]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:43 smithi171 ceph-mon[90031]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:43 smithi027 ceph-mon[98970]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:43 smithi027 ceph-mon[93921]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:45.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:45 smithi171 ceph-mon[90031]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:45 smithi027 ceph-mon[93921]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:45 smithi027 ceph-mon[98970]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:47 smithi171 ceph-mon[90031]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:47 smithi027 ceph-mon[93921]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:47 smithi027 ceph-mon[98970]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:06:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:06:49] "GET /metrics HTTP/1.1" 200 33811 "" "Prometheus/2.51.0" 2024-08-29T15:06:49.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:49 smithi027 ceph-mon[98970]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:49.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:06:49.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:49 smithi027 ceph-mon[93921]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:49.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:06:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:49 smithi171 ceph-mon[90031]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:06:51.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:51 smithi027 ceph-mon[98970]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:51.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:51 smithi027 ceph-mon[93921]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:52.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:51 smithi171 ceph-mon[90031]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:52.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'echo wait for half of osds to upgrade' 2024-08-29T15:06:53.124 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:06:54.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:54.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:06:54.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:06:54.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:06:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:06:54.659 INFO:teuthology.orchestra.run.smithi027.stdout:wait for half of osds to upgrade 2024-08-29T15:06:55.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-08-29T15:06:55.554 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:06:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:55 smithi027 ceph-mon[93921]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:55 smithi027 ceph-mon[98970]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:55 smithi171 ceph-mon[90031]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:58.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:57 smithi027 ceph-mon[98970]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:58.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:57 smithi027 ceph-mon[93921]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:57 smithi171 ceph-mon[90031]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:06:59.028 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:06:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:58 smithi171 ceph-mon[90031]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:06:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:06:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:58 smithi027 ceph-mon[93921]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:06:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:06:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:06:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:06:59] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.51.0" 2024-08-29T15:06:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:58 smithi027 ceph-mon[98970]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:06:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:06:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:06:59.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid af03926c-6615-11ef-bcd4-c7b262605968 -e sha1=1f4056d968930e12926c25a8f13ee8ed6047c791 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-29T15:06:59.871 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:07:00.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[98970]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[98970]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[93921]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[93921]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:07:00.272 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:00.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:00.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:00.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:00.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:00 smithi171 ceph-mon[90031]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:00 smithi171 ceph-mon[90031]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:07:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:01 smithi027 ceph-mon[93921]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:07:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:01 smithi027 ceph-mon[98970]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:07:01.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:01 smithi171 ceph-mon[90031]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1f4056d968930e12926c25a8f13ee8ed6047c791 2024-08-29T15:07:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:02 smithi027 ceph-mon[98970]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:02 smithi027 ceph-mon[93921]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:02.384 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:07:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:02 smithi171 ceph-mon[90031]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:02.995 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:07:02.995 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (6m) 9s ago 12m 22.7M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (6m) 10s ago 12m 78.4M - c8b91775d855 99f2571ddaab 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (11m) 9s ago 11m 20.4M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (11m) 10s ago 11m 16.7M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (7m) 10s ago 16m 456M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (8m) 9s ago 18m 515M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (5m) 9s ago 18m 52.6M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 1ad77d7a5221 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (3m) 10s ago 16m 42.8M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec e4fb79954e66 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (5m) 9s ago 16m 47.1M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 78f3772b32b4 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (7m) 9s ago 11m 11.2M - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (7m) 10s ago 11m 11.0M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (16m) 9s ago 16m 53.2M 2480M 18.2.4-774-gfe43f1c9 96384990183a fa01ba5e3737 2024-08-29T15:07:02.996 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (15m) 9s ago 15m 53.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:07:02.997 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (15m) 9s ago 15m 58.0M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:07:02.997 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (14m) 9s ago 14m 47.7M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:07:02.997 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (14m) 10s ago 14m 56.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:07:02.997 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (14m) 10s ago 14m 48.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:07:02.997 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (13m) 10s ago 13m 47.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:07:02.997 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (13m) 10s ago 13m 52.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:07:02.997 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (7m) 10s ago 12m 48.1M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:07:03.354 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 3 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:07:03.355 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:07:03.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:03.356 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:07:03.356 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 10, 2024-08-29T15:07:03.356 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 5 2024-08-29T15:07:03.356 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:07:03.356 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/8 daemons upgraded", 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:07:03.658 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:07:03.944 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:03.944 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all mgr 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all mon 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all crash 2024-08-29T15:07:03.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: Upgrade: osd.0 is safe to restart 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/669466373' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:07:03.946 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all mgr 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all mon 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all crash 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: Upgrade: osd.0 is safe to restart 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:03.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:03 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/669466373' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:04.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:04.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: Upgrade: Target is version 19.3.0-4579-g1f4056d9 (squid) 2024-08-29T15:07:04.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc'] 2024-08-29T15:07:04.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:04.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:04.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all mgr 2024-08-29T15:07:04.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all mon 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all crash 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: Upgrade: osd.0 is safe to restart 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:04.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:03 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/669466373' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:04 smithi171 ceph-mon[90031]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:04 smithi171 ceph-mon[90031]: Upgrade: Updating osd.0 2024-08-29T15:07:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:04 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:04 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-29T15:07:05.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:04 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:05.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:04 smithi171 ceph-mon[90031]: Deploying daemon osd.0 on smithi027 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[98970]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[98970]: Upgrade: Updating osd.0 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[98970]: Deploying daemon osd.0 on smithi027 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[93921]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[93921]: Upgrade: Updating osd.0 2024-08-29T15:07:05.302 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:05.303 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-29T15:07:05.303 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:05.303 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:04 smithi027 ceph-mon[93921]: Deploying daemon osd.0 on smithi027 2024-08-29T15:07:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:05 smithi027 ceph-mon[98970]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:06.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:05 smithi027 ceph-mon[93921]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:05 smithi171 ceph-mon[90031]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:07.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:06 smithi027 systemd[1]: Stopping Ceph osd.0 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:07:07.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[33347]: 2024-08-29T15:07:07.113+0000 7ff0779b6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:07:07.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[33347]: 2024-08-29T15:07:07.113+0000 7ff0779b6640 -1 osd.0 90 *** Got signal Terminated *** 2024-08-29T15:07:07.319 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[33347]: 2024-08-29T15:07:07.113+0000 7ff0779b6640 -1 osd.0 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:07:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:07 smithi171 ceph-mon[90031]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:07 smithi171 ceph-mon[90031]: osd.0 marked itself down and dead 2024-08-29T15:07:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:07 smithi027 ceph-mon[98970]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:07 smithi027 ceph-mon[98970]: osd.0 marked itself down and dead 2024-08-29T15:07:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:07 smithi027 ceph-mon[93921]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:07 smithi027 ceph-mon[93921]: osd.0 marked itself down and dead 2024-08-29T15:07:08.809 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:08 smithi027 podman[109466]: 2024-08-29 15:07:08.477607711 +0000 UTC m=+1.455078637 container died fa01ba5e373707b69d1bad9157164aeee9001ae83c3b03c397a82ac7d86fe9a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-fe43f1c, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.1, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-08-29T15:07:09.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:07:09] "GET /metrics HTTP/1.1" 200 33813 "" "Prometheus/2.51.0" 2024-08-29T15:07:09.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:09 smithi027 ceph-mon[98970]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:07:09.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:09 smithi027 ceph-mon[98970]: osdmap e91: 8 total, 7 up, 8 in 2024-08-29T15:07:09.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:09 smithi027 ceph-mon[98970]: osdmap e92: 8 total, 7 up, 8 in 2024-08-29T15:07:09.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:09 smithi027 ceph-mon[93921]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:07:09.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:09 smithi027 ceph-mon[93921]: osdmap e91: 8 total, 7 up, 8 in 2024-08-29T15:07:09.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:09 smithi027 ceph-mon[93921]: osdmap e92: 8 total, 7 up, 8 in 2024-08-29T15:07:09.069 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:08 smithi027 podman[109466]: 2024-08-29 15:07:08.881198809 +0000 UTC m=+1.858669727 container cleanup fa01ba5e373707b69d1bad9157164aeee9001ae83c3b03c397a82ac7d86fe9a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=reef-fe43f1c) 2024-08-29T15:07:09.069 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:08 smithi027 bash[109466]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0 2024-08-29T15:07:09.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:09 smithi171 ceph-mon[90031]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:07:09.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:09 smithi171 ceph-mon[90031]: osdmap e91: 8 total, 7 up, 8 in 2024-08-29T15:07:09.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:09 smithi171 ceph-mon[90031]: osdmap e92: 8 total, 7 up, 8 in 2024-08-29T15:07:09.461 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:09 smithi027 podman[109481]: 2024-08-29 15:07:09.115921893 +0000 UTC m=+0.632310985 container remove fa01ba5e373707b69d1bad9157164aeee9001ae83c3b03c397a82ac7d86fe9a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, maintainer=Guillaume Abrioux ) 2024-08-29T15:07:09.818 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:09 smithi027 podman[109567]: 2024-08-29 15:07:09.363908094 +0000 UTC m=+0.016773426 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:07:09.819 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:09 smithi027 podman[109567]: 2024-08-29 15:07:09.466763582 +0000 UTC m=+0.119628905 container create 82853c24a183a161276329ddba0190a020869990dd8dd9ae0e6cf5ad356976d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True) 2024-08-29T15:07:09.819 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:09 smithi027 podman[109567]: 2024-08-29 15:07:09.702730425 +0000 UTC m=+0.355595747 container init 82853c24a183a161276329ddba0190a020869990dd8dd9ae0e6cf5ad356976d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-29T15:07:09.819 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:09 smithi027 podman[109567]: 2024-08-29 15:07:09.706092257 +0000 UTC m=+0.358957586 container start 82853c24a183a161276329ddba0190a020869990dd8dd9ae0e6cf5ad356976d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:07:10.216 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:10 smithi027 ceph-mon[93921]: pgmap v174: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:10.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:10 smithi027 ceph-mon[98970]: pgmap v174: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:10.216 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:09 smithi027 podman[109567]: 2024-08-29 15:07:09.820274291 +0000 UTC m=+0.473139629 container attach 82853c24a183a161276329ddba0190a020869990dd8dd9ae0e6cf5ad356976d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True) 2024-08-29T15:07:10.216 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:09 smithi027 podman[109567]: 2024-08-29 15:07:09.900159772 +0000 UTC m=+0.553025095 container died 82853c24a183a161276329ddba0190a020869990dd8dd9ae0e6cf5ad356976d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-29T15:07:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:10 smithi171 ceph-mon[90031]: pgmap v174: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:10.774 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:10 smithi027 podman[109591]: 2024-08-29 15:07:10.481992659 +0000 UTC m=+0.576008308 container remove 82853c24a183a161276329ddba0190a020869990dd8dd9ae0e6cf5ad356976d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2) 2024-08-29T15:07:10.774 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:10 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.0.service: Deactivated successfully. 2024-08-29T15:07:10.775 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:10 smithi027 systemd[1]: Stopped Ceph osd.0 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:07:10.775 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:10 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.0.service: Consumed 7.206s CPU time. 2024-08-29T15:07:10.775 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:10 smithi027 systemd[1]: Starting Ceph osd.0 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:07:11.055 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:10 smithi027 podman[109690]: 2024-08-29 15:07:10.834980235 +0000 UTC m=+0.018714812 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:07:11.056 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:10 smithi027 podman[109690]: 2024-08-29 15:07:10.949752088 +0000 UTC m=+0.133486647 container create eb476eb61a302ca3c631ab2f83c4ffe087ecc9fdaa50046740362fbe9b2c9f01 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-29T15:07:11.317 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:07:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:07:11.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 podman[109690]: 2024-08-29 15:07:11.177408294 +0000 UTC m=+0.361142853 container init eb476eb61a302ca3c631ab2f83c4ffe087ecc9fdaa50046740362fbe9b2c9f01 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-08-29T15:07:11.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 podman[109690]: 2024-08-29 15:07:11.181283697 +0000 UTC m=+0.365018261 container start eb476eb61a302ca3c631ab2f83c4ffe087ecc9fdaa50046740362fbe9b2c9f01 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-29T15:07:11.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 podman[109690]: 2024-08-29 15:07:11.302566512 +0000 UTC m=+0.486301076 container attach eb476eb61a302ca3c631ab2f83c4ffe087ecc9fdaa50046740362fbe9b2c9f01 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:07:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:11 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:07:11.568 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.568 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-29T15:07:11.569 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-29T15:07:12.049 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-29T15:07:12.049 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-29T15:07:12.049 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-29T15:07:12.049 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-29T15:07:12.049 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-29T15:07:12.050 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-29T15:07:12.050 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-29T15:07:12.050 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-29T15:07:12.050 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate[109713]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-29T15:07:12.050 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 bash[109690]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-29T15:07:12.050 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:11 smithi027 podman[109690]: 2024-08-29 15:07:11.760294822 +0000 UTC m=+0.944029389 container died eb476eb61a302ca3c631ab2f83c4ffe087ecc9fdaa50046740362fbe9b2c9f01 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-29T15:07:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:12 smithi027 ceph-mon[93921]: pgmap v176: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:12 smithi027 ceph-mon[98970]: pgmap v176: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:12 smithi171 ceph-mon[90031]: pgmap v176: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:12.639 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 podman[109832]: 2024-08-29 15:07:12.332033737 +0000 UTC m=+0.565604852 container remove eb476eb61a302ca3c631ab2f83c4ffe087ecc9fdaa50046740362fbe9b2c9f01 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-29T15:07:12.639 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 podman[109892]: 2024-08-29 15:07:12.532847122 +0000 UTC m=+0.108689800 container create 13a2862581674ca9107c716f566f20e62e1f40f49de6c7a6fc733c18228c2771 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-29T15:07:12.639 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 podman[109892]: 2024-08-29 15:07:12.440470847 +0000 UTC m=+0.016313534 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:07:13.068 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 podman[109892]: 2024-08-29 15:07:12.777366629 +0000 UTC m=+0.353209308 container init 13a2862581674ca9107c716f566f20e62e1f40f49de6c7a6fc733c18228c2771 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.schema-version=1.0) 2024-08-29T15:07:13.068 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 podman[109892]: 2024-08-29 15:07:12.78083946 +0000 UTC m=+0.356682143 container start 13a2862581674ca9107c716f566f20e62e1f40f49de6c7a6fc733c18228c2771 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:07:13.068 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 ceph-osd[109911]: -- 172.21.15.27:0/1222049003 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e7a64a1a40 con 0x55e7a647fc00 2024-08-29T15:07:13.068 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 bash[109892]: 13a2862581674ca9107c716f566f20e62e1f40f49de6c7a6fc733c18228c2771 2024-08-29T15:07:13.069 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:12 smithi027 systemd[1]: Started Ceph osd.0 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:07:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:13 smithi171 ceph-mon[90031]: pgmap v177: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[93921]: pgmap v177: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[98970]: pgmap v177: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:14.818 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:14 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[109907]: 2024-08-29T15:07:14.422+0000 7fe727085640 -1 Falling back to public interface 2024-08-29T15:07:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:15 smithi171 ceph-mon[90031]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:15 smithi027 ceph-mon[98970]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:15 smithi027 ceph-mon[93921]: pgmap v178: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[93921]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[98970]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:17 smithi171 ceph-mon[90031]: pgmap v179: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:17.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:18.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[109907]: 2024-08-29T15:07:17.967+0000 7fe727085640 -1 osd.0 90 log_to_monitors true 2024-08-29T15:07:18.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:18 smithi027 ceph-mon[98970]: from='osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-29T15:07:18.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:18 smithi027 ceph-mon[98970]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-29T15:07:18.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:18 smithi027 ceph-mon[93921]: from='osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-29T15:07:18.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:18 smithi027 ceph-mon[93921]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-29T15:07:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:18 smithi171 ceph-mon[90031]: from='osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-29T15:07:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:18 smithi171 ceph-mon[90031]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-29T15:07:18.947 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:07:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[109907]: 2024-08-29T15:07:18.797+0000 7fe71ae27640 -1 osd.0 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T15:07:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:07:19] "GET /metrics HTTP/1.1" 200 33945 "" "Prometheus/2.51.0" 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[93921]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[93921]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[93921]: osdmap e93: 8 total, 7 up, 8 in 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[93921]: from='osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[93921]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[93921]: from='osd.0 ' entity='osd.0' 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[98970]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:19.717 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[98970]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-29T15:07:19.718 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[98970]: osdmap e93: 8 total, 7 up, 8 in 2024-08-29T15:07:19.718 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[98970]: from='osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:19.718 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[98970]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:19.718 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:19 smithi027 ceph-mon[98970]: from='osd.0 ' entity='osd.0' 2024-08-29T15:07:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:19 smithi171 ceph-mon[90031]: pgmap v180: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:19 smithi171 ceph-mon[90031]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-29T15:07:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:19 smithi171 ceph-mon[90031]: osdmap e93: 8 total, 7 up, 8 in 2024-08-29T15:07:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:19 smithi171 ceph-mon[90031]: from='osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:19 smithi171 ceph-mon[90031]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:19.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:19 smithi171 ceph-mon[90031]: from='osd.0 ' entity='osd.0' 2024-08-29T15:07:20.365 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:07:20.253+0000 7f7dea97a640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:07:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423] boot 2024-08-29T15:07:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: osdmap e94: 8 total, 8 up, 8 in 2024-08-29T15:07:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-29T15:07:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-29T15:07:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:07:20.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423] boot 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: osdmap e94: 8 total, 8 up, 8 in 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:20.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: osd.0 [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423] boot 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: osdmap e94: 8 total, 8 up, 8 in 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:20.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:20.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:20.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:20.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:20.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-29T15:07:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[98970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-29T15:07:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[98970]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[98970]: pgmap v183: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[98970]: osdmap e95: 8 total, 8 up, 8 in 2024-08-29T15:07:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[93921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-29T15:07:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[93921]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[93921]: pgmap v183: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:21 smithi027 ceph-mon[93921]: osdmap e95: 8 total, 8 up, 8 in 2024-08-29T15:07:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:21 smithi171 ceph-mon[90031]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-29T15:07:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:21 smithi171 ceph-mon[90031]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:21 smithi171 ceph-mon[90031]: pgmap v183: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:21 smithi171 ceph-mon[90031]: osdmap e95: 8 total, 8 up, 8 in 2024-08-29T15:07:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:23 smithi027 ceph-mon[93921]: pgmap v185: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:23 smithi027 ceph-mon[98970]: pgmap v185: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:23 smithi171 ceph-mon[90031]: pgmap v185: 73 pgs: 17 active+undersized, 9 active+undersized+degraded, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-08-29T15:07:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:24 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2024-08-29T15:07:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:24 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:07:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:24 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2024-08-29T15:07:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:24 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:07:25.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:24 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2024-08-29T15:07:25.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:24 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:07:26.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:25 smithi171 ceph-mon[90031]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:25 smithi027 ceph-mon[98970]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:26.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:25 smithi027 ceph-mon[93921]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:27 smithi171 ceph-mon[90031]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:27 smithi027 ceph-mon[93921]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:27 smithi027 ceph-mon[98970]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:07:29] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.51.0" 2024-08-29T15:07:30.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:29 smithi171 ceph-mon[90031]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:30.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:30.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:29 smithi027 ceph-mon[98970]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:29 smithi027 ceph-mon[93921]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:31 smithi171 ceph-mon[90031]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:31 smithi027 ceph-mon[98970]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:31 smithi027 ceph-mon[93921]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:33.986 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:07:33.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:33 smithi027 ceph-mon[98970]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:33.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:33 smithi027 ceph-mon[93921]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:33 smithi171 ceph-mon[90031]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:34.575 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:07:34.575 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (7m) 18s ago 13m 22.7M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:07:34.575 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (6m) 41s ago 12m 78.4M - c8b91775d855 99f2571ddaab 2024-08-29T15:07:34.575 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (11m) 18s ago 11m 20.5M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (11m) 41s ago 11m 16.7M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (8m) 41s ago 16m 456M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (8m) 18s ago 18m 515M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (6m) 18s ago 18m 53.8M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 1ad77d7a5221 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (3m) 41s ago 17m 42.8M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec e4fb79954e66 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (5m) 18s ago 17m 45.4M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 78f3772b32b4 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (8m) 18s ago 12m 11.4M - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (7m) 41s ago 12m 11.0M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (22s) 18s ago 16m 11.6M 2480M 19.3.0-4579-g1f4056d9 d17246d170ec 13a286258167 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (16m) 18s ago 16m 53.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 03818ecc9de5 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (15m) 18s ago 15m 58.2M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (15m) 18s ago 15m 47.2M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:07:34.576 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (15m) 41s ago 15m 56.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:07:34.577 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (14m) 41s ago 14m 48.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:07:34.577 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (14m) 41s ago 14m 47.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:07:34.577 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (13m) 41s ago 13m 52.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:07:34.577 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (7m) 41s ago 13m 48.1M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:07:34.928 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 3 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 7, 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 1 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:07:34.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:07:34.930 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:07:34.930 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 9, 2024-08-29T15:07:34.930 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 6 2024-08-29T15:07:34.930 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:07:34.930 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:07:34.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:34 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:34.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:34 smithi027 ceph-mon[93921]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:34.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:34 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:34.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:34 smithi027 ceph-mon[98970]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:34 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:34 smithi171 ceph-mon[90031]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:35.232 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:07:35.232 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:07:35.232 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:07:35.232 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-08-29T15:07:35.233 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T15:07:35.233 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "1/8 daemons upgraded", 2024-08-29T15:07:35.233 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-08-29T15:07:35.233 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:07:35.233 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[98970]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[98970]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[98970]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2602341841' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[93921]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[93921]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:35.953 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[93921]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:35.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2602341841' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:35.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:35 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-29T15:07:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:35 smithi171 ceph-mon[90031]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:35 smithi171 ceph-mon[90031]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:35 smithi171 ceph-mon[90031]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2602341841' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:35 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-29T15:07:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[98970]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[98970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-29T15:07:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[98970]: Upgrade: osd.1 is safe to restart 2024-08-29T15:07:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-29T15:07:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:36.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[93921]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:36.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[93921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-29T15:07:36.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[93921]: Upgrade: osd.1 is safe to restart 2024-08-29T15:07:36.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:36.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-29T15:07:36.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:36 smithi171 ceph-mon[90031]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:07:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:36 smithi171 ceph-mon[90031]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-29T15:07:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:36 smithi171 ceph-mon[90031]: Upgrade: osd.1 is safe to restart 2024-08-29T15:07:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-29T15:07:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:37 smithi171 ceph-mon[90031]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:37 smithi171 ceph-mon[90031]: Upgrade: Updating osd.1 2024-08-29T15:07:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:37 smithi171 ceph-mon[90031]: Deploying daemon osd.1 on smithi027 2024-08-29T15:07:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:37 smithi027 ceph-mon[93921]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:37 smithi027 ceph-mon[93921]: Upgrade: Updating osd.1 2024-08-29T15:07:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:37 smithi027 ceph-mon[93921]: Deploying daemon osd.1 on smithi027 2024-08-29T15:07:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:37 smithi027 ceph-mon[98970]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:37 smithi027 ceph-mon[98970]: Upgrade: Updating osd.1 2024-08-29T15:07:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:37 smithi027 ceph-mon[98970]: Deploying daemon osd.1 on smithi027 2024-08-29T15:07:39.057 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:07:39] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.51.0" 2024-08-29T15:07:39.689 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:39 smithi027 systemd[1]: Stopping Ceph osd.1 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:07:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:39 smithi171 ceph-mon[90031]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:39 smithi171 ceph-mon[90031]: osd.1 marked itself down and dead 2024-08-29T15:07:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-mon[93921]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-mon[93921]: osd.1 marked itself down and dead 2024-08-29T15:07:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-mon[98970]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-mon[98970]: osd.1 marked itself down and dead 2024-08-29T15:07:40.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[38855]: 2024-08-29T15:07:39.687+0000 7fe572282640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:07:40.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[38855]: 2024-08-29T15:07:39.687+0000 7fe572282640 -1 osd.1 95 *** Got signal Terminated *** 2024-08-29T15:07:40.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[38855]: 2024-08-29T15:07:39.687+0000 7fe572282640 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:07:40.498 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:40 smithi027 podman[114046]: 2024-08-29 15:07:40.176099933 +0000 UTC m=+0.592626746 container died 03818ecc9de53e649e7826fd5f9eafbe3c2ea0b70d88cb91dc31761cda2aaad0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-08-29T15:07:40.815 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:40 smithi027 ceph-mon[98970]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:07:40.815 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:40 smithi027 ceph-mon[98970]: osdmap e96: 8 total, 7 up, 8 in 2024-08-29T15:07:40.815 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:40 smithi027 ceph-mon[93921]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:07:40.815 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:40 smithi027 ceph-mon[93921]: osdmap e96: 8 total, 7 up, 8 in 2024-08-29T15:07:40.815 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:40 smithi027 podman[114046]: 2024-08-29 15:07:40.612814152 +0000 UTC m=+1.029340960 container cleanup 03818ecc9de53e649e7826fd5f9eafbe3c2ea0b70d88cb91dc31761cda2aaad0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:07:40.816 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:40 smithi027 bash[114046]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1 2024-08-29T15:07:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:40 smithi171 ceph-mon[90031]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:07:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:40 smithi171 ceph-mon[90031]: osdmap e96: 8 total, 7 up, 8 in 2024-08-29T15:07:41.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:40 smithi027 podman[114063]: 2024-08-29 15:07:40.814492186 +0000 UTC m=+0.632354212 container remove 03818ecc9de53e649e7826fd5f9eafbe3c2ea0b70d88cb91dc31761cda2aaad0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, RELEASE=reef-fe43f1c, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-29T15:07:41.501 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:41 smithi027 podman[114152]: 2024-08-29 15:07:41.132244323 +0000 UTC m=+0.016374917 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:07:41.502 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:41 smithi027 podman[114152]: 2024-08-29 15:07:41.240350358 +0000 UTC m=+0.124480943 container create c5576f23bb1e8fb8fdd2108e104505a06b2d22725b8f4000c401ae5e90926415 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d) 2024-08-29T15:07:41.502 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:41 smithi027 podman[114152]: 2024-08-29 15:07:41.501328133 +0000 UTC m=+0.385458730 container init c5576f23bb1e8fb8fdd2108e104505a06b2d22725b8f4000c401ae5e90926415 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-29T15:07:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:41 smithi027 ceph-mon[93921]: pgmap v195: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:41.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:41 smithi027 ceph-mon[93921]: osdmap e97: 8 total, 7 up, 8 in 2024-08-29T15:07:41.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:41 smithi027 ceph-mon[98970]: pgmap v195: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:41.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:41 smithi027 ceph-mon[98970]: osdmap e97: 8 total, 7 up, 8 in 2024-08-29T15:07:41.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:41 smithi027 podman[114152]: 2024-08-29 15:07:41.505036989 +0000 UTC m=+0.389167580 container start c5576f23bb1e8fb8fdd2108e104505a06b2d22725b8f4000c401ae5e90926415 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD) 2024-08-29T15:07:41.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:41 smithi027 podman[114152]: 2024-08-29 15:07:41.567992276 +0000 UTC m=+0.452122875 container attach c5576f23bb1e8fb8fdd2108e104505a06b2d22725b8f4000c401ae5e90926415 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:07:41.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:41 smithi027 podman[114152]: 2024-08-29 15:07:41.703681292 +0000 UTC m=+0.587811885 container died c5576f23bb1e8fb8fdd2108e104505a06b2d22725b8f4000c401ae5e90926415 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-08-29T15:07:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:41 smithi171 ceph-mon[90031]: pgmap v195: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:41 smithi171 ceph-mon[90031]: osdmap e97: 8 total, 7 up, 8 in 2024-08-29T15:07:42.568 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 podman[114175]: 2024-08-29 15:07:42.305592524 +0000 UTC m=+0.596248437 container remove c5576f23bb1e8fb8fdd2108e104505a06b2d22725b8f4000c401ae5e90926415 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:07:42.568 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.1.service: Deactivated successfully. 2024-08-29T15:07:42.568 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 systemd[1]: Stopped Ceph osd.1 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:07:42.568 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.1.service: Consumed 7.954s CPU time. 2024-08-29T15:07:42.568 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 systemd[1]: Starting Ceph osd.1 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:07:43.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 podman[114273]: 2024-08-29 15:07:42.651086226 +0000 UTC m=+0.016630317 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:07:43.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 podman[114273]: 2024-08-29 15:07:42.748658706 +0000 UTC m=+0.114202794 container create 1c0a7017b71d8c45ea359b746dba778b1fff99750777a8977cba4c98f3cb929a (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True) 2024-08-29T15:07:43.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 podman[114273]: 2024-08-29 15:07:42.984542137 +0000 UTC m=+0.350086219 container init 1c0a7017b71d8c45ea359b746dba778b1fff99750777a8977cba4c98f3cb929a (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:07:43.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:42 smithi027 podman[114273]: 2024-08-29 15:07:42.987962809 +0000 UTC m=+0.353506899 container start 1c0a7017b71d8c45ea359b746dba778b1fff99750777a8977cba4c98f3cb929a (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:07:43.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 podman[114273]: 2024-08-29 15:07:43.06773139 +0000 UTC m=+0.433275488 container attach 1c0a7017b71d8c45ea359b746dba778b1fff99750777a8977cba4c98f3cb929a (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-29T15:07:43.532 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.532 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.532 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-29T15:07:43.533 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-29T15:07:43.534 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-29T15:07:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-mon[93921]: pgmap v197: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-mon[98970]: pgmap v197: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:43.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate[114287]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 bash[114273]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-29T15:07:43.819 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:43 smithi027 podman[114273]: 2024-08-29 15:07:43.56240021 +0000 UTC m=+0.927944292 container died 1c0a7017b71d8c45ea359b746dba778b1fff99750777a8977cba4c98f3cb929a (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-29T15:07:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:43 smithi171 ceph-mon[90031]: pgmap v197: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:07:44.546 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 podman[114408]: 2024-08-29 15:07:44.188789926 +0000 UTC m=+0.619548307 container remove 1c0a7017b71d8c45ea359b746dba778b1fff99750777a8977cba4c98f3cb929a (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:07:44.546 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 podman[114467]: 2024-08-29 15:07:44.406594596 +0000 UTC m=+0.113992595 container create fe9dd18c3ce6ed073e91ddd0660c23bd2c9f9cbd9d2ecbbf989823852e2ad4e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-08-29T15:07:44.546 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 podman[114467]: 2024-08-29 15:07:44.309903834 +0000 UTC m=+0.017301803 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:07:45.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:44 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-08-29T15:07:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:44 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-08-29T15:07:45.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 podman[114467]: 2024-08-29 15:07:44.642290291 +0000 UTC m=+0.349688258 container init fe9dd18c3ce6ed073e91ddd0660c23bd2c9f9cbd9d2ecbbf989823852e2ad4e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, GIT_CLEAN=True, ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-29T15:07:45.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 podman[114467]: 2024-08-29 15:07:44.645605467 +0000 UTC m=+0.353003424 container start fe9dd18c3ce6ed073e91ddd0660c23bd2c9f9cbd9d2ecbbf989823852e2ad4e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-29T15:07:45.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 ceph-osd[114486]: -- 172.21.15.27:0/556828978 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5619e8a39a40 con 0x5619e8a17c00 2024-08-29T15:07:45.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 bash[114467]: fe9dd18c3ce6ed073e91ddd0660c23bd2c9f9cbd9d2ecbbf989823852e2ad4e9 2024-08-29T15:07:45.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:44 smithi027 systemd[1]: Started Ceph osd.1 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:07:45.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:44 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 5 pgs degraded (PG_DEGRADED) 2024-08-29T15:07:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[93921]: pgmap v198: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[98970]: pgmap v198: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:46.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:45 smithi171 ceph-mon[90031]: pgmap v198: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:46.568 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:46 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[114482]: 2024-08-29T15:07:46.279+0000 7f1a06463640 -1 Falling back to public interface 2024-08-29T15:07:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:47 smithi027 ceph-mon[98970]: pgmap v199: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:47 smithi027 ceph-mon[93921]: pgmap v199: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:47 smithi171 ceph-mon[90031]: pgmap v199: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:07:49] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.51.0" 2024-08-29T15:07:49.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[93921]: pgmap v200: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:49.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[98970]: pgmap v200: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:49 smithi171 ceph-mon[90031]: pgmap v200: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:50.107 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[114482]: 2024-08-29T15:07:49.818+0000 7f1a06463640 -1 osd.1 95 log_to_monitors true 2024-08-29T15:07:50.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:50 smithi027 ceph-mon[93921]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-29T15:07:50.448 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:07:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[114482]: 2024-08-29T15:07:50.136+0000 7f19fe20d640 -1 osd.1 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T15:07:50.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:50 smithi027 ceph-mon[98970]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-29T15:07:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:50 smithi171 ceph-mon[90031]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-29T15:07:51.440 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[93921]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-29T15:07:51.440 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[93921]: osdmap e98: 8 total, 7 up, 8 in 2024-08-29T15:07:51.440 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[93921]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:51.440 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[93921]: pgmap v202: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:51.441 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[98970]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-29T15:07:51.441 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[98970]: osdmap e98: 8 total, 7 up, 8 in 2024-08-29T15:07:51.441 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[98970]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:51.441 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:51 smithi027 ceph-mon[98970]: pgmap v202: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:51 smithi171 ceph-mon[90031]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-29T15:07:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:51 smithi171 ceph-mon[90031]: osdmap e98: 8 total, 7 up, 8 in 2024-08-29T15:07:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:51 smithi171 ceph-mon[90031]: from='osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:07:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:51 smithi171 ceph-mon[90031]: pgmap v202: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:52.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:07:52.236+0000 7f7dea97a640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:07:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131] boot 2024-08-29T15:07:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: osdmap e99: 8 total, 8 up, 8 in 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131] boot 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: osdmap e99: 8 total, 8 up, 8 in 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-29T15:07:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:52.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:52.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:52 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:07:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: osd.1 [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131] boot 2024-08-29T15:07:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: osdmap e99: 8 total, 8 up, 8 in 2024-08-29T15:07:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-29T15:07:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:07:52.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:07:52.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:52 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:07:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: osdmap e100: 8 total, 8 up, 8 in 2024-08-29T15:07:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-29T15:07:53.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-29T15:07:53.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:53.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:53 smithi171 ceph-mon[90031]: pgmap v205: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: osdmap e100: 8 total, 8 up, 8 in 2024-08-29T15:07:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[98970]: pgmap v205: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: osdmap e100: 8 total, 8 up, 8 in 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-29T15:07:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-29T15:07:53.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:07:53.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:53 smithi027 ceph-mon[93921]: pgmap v205: 73 pgs: 27 active+undersized, 5 active+undersized+degraded, 41 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:07:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:54 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:07:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:54 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:07:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:54 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:07:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:55 smithi171 ceph-mon[90031]: pgmap v206: 73 pgs: 9 active+undersized, 1 active+undersized+degraded, 63 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-08-29T15:07:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:55 smithi027 ceph-mon[93921]: pgmap v206: 73 pgs: 9 active+undersized, 1 active+undersized+degraded, 63 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-08-29T15:07:55.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:55 smithi027 ceph-mon[98970]: pgmap v206: 73 pgs: 9 active+undersized, 1 active+undersized+degraded, 63 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-08-29T15:07:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:56 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-08-29T15:07:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:56 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:07:56.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:56 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-08-29T15:07:56.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:56 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:07:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:56 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-08-29T15:07:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:56 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:07:57.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:57 smithi171 ceph-mon[90031]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:57.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:57 smithi027 ceph-mon[93921]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:57.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:57 smithi027 ceph-mon[98970]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:07:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:07:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:07:59] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.51.0" 2024-08-29T15:08:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:59 smithi171 ceph-mon[90031]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:07:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:59 smithi027 ceph-mon[98970]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:07:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:59 smithi027 ceph-mon[93921]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:07:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:01 smithi171 ceph-mon[90031]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:01 smithi027 ceph-mon[98970]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:01 smithi027 ceph-mon[93921]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:03 smithi171 ceph-mon[90031]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:03 smithi027 ceph-mon[93921]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:03 smithi027 ceph-mon[98970]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:04 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:05.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:04 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:04 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:05.553 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:08:05.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:05 smithi027 ceph-mon[93921]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:05.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:05 smithi027 ceph-mon[98970]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:05 smithi171 ceph-mon[90031]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (7m) 18s ago 13m 22.7M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (7m) 73s ago 13m 78.4M - c8b91775d855 99f2571ddaab 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (12m) 18s ago 12m 20.8M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (12m) 73s ago 12m 16.7M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (8m) 73s ago 17m 456M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (9m) 18s ago 19m 516M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (6m) 18s ago 19m 58.6M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 1ad77d7a5221 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (4m) 73s ago 17m 42.8M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec e4fb79954e66 2024-08-29T15:08:06.143 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6m) 18s ago 17m 49.1M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 78f3772b32b4 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (8m) 18s ago 13m 11.3M - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (8m) 73s ago 12m 11.0M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (53s) 18s ago 17m 47.4M 2480M 19.3.0-4579-g1f4056d9 d17246d170ec 13a286258167 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (21s) 18s ago 16m 12.1M 2480M 19.3.0-4579-g1f4056d9 d17246d170ec fe9dd18c3ce6 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (16m) 18s ago 16m 59.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a 5baee5a0efbf 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (16m) 18s ago 15m 48.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (15m) 73s ago 15m 56.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (15m) 73s ago 15m 48.4M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (14m) 73s ago 14m 47.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (14m) 73s ago 14m 52.3M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:08:06.144 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (8m) 73s ago 13m 48.1M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:08:06.496 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:08:06.496 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:08:06.496 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 3 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 6, 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:08:06.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:06.498 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:08:06.498 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 8, 2024-08-29T15:08:06.498 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 7 2024-08-29T15:08:06.498 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:08:06.498 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:08:06.797 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:08:06.797 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:08:06.797 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:08:06.797 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-08-29T15:08:06.797 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T15:08:06.797 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/8 daemons upgraded", 2024-08-29T15:08:06.798 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-08-29T15:08:06.798 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:08:06.798 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:08:06.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:06 smithi027 ceph-mon[98970]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:06.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:06 smithi027 ceph-mon[98970]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:06.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3457966276' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:06.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:06 smithi027 ceph-mon[93921]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:06.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:06 smithi027 ceph-mon[93921]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:06.799 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3457966276' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:06 smithi171 ceph-mon[90031]: from='client.44294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:06 smithi171 ceph-mon[90031]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3457966276' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:07.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[93921]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:07.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[93921]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:07.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[93921]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:07.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-29T15:08:07.936 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[98970]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:07.936 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[98970]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:07.936 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[98970]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:07.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-29T15:08:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:07 smithi171 ceph-mon[90031]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:07 smithi171 ceph-mon[90031]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:07 smithi171 ceph-mon[90031]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:08.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-29T15:08:09.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[93921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-29T15:08:09.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[93921]: Upgrade: osd.2 is safe to restart 2024-08-29T15:08:09.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:09.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-29T15:08:09.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[98970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-29T15:08:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[98970]: Upgrade: osd.2 is safe to restart 2024-08-29T15:08:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-29T15:08:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:08 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:09.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:08 smithi171 ceph-mon[90031]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-29T15:08:09.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:08 smithi171 ceph-mon[90031]: Upgrade: osd.2 is safe to restart 2024-08-29T15:08:09.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:08 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:09.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:08 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-29T15:08:09.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:08 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:09.260 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:08:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:08:09] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.51.0" 2024-08-29T15:08:09.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:09 smithi027 ceph-mon[98970]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:09.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:09 smithi027 ceph-mon[98970]: Upgrade: Updating osd.2 2024-08-29T15:08:09.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:09 smithi027 ceph-mon[98970]: Deploying daemon osd.2 on smithi027 2024-08-29T15:08:09.931 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:09 smithi027 ceph-mon[93921]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:09.931 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:09 smithi027 ceph-mon[93921]: Upgrade: Updating osd.2 2024-08-29T15:08:09.931 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:09 smithi027 ceph-mon[93921]: Deploying daemon osd.2 on smithi027 2024-08-29T15:08:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:09 smithi171 ceph-mon[90031]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:09 smithi171 ceph-mon[90031]: Upgrade: Updating osd.2 2024-08-29T15:08:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:09 smithi171 ceph-mon[90031]: Deploying daemon osd.2 on smithi027 2024-08-29T15:08:11.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:11 smithi027 ceph-mon[98970]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:11.961 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:11 smithi027 ceph-mon[93921]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:11.961 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:11 smithi027 systemd[1]: Stopping Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:08:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:11 smithi171 ceph-mon[90031]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:12.318 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[44691]: 2024-08-29T15:08:11.959+0000 7f22a4f68640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:08:12.318 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[44691]: 2024-08-29T15:08:11.959+0000 7f22a4f68640 -1 osd.2 100 *** Got signal Terminated *** 2024-08-29T15:08:12.318 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:11 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[44691]: 2024-08-29T15:08:11.959+0000 7f22a4f68640 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:08:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:12 smithi171 ceph-mon[90031]: osd.2 marked itself down and dead 2024-08-29T15:08:13.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:12 smithi027 ceph-mon[93921]: osd.2 marked itself down and dead 2024-08-29T15:08:13.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:12 smithi027 ceph-mon[98970]: osd.2 marked itself down and dead 2024-08-29T15:08:13.403 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:13 smithi027 podman[118619]: 2024-08-29 15:08:13.104633539 +0000 UTC m=+1.256539494 container died 5baee5a0efbf9312ad346526e4b24e5fdb65fae7fafc6b70df512447b884fd18 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:08:13.749 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:13 smithi027 podman[118619]: 2024-08-29 15:08:13.492930579 +0000 UTC m=+1.644836532 container cleanup 5baee5a0efbf9312ad346526e4b24e5fdb65fae7fafc6b70df512447b884fd18 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-08-29T15:08:13.749 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:13 smithi027 bash[118619]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2 2024-08-29T15:08:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:13 smithi171 ceph-mon[90031]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:13 smithi171 ceph-mon[90031]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:08:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:13 smithi171 ceph-mon[90031]: osdmap e101: 8 total, 7 up, 8 in 2024-08-29T15:08:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[98970]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[98970]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:08:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[98970]: osdmap e101: 8 total, 7 up, 8 in 2024-08-29T15:08:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[93921]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[93921]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:08:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[93921]: osdmap e101: 8 total, 7 up, 8 in 2024-08-29T15:08:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:14.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:13 smithi027 podman[118634]: 2024-08-29 15:08:13.838061255 +0000 UTC m=+0.728750295 container remove 5baee5a0efbf9312ad346526e4b24e5fdb65fae7fafc6b70df512447b884fd18 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-29T15:08:14.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:14 smithi027 podman[118725]: 2024-08-29 15:08:14.272240798 +0000 UTC m=+0.113087712 container create 61e9ad639009fcad8bd5a4dfdebbd5ae786a1fc64f5d7a06fe466bd641902c9e (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-08-29T15:08:14.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:14 smithi027 podman[118725]: 2024-08-29 15:08:14.175541597 +0000 UTC m=+0.016388504 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:08:14.569 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:14 smithi027 podman[118725]: 2024-08-29 15:08:14.499707223 +0000 UTC m=+0.340554132 container init 61e9ad639009fcad8bd5a4dfdebbd5ae786a1fc64f5d7a06fe466bd641902c9e (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-29T15:08:14.569 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:14 smithi027 podman[118725]: 2024-08-29 15:08:14.503231632 +0000 UTC m=+0.344078549 container start 61e9ad639009fcad8bd5a4dfdebbd5ae786a1fc64f5d7a06fe466bd641902c9e (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True) 2024-08-29T15:08:14.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:14 smithi027 ceph-mon[98970]: osdmap e102: 8 total, 7 up, 8 in 2024-08-29T15:08:14.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:14 smithi027 ceph-mon[93921]: osdmap e102: 8 total, 7 up, 8 in 2024-08-29T15:08:14.988 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:14 smithi027 podman[118725]: 2024-08-29 15:08:14.592324589 +0000 UTC m=+0.433171508 container attach 61e9ad639009fcad8bd5a4dfdebbd5ae786a1fc64f5d7a06fe466bd641902c9e (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-08-29T15:08:14.988 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:14 smithi027 podman[118725]: 2024-08-29 15:08:14.697084325 +0000 UTC m=+0.537931237 container died 61e9ad639009fcad8bd5a4dfdebbd5ae786a1fc64f5d7a06fe466bd641902c9e (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-08-29T15:08:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:14 smithi171 ceph-mon[90031]: osdmap e102: 8 total, 7 up, 8 in 2024-08-29T15:08:15.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 podman[118749]: 2024-08-29 15:08:15.287203325 +0000 UTC m=+0.582583554 container remove 61e9ad639009fcad8bd5a4dfdebbd5ae786a1fc64f5d7a06fe466bd641902c9e (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-29T15:08:15.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service: Deactivated successfully. 2024-08-29T15:08:15.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 systemd[1]: Stopped Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:08:15.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service: Consumed 7.023s CPU time. 2024-08-29T15:08:15.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 systemd[1]: Starting Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:08:15.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:15 smithi027 ceph-mon[93921]: pgmap v218: 73 pgs: 10 active+undersized, 2 peering, 1 stale+active+clean, 2 active+undersized+degraded, 58 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:15.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:15 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:15.992 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:15 smithi027 ceph-mon[98970]: pgmap v218: 73 pgs: 10 active+undersized, 2 peering, 1 stale+active+clean, 2 active+undersized+degraded, 58 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:15.992 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:15 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:15.992 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 podman[118850]: 2024-08-29 15:08:15.629575345 +0000 UTC m=+0.016503878 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:08:15.993 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 podman[118850]: 2024-08-29 15:08:15.760136467 +0000 UTC m=+0.147064988 container create ff08f4fbf8ea39092324d945541fdbdc4da716f550ca5fca932a38c59eab0595 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-29T15:08:15.993 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 podman[118850]: 2024-08-29 15:08:15.991426261 +0000 UTC m=+0.378354782 container init ff08f4fbf8ea39092324d945541fdbdc4da716f550ca5fca932a38c59eab0595 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-29T15:08:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:15 smithi171 ceph-mon[90031]: pgmap v218: 73 pgs: 10 active+undersized, 2 peering, 1 stale+active+clean, 2 active+undersized+degraded, 58 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:15 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:15 smithi027 podman[118850]: 2024-08-29 15:08:15.99556033 +0000 UTC m=+0.382488859 container start ff08f4fbf8ea39092324d945541fdbdc4da716f550ca5fca932a38c59eab0595 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 podman[118850]: 2024-08-29 15:08:16.083092125 +0000 UTC m=+0.470020664 container attach ff08f4fbf8ea39092324d945541fdbdc4da716f550ca5fca932a38c59eab0595 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:08:16.251 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.252 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.252 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.252 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:16.252 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:08:16.252 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:08:16.545 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-29T15:08:16.545 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-29T15:08:16.545 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:08:16.545 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:08:16.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:08:16.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:08:16.819 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-29T15:08:16.819 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-29T15:08:16.819 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:08:16.819 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:08:16.819 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[118865]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-29T15:08:16.819 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 bash[118850]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-29T15:08:16.819 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:16 smithi027 podman[118850]: 2024-08-29 15:08:16.573323761 +0000 UTC m=+0.960252282 container died ff08f4fbf8ea39092324d945541fdbdc4da716f550ca5fca932a38c59eab0595 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-08-29T15:08:17.315 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 podman[118977]: 2024-08-29 15:08:17.103917317 +0000 UTC m=+0.519773198 container remove ff08f4fbf8ea39092324d945541fdbdc4da716f550ca5fca932a38c59eab0595 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:08:17.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 podman[119035]: 2024-08-29 15:08:17.217865156 +0000 UTC m=+0.016840609 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:08:17.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 podman[119035]: 2024-08-29 15:08:17.321587299 +0000 UTC m=+0.120562785 container create 6dd209a7d75d1065ccec8bfd1dadd6e2037a030977b6dc803cacb47cec02ee52 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-08-29T15:08:17.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 podman[119035]: 2024-08-29 15:08:17.548890169 +0000 UTC m=+0.347865613 container init 6dd209a7d75d1065ccec8bfd1dadd6e2037a030977b6dc803cacb47cec02ee52 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-29T15:08:17.569 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 podman[119035]: 2024-08-29 15:08:17.552203831 +0000 UTC m=+0.351179282 container start 6dd209a7d75d1065ccec8bfd1dadd6e2037a030977b6dc803cacb47cec02ee52 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-29T15:08:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:17 smithi171 ceph-mon[90031]: pgmap v219: 73 pgs: 15 active+undersized, 2 peering, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[93921]: pgmap v219: 73 pgs: 15 active+undersized, 2 peering, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[98970]: pgmap v219: 73 pgs: 15 active+undersized, 2 peering, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:18.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 ceph-osd[119055]: -- 172.21.15.27:0/2521877565 <== mon.2 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557685991a40 con 0x55768675e000 2024-08-29T15:08:18.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 bash[119035]: 6dd209a7d75d1065ccec8bfd1dadd6e2037a030977b6dc803cacb47cec02ee52 2024-08-29T15:08:18.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:17 smithi027 systemd[1]: Started Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:08:19.282 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[119051]: 2024-08-29T15:08:19.189+0000 7f5f90aed640 -1 Falling back to public interface 2024-08-29T15:08:19.282 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:08:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:08:19] "GET /metrics HTTP/1.1" 200 33940 "" "Prometheus/2.51.0" 2024-08-29T15:08:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:19 smithi027 ceph-mon[98970]: pgmap v220: 73 pgs: 15 active+undersized, 2 peering, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:19 smithi027 ceph-mon[93921]: pgmap v220: 73 pgs: 15 active+undersized, 2 peering, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:19 smithi171 ceph-mon[90031]: pgmap v220: 73 pgs: 15 active+undersized, 2 peering, 2 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-08-29T15:08:21.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:20 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:21.160 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:20 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:21.218 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:20 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[98970]: pgmap v221: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[93921]: pgmap v221: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:21 smithi171 ceph-mon[90031]: pgmap v221: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:08:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:22.801 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:22 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[119051]: 2024-08-29T15:08:22.499+0000 7f5f90aed640 -1 osd.2 100 log_to_monitors true 2024-08-29T15:08:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:23 smithi171 ceph-mon[90031]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:08:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:23 smithi171 ceph-mon[90031]: from='osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:08:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:23 smithi171 ceph-mon[90031]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:08:23.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[119051]: 2024-08-29T15:08:23.193+0000 7f5f8488f640 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T15:08:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[93921]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:08:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[93921]: from='osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:08:23.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[93921]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:08:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[98970]: pgmap v222: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 246 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-08-29T15:08:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[98970]: from='osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:08:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:23 smithi027 ceph-mon[98970]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:08:24.184 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[98970]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T15:08:24.184 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[98970]: osdmap e103: 8 total, 7 up, 8 in 2024-08-29T15:08:24.184 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[98970]: from='osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:24.184 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[98970]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:24 smithi171 ceph-mon[90031]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T15:08:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:24 smithi171 ceph-mon[90031]: osdmap e103: 8 total, 7 up, 8 in 2024-08-29T15:08:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:24 smithi171 ceph-mon[90031]: from='osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:24 smithi171 ceph-mon[90031]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:24.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[93921]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T15:08:24.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[93921]: osdmap e103: 8 total, 7 up, 8 in 2024-08-29T15:08:24.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[93921]: from='osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:24.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:24 smithi027 ceph-mon[93921]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[93921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:08:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[93921]: osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233] boot 2024-08-29T15:08:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[93921]: osdmap e104: 8 total, 8 up, 8 in 2024-08-29T15:08:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-29T15:08:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[93921]: pgmap v225: 73 pgs: 11 peering, 7 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[93921]: osdmap e105: 8 total, 8 up, 8 in 2024-08-29T15:08:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[98970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:08:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[98970]: osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233] boot 2024-08-29T15:08:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[98970]: osdmap e104: 8 total, 8 up, 8 in 2024-08-29T15:08:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-29T15:08:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[98970]: pgmap v225: 73 pgs: 11 peering, 7 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-mon[98970]: osdmap e105: 8 total, 8 up, 8 in 2024-08-29T15:08:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:25 smithi171 ceph-mon[90031]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:08:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:25 smithi171 ceph-mon[90031]: osd.2 [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233] boot 2024-08-29T15:08:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:25 smithi171 ceph-mon[90031]: osdmap e104: 8 total, 8 up, 8 in 2024-08-29T15:08:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-29T15:08:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:25 smithi171 ceph-mon[90031]: pgmap v225: 73 pgs: 11 peering, 7 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:25 smithi171 ceph-mon[90031]: osdmap e105: 8 total, 8 up, 8 in 2024-08-29T15:08:25.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:08:25 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:08:25.636+0000 7f7dea97a640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:08:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-29T15:08:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:26 smithi171 ceph-mon[90031]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:08:26.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-29T15:08:26.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-29T15:08:26.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[93921]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:08:26.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-29T15:08:26.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:26 smithi027 ceph-mon[98970]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-08-29T15:08:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:27 smithi027 ceph-mon[93921]: pgmap v227: 73 pgs: 15 peering, 3 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:08:27.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:27 smithi027 ceph-mon[98970]: pgmap v227: 73 pgs: 15 peering, 3 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:27.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:08:28.002 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:27 smithi171 ceph-mon[90031]: pgmap v227: 73 pgs: 15 peering, 3 active+undersized, 1 active+undersized+degraded, 54 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:28.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:08:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:08:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:08:29] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.51.0" 2024-08-29T15:08:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:29 smithi171 ceph-mon[90031]: pgmap v228: 73 pgs: 15 peering, 1 active+undersized, 1 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:29 smithi027 ceph-mon[98970]: pgmap v228: 73 pgs: 15 peering, 1 active+undersized, 1 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:29 smithi027 ceph-mon[93921]: pgmap v228: 73 pgs: 15 peering, 1 active+undersized, 1 active+undersized+degraded, 56 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:08:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:31.024 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:30 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-08-29T15:08:31.025 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:30 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:08:31.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:30 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-08-29T15:08:31.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:30 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:08:31.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:30 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-08-29T15:08:31.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:30 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:08:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:31 smithi171 ceph-mon[90031]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:31 smithi027 ceph-mon[93921]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:31 smithi027 ceph-mon[98970]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:33 smithi171 ceph-mon[90031]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:33 smithi027 ceph-mon[93921]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:33 smithi027 ceph-mon[98970]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:35 smithi171 ceph-mon[90031]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:35 smithi027 ceph-mon[93921]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:36.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:35 smithi027 ceph-mon[98970]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:37.123 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-08-29T15:08:37.711 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-29T15:08:37.711 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (8m) 16s ago 14m 22.7M - 0.27.0 11f11916f8cd 4ac5fe286fab 2024-08-29T15:08:37.711 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi171 *:3000 running (7m) 15s ago 13m 76.8M - c8b91775d855 99f2571ddaab 2024-08-29T15:08:37.711 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi027.afjjno smithi027 running (12m) 16s ago 12m 21.0M - 18.2.4-774-gfe43f1c9 96384990183a 2387b5d29d83 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:mds.foo.smithi171.grbaeh smithi171 running (12m) 15s ago 12m 17.1M - 18.2.4-774-gfe43f1c9 96384990183a c035e3200ae3 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi171 *:8443,9283,8765 running (9m) 15s ago 18m 456M - 19.3.0-4579-g1f4056d9 d17246d170ec 85b20cdc0955 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,9283,8765 running (9m) 16s ago 19m 517M - 19.3.0-4579-g1f4056d9 d17246d170ec f766ab17b7e2 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (7m) 16s ago 19m 58.4M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 1ad77d7a5221 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi171 running (5m) 15s ago 18m 46.0M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec e4fb79954e66 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6m) 16s ago 18m 48.9M 2048M 19.3.0-4579-g1f4056d9 d17246d170ec 78f3772b32b4 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (9m) 16s ago 13m 11.3M - 1.7.0 72c9c2088986 e53886849ab5 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi171 *:9100 running (9m) 15s ago 13m 12.1M - 1.7.0 72c9c2088986 46afc3b54404 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (85s) 16s ago 17m 48.5M 2480M 19.3.0-4579-g1f4056d9 d17246d170ec 13a286258167 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (53s) 16s ago 17m 41.3M 2480M 19.3.0-4579-g1f4056d9 d17246d170ec fe9dd18c3ce6 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (20s) 16s ago 16m 12.2M 2480M 19.3.0-4579-g1f4056d9 d17246d170ec 6dd209a7d75d 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (16m) 16s ago 16m 49.2M 2480M 18.2.4-774-gfe43f1c9 96384990183a 598548b545e1 2024-08-29T15:08:37.712 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi171 running (16m) 15s ago 16m 57.9M 2480M 18.2.4-774-gfe43f1c9 96384990183a f00fb58eb4c2 2024-08-29T15:08:37.713 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi171 running (15m) 15s ago 15m 48.8M 2480M 18.2.4-774-gfe43f1c9 96384990183a 570b002456a5 2024-08-29T15:08:37.713 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi171 running (15m) 15s ago 15m 48.6M 2480M 18.2.4-774-gfe43f1c9 96384990183a 228ad60cc492 2024-08-29T15:08:37.713 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi171 running (14m) 15s ago 14m 53.5M 2480M 18.2.4-774-gfe43f1c9 96384990183a c43a0db5d0af 2024-08-29T15:08:37.713 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi171 *:9095 running (8m) 15s ago 14m 46.3M - 2.51.0 1d3b7f56885b b0e73daa625e 2024-08-29T15:08:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:37 smithi171 ceph-mon[90031]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:38.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:37 smithi027 ceph-mon[98970]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:38.066 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:08:38.066 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 3 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 2 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 5, 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 3 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": { 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 2 2024-08-29T15:08:38.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:08:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-08-29T15:08:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-774-gfe43f1c9 (fe43f1c9f9e2945dda7244d8acbd33fb921449af) reef (stable)": 7, 2024-08-29T15:08:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 19.3.0-4579-g1f4056d9 (1f4056d968930e12926c25a8f13ee8ed6047c791) squid (dev)": 8 2024-08-29T15:08:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:08:38.068 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:08:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:37 smithi027 ceph-mon[93921]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:38.367 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:08:38.367 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:08:38.367 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-08-29T15:08:38.367 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-08-29T15:08:38.367 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-08-29T15:08:38.367 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "3/8 daemons upgraded", 2024-08-29T15:08:38.368 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-08-29T15:08:38.368 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-08-29T15:08:38.368 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:08:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[98970]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[98970]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[98970]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2593318529' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:39.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[93921]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[93921]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[93921]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2593318529' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:39.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:38 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:38 smithi171 ceph-mon[90031]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:38 smithi171 ceph-mon[90031]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:38 smithi171 ceph-mon[90031]: from='client.44342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2593318529' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:38 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:08:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:08:39] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.51.0" 2024-08-29T15:08:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:39 smithi171 ceph-mon[90031]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:39 smithi171 ceph-mon[90031]: from='client.34401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:39 smithi027 ceph-mon[98970]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:39 smithi027 ceph-mon[98970]: from='client.34401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:39 smithi027 ceph-mon[93921]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:39 smithi027 ceph-mon[93921]: from='client.34401 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:08:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:40 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-29T15:08:41.034 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:40 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-29T15:08:41.034 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:40 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-29T15:08:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:41 smithi171 ceph-mon[90031]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:41 smithi171 ceph-mon[90031]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-29T15:08:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:41 smithi171 ceph-mon[90031]: Upgrade: osd.3 is safe to restart 2024-08-29T15:08:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:41 smithi027 ceph-mon[93921]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:41 smithi027 ceph-mon[93921]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-29T15:08:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:41 smithi027 ceph-mon[93921]: Upgrade: osd.3 is safe to restart 2024-08-29T15:08:42.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:41 smithi027 ceph-mon[98970]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:42.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:41 smithi027 ceph-mon[98970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-29T15:08:42.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:41 smithi027 ceph-mon[98970]: Upgrade: osd.3 is safe to restart 2024-08-29T15:08:43.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[98970]: Upgrade: Updating osd.3 2024-08-29T15:08:43.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:43.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-29T15:08:43.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[98970]: Deploying daemon osd.3 on smithi027 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[98970]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[93921]: Upgrade: Updating osd.3 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[93921]: Deploying daemon osd.3 on smithi027 2024-08-29T15:08:43.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:43 smithi027 ceph-mon[93921]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:43 smithi171 ceph-mon[90031]: Upgrade: Updating osd.3 2024-08-29T15:08:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-29T15:08:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:43 smithi171 ceph-mon[90031]: Deploying daemon osd.3 on smithi027 2024-08-29T15:08:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:43 smithi171 ceph-mon[90031]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:44.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:44.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:44.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:44.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:45.311 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:45 smithi027 systemd[1]: Stopping Ceph osd.3 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:08:45.568 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[50900]: 2024-08-29T15:08:45.309+0000 7f227ed3f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:08:45.568 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[50900]: 2024-08-29T15:08:45.309+0000 7f227ed3f640 -1 osd.3 105 *** Got signal Terminated *** 2024-08-29T15:08:45.568 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[50900]: 2024-08-29T15:08:45.309+0000 7f227ed3f640 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:08:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:45 smithi171 ceph-mon[90031]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:45 smithi171 ceph-mon[90031]: osd.3 marked itself down and dead 2024-08-29T15:08:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:45 smithi027 ceph-mon[93921]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:45 smithi027 ceph-mon[93921]: osd.3 marked itself down and dead 2024-08-29T15:08:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:45 smithi027 ceph-mon[98970]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:45 smithi027 ceph-mon[98970]: osd.3 marked itself down and dead 2024-08-29T15:08:46.495 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:46 smithi027 podman[123351]: 2024-08-29 15:08:46.171151622 +0000 UTC m=+0.961731999 container died 598548b545e13e1c17e248d139aa94107cfc237b74050d2215a13735f3e3d60a (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-08-29T15:08:46.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:46 smithi027 ceph-mon[98970]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:08:46.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:46 smithi027 ceph-mon[98970]: osdmap e106: 8 total, 7 up, 8 in 2024-08-29T15:08:46.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:46 smithi027 ceph-mon[98970]: osdmap e107: 8 total, 7 up, 8 in 2024-08-29T15:08:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:46 smithi027 ceph-mon[93921]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:08:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:46 smithi027 ceph-mon[93921]: osdmap e106: 8 total, 7 up, 8 in 2024-08-29T15:08:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:46 smithi027 ceph-mon[93921]: osdmap e107: 8 total, 7 up, 8 in 2024-08-29T15:08:46.819 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:46 smithi027 podman[123351]: 2024-08-29 15:08:46.702156417 +0000 UTC m=+1.492736794 container cleanup 598548b545e13e1c17e248d139aa94107cfc237b74050d2215a13735f3e3d60a (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, maintainer=Guillaume Abrioux , io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-08-29T15:08:46.819 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:46 smithi027 bash[123351]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3 2024-08-29T15:08:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:46 smithi171 ceph-mon[90031]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:08:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:46 smithi171 ceph-mon[90031]: osdmap e106: 8 total, 7 up, 8 in 2024-08-29T15:08:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:46 smithi171 ceph-mon[90031]: osdmap e107: 8 total, 7 up, 8 in 2024-08-29T15:08:47.318 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:46 smithi027 podman[123366]: 2024-08-29 15:08:46.937272857 +0000 UTC m=+0.762007743 container remove 598548b545e13e1c17e248d139aa94107cfc237b74050d2215a13735f3e3d60a (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, RELEASE=reef-fe43f1c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-08-29T15:08:47.667 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:47 smithi027 podman[123457]: 2024-08-29 15:08:47.363205245 +0000 UTC m=+0.101510882 container create de3ff9384f8bbb5ba41a91cd053611d9a8145abbde358331438c2b2a3118fa73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:08:47.667 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:47 smithi027 podman[123457]: 2024-08-29 15:08:47.277983347 +0000 UTC m=+0.016289001 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:08:47.667 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:47 smithi027 podman[123457]: 2024-08-29 15:08:47.56574481 +0000 UTC m=+0.304050447 container init de3ff9384f8bbb5ba41a91cd053611d9a8145abbde358331438c2b2a3118fa73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-29T15:08:47.667 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:47 smithi027 podman[123457]: 2024-08-29 15:08:47.569350565 +0000 UTC m=+0.307656204 container start de3ff9384f8bbb5ba41a91cd053611d9a8145abbde358331438c2b2a3118fa73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:08:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:47 smithi171 ceph-mon[90031]: pgmap v238: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:47 smithi027 ceph-mon[98970]: pgmap v238: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:47 smithi027 ceph-mon[93921]: pgmap v238: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:48.069 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:47 smithi027 podman[123457]: 2024-08-29 15:08:47.665623249 +0000 UTC m=+0.403928887 container attach de3ff9384f8bbb5ba41a91cd053611d9a8145abbde358331438c2b2a3118fa73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-29T15:08:48.069 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:47 smithi027 podman[123457]: 2024-08-29 15:08:47.769810977 +0000 UTC m=+0.508116616 container died de3ff9384f8bbb5ba41a91cd053611d9a8145abbde358331438c2b2a3118fa73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD) 2024-08-29T15:08:48.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:48 smithi027 podman[123480]: 2024-08-29 15:08:48.503788383 +0000 UTC m=+0.727940156 container remove de3ff9384f8bbb5ba41a91cd053611d9a8145abbde358331438c2b2a3118fa73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-29T15:08:48.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:48 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.3.service: Deactivated successfully. 2024-08-29T15:08:48.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:48 smithi027 systemd[1]: Stopped Ceph osd.3 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:08:48.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:48 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.3.service: Consumed 6.262s CPU time. 2024-08-29T15:08:48.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:48 smithi027 systemd[1]: Starting Ceph osd.3 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:08:49.199 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:08:49] "GET /metrics HTTP/1.1" 200 33935 "" "Prometheus/2.51.0" 2024-08-29T15:08:49.199 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:48 smithi027 podman[123580]: 2024-08-29 15:08:48.938389677 +0000 UTC m=+0.101249591 container create 9f70b396270d8c2e9bd2be76517f02c49d421becfefb733bd51f99826e8ce423 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-29T15:08:49.199 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:48 smithi027 podman[123580]: 2024-08-29 15:08:48.853274064 +0000 UTC m=+0.016133983 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:08:49.200 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 podman[123580]: 2024-08-29 15:08:49.140868484 +0000 UTC m=+0.303728396 container init 9f70b396270d8c2e9bd2be76517f02c49d421becfefb733bd51f99826e8ce423 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-08-29T15:08:49.200 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 podman[123580]: 2024-08-29 15:08:49.144297016 +0000 UTC m=+0.307156930 container start 9f70b396270d8c2e9bd2be76517f02c49d421becfefb733bd51f99826e8ce423 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-08-29T15:08:49.200 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 podman[123580]: 2024-08-29 15:08:49.198987797 +0000 UTC m=+0.361847724 container attach 9f70b396270d8c2e9bd2be76517f02c49d421becfefb733bd51f99826e8ce423 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.569 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:08:49.570 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-29T15:08:49.570 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-29T15:08:49.570 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-29T15:08:49.570 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-29T15:08:50.003 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-mon[98970]: pgmap v240: 73 pgs: 4 active+undersized, 11 stale+active+clean, 58 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:50.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-mon[93921]: pgmap v240: 73 pgs: 4 active+undersized, 11 stale+active+clean, 58 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:50.004 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-29T15:08:50.004 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-29T15:08:50.004 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-29T15:08:50.004 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-29T15:08:50.005 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-29T15:08:50.005 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-29T15:08:50.005 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-29T15:08:50.005 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-29T15:08:50.005 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate[123595]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-29T15:08:50.005 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 bash[123580]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-29T15:08:50.005 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:49 smithi027 podman[123580]: 2024-08-29 15:08:49.708029673 +0000 UTC m=+0.870889587 container died 9f70b396270d8c2e9bd2be76517f02c49d421becfefb733bd51f99826e8ce423 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-08-29T15:08:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:49 smithi171 ceph-mon[90031]: pgmap v240: 73 pgs: 4 active+undersized, 11 stale+active+clean, 58 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:08:50.753 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 podman[123717]: 2024-08-29 15:08:50.428922876 +0000 UTC m=+0.713935707 container remove 9f70b396270d8c2e9bd2be76517f02c49d421becfefb733bd51f99826e8ce423 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731) 2024-08-29T15:08:50.753 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 podman[123777]: 2024-08-29 15:08:50.543719761 +0000 UTC m=+0.016744895 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:08:50.753 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 podman[123777]: 2024-08-29 15:08:50.654902187 +0000 UTC m=+0.127927312 container create 14455322d13f1ebb461d9cdf90f07876d6a0a4a25bd4b2515e568ceda9cf1a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:08:51.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:50 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:51.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:50 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:51.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:50 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:08:51.068 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 podman[123777]: 2024-08-29 15:08:50.857447839 +0000 UTC m=+0.330472977 container init 14455322d13f1ebb461d9cdf90f07876d6a0a4a25bd4b2515e568ceda9cf1a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:08:51.069 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 podman[123777]: 2024-08-29 15:08:50.860802478 +0000 UTC m=+0.333827604 container start 14455322d13f1ebb461d9cdf90f07876d6a0a4a25bd4b2515e568ceda9cf1a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-08-29T15:08:51.069 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 ceph-osd[123796]: -- 172.21.15.27:0/784063283 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557529313a40 con 0x5575292f1c00 2024-08-29T15:08:51.069 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 bash[123777]: 14455322d13f1ebb461d9cdf90f07876d6a0a4a25bd4b2515e568ceda9cf1a02 2024-08-29T15:08:51.069 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:50 smithi027 systemd[1]: Started Ceph osd.3 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:08:52.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:51 smithi171 ceph-mon[90031]: pgmap v241: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:52.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:51 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[98970]: pgmap v241: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[93921]: pgmap v241: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:51 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:52.765 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:52 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:08:52.739+0000 7f600a060640 -1 Falling back to public interface 2024-08-29T15:08:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:53 smithi171 ceph-mon[90031]: pgmap v242: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:53 smithi027 ceph-mon[93921]: pgmap v242: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:53 smithi027 ceph-mon[98970]: pgmap v242: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:55.659 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[98970]: pgmap v243: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:55.659 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.659 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.659 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.660 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.660 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[93921]: pgmap v243: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:55.660 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.660 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.660 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.660 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:55 smithi171 ceph-mon[90031]: pgmap v243: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:56.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:56 smithi027 ceph-mon[93921]: from='osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-29T15:08:56.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:56 smithi027 ceph-mon[93921]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-29T15:08:56.523 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:56 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:08:56.270+0000 7f600a060640 -1 osd.3 105 log_to_monitors true 2024-08-29T15:08:56.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:56 smithi027 ceph-mon[98970]: from='osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-29T15:08:56.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:56 smithi027 ceph-mon[98970]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-29T15:08:56.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:56 smithi171 ceph-mon[90031]: from='osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-29T15:08:56.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:56 smithi171 ceph-mon[90031]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-29T15:08:57.318 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:08:56 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:08:56.875+0000 7f5ffde02640 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T15:08:57.655 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[98970]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:57.655 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[98970]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[98970]: osdmap e108: 8 total, 7 up, 8 in 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[98970]: from='osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[98970]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[98970]: from='osd.3 ' entity='osd.3' 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[93921]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[93921]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[93921]: osdmap e108: 8 total, 7 up, 8 in 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[93921]: from='osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[93921]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:57.656 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:57 smithi027 ceph-mon[93921]: from='osd.3 ' entity='osd.3' 2024-08-29T15:08:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:57 smithi171 ceph-mon[90031]: pgmap v244: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 243 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:57 smithi171 ceph-mon[90031]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-29T15:08:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:57 smithi171 ceph-mon[90031]: osdmap e108: 8 total, 7 up, 8 in 2024-08-29T15:08:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:57 smithi171 ceph-mon[90031]: from='osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:57.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:57 smithi171 ceph-mon[90031]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:08:57.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:57 smithi171 ceph-mon[90031]: from='osd.3 ' entity='osd.3' 2024-08-29T15:08:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:08:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975] boot 2024-08-29T15:08:58.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: osdmap e109: 8 total, 8 up, 8 in 2024-08-29T15:08:58.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-29T15:08:58.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:58.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:58.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:08:58.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975] boot 2024-08-29T15:08:58.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: osdmap e109: 8 total, 8 up, 8 in 2024-08-29T15:08:58.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-29T15:08:58.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:58.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:58.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:08:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: osd.3 [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975] boot 2024-08-29T15:08:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: osdmap e109: 8 total, 8 up, 8 in 2024-08-29T15:08:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-29T15:08:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:08:59] "GET /metrics HTTP/1.1" 200 33793 "" "Prometheus/2.51.0" 2024-08-29T15:08:59.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: pgmap v247: 73 pgs: 2 peering, 26 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:59.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: Upgrade: Finalizing container_image settings 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:08:59.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: Upgrade: Complete! 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.771 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.772 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:08:59 smithi171 ceph-mon[90031]: osdmap e110: 8 total, 8 up, 8 in 2024-08-29T15:08:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: pgmap v247: 73 pgs: 2 peering, 26 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: Upgrade: Finalizing container_image settings 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:08:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: pgmap v247: 73 pgs: 2 peering, 26 active+undersized, 3 active+undersized+degraded, 42 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: Upgrade: Setting container_image for all nvmeof 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: Upgrade: Finalizing container_image settings 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-29T15:08:59.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: Upgrade: Complete! 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:08:59.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.824 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[98970]: osdmap e110: 8 total, 8 up, 8 in 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: Upgrade: Complete! 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-29T15:08:59.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:08:59.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:08:59 smithi027 ceph-mon[93921]: osdmap e110: 8 total, 8 up, 8 in 2024-08-29T15:09:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:00 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:09:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:00 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:09:00.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:00 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:09:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:01 smithi171 ceph-mon[90031]: pgmap v249: 73 pgs: 7 peering, 11 active+undersized, 2 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:09:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:01 smithi027 ceph-mon[93921]: pgmap v249: 73 pgs: 7 peering, 11 active+undersized, 2 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:09:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:01 smithi027 ceph-mon[98970]: pgmap v249: 73 pgs: 7 peering, 11 active+undersized, 2 active+undersized+degraded, 53 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-08-29T15:09:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:02 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-08-29T15:09:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:02 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:09:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:02 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-08-29T15:09:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:02 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:09:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:02 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-08-29T15:09:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:02 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:09:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:03 smithi171 ceph-mon[90031]: pgmap v250: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:03 smithi027 ceph-mon[93921]: pgmap v250: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:03.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:03 smithi027 ceph-mon[98970]: pgmap v250: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:05.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:05 smithi171 ceph-mon[90031]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:05.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:05 smithi027 ceph-mon[93921]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:05.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:05 smithi027 ceph-mon[98970]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:07.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:07 smithi027 ceph-mon[93921]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:07.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:07 smithi027 ceph-mon[98970]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:07 smithi171 ceph-mon[90031]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:09.286 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:09:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:09:09] "GET /metrics HTTP/1.1" 200 33793 "" "Prometheus/2.51.0" 2024-08-29T15:09:09.312 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:09.312 INFO:teuthology.run_tasks:Running task print... 2024-08-29T15:09:09.323 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-08-29T15:09:09.323 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-29T15:09:09.341 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-29T15:09:09.341 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-29T15:09:09.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph mgr dump --format=json 2024-08-29T15:09:09.558 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:09.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:09 smithi027 ceph-mon[98970]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:09.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:09 smithi027 ceph-mon[98970]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:09:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:09 smithi027 ceph-mon[93921]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:09 smithi027 ceph-mon[93921]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:09:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:09 smithi171 ceph-mon[90031]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:09 smithi171 ceph-mon[90031]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:09:11.375 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:09:11.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:11 smithi027 ceph-mon[93921]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:11.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1039446491' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-29T15:09:11.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:11 smithi027 ceph-mon[98970]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:11.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1039446491' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-29T15:09:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:11 smithi171 ceph-mon[90031]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1039446491' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-29T15:09:12.031 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":42,"flags":0,"active_gid":34104,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":406081300},{"type":"v1","addr":"172.21.15.27:6801","nonce":406081300}]},"active_addr":"172.21.15.27:6801/406081300","active_change":"2024-08-29T15:01:28.196159+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":25411,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":90,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":4018072754}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":1273747260}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":1227943899}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":963937666}]}]} 2024-08-29T15:09:12.040 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-29T15:09:12.040 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-29T15:09:12.041 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T15:09:12.249 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:13.669 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:13 smithi027 ceph-mon[98970]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:13.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:13 smithi027 ceph-mon[93921]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:14.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:13 smithi171 ceph-mon[90031]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:14.058 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:09:14.058 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":110,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T15:08:58.870443+0000","last_up_change":"2024-08-29T15:08:57.868623+0000","last_in_change":"2024-08-29T14:53:28.429586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-29T14:55:31.336677+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-29T14:55:31.828806+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6803","nonce":297906423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6805","nonce":297906423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6809","nonce":297906423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6807","nonce":297906423}]},"public_addr":"172.21.15.27:6803/297906423","cluster_addr":"172.21.15.27:6805/297906423","heartbeat_back_addr":"172.21.15.27:6809/297906423","heartbeat_front_addr":"172.21.15.27:6807/297906423","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6811","nonce":2860215131}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6813","nonce":2860215131}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6817","nonce":2860215131}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6815","nonce":2860215131}]},"public_addr":"172.21.15.27:6811/2860215131","cluster_addr":"172.21.15.27:6813/2860215131","heartbeat_back_addr":"172.21.15.27:6817/2860215131","heartbeat_front_addr":"172.21.15.27:6815/2860215131","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6819","nonce":1131810233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6821","nonce":1131810233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6825","nonce":1131810233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6823","nonce":1131810233}]},"public_addr":"172.21.15.27:6819/1131810233","cluster_addr":"172.21.15.27:6821/1131810233","heartbeat_back_addr":"172.21.15.27:6825/1131810233","heartbeat_front_addr":"172.21.15.27:6823/1131810233","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6827","nonce":3647667975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6829","nonce":3647667975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6833","nonce":3647667975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6831","nonce":3647667975}]},"public_addr":"172.21.15.27:6827/3647667975","cluster_addr":"172.21.15.27:6829/3647667975","heartbeat_back_addr":"172.21.15.27:6833/3647667975","heartbeat_front_addr":"172.21.15.27:6831/3647667975","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6809","nonce":223130519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6811","nonce":223130519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6815","nonce":223130519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6813","nonce":223130519}]},"public_addr":"172.21.15.171:6809/223130519","cluster_addr":"172.21.15.171:6811/223130519","heartbeat_back_addr":"172.21.15.171:6815/223130519","heartbeat_front_addr":"172.21.15.171:6813/223130519","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/921286138":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2657305076":"2024-08-30T14:58:14.775345+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/1523003109":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.171:0/2152868901":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2828185086":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6801/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/258529454":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6800/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.171:0/2912002540":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2955701830":"2024-08-30T14:58:14.775345+0000","172.21.15.171:6834/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.171:6835/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.171:0/3744606577":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/92805749":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.171:0/1480641142":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.171:0/3978338528":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/1565341554":"2024-08-30T15:01:28.195875+0000","172.21.15.171:0/1064847764":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T15:09:14.698 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-29T15:09:14.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T15:09:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:09:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:09:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3193851857' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:09:14.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:09:14.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:09:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3193851857' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:09:14.907 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:09:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:09:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3193851857' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:09:15.974 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:15 smithi027 ceph-mon[98970]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:15.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:15 smithi027 ceph-mon[93921]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:15 smithi171 ceph-mon[90031]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:16.676 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:09:16.676 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":110,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T15:08:58.870443+0000","last_up_change":"2024-08-29T15:08:57.868623+0000","last_in_change":"2024-08-29T14:53:28.429586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-29T14:55:31.336677+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-29T14:55:31.828806+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6803","nonce":297906423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6805","nonce":297906423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6809","nonce":297906423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6807","nonce":297906423}]},"public_addr":"172.21.15.27:6803/297906423","cluster_addr":"172.21.15.27:6805/297906423","heartbeat_back_addr":"172.21.15.27:6809/297906423","heartbeat_front_addr":"172.21.15.27:6807/297906423","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6811","nonce":2860215131}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6813","nonce":2860215131}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6817","nonce":2860215131}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6815","nonce":2860215131}]},"public_addr":"172.21.15.27:6811/2860215131","cluster_addr":"172.21.15.27:6813/2860215131","heartbeat_back_addr":"172.21.15.27:6817/2860215131","heartbeat_front_addr":"172.21.15.27:6815/2860215131","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6819","nonce":1131810233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6821","nonce":1131810233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6825","nonce":1131810233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1131810233},{"type":"v1","addr":"172.21.15.27:6823","nonce":1131810233}]},"public_addr":"172.21.15.27:6819/1131810233","cluster_addr":"172.21.15.27:6821/1131810233","heartbeat_back_addr":"172.21.15.27:6825/1131810233","heartbeat_front_addr":"172.21.15.27:6823/1131810233","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6827","nonce":3647667975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6829","nonce":3647667975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6833","nonce":3647667975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6831","nonce":3647667975}]},"public_addr":"172.21.15.27:6827/3647667975","cluster_addr":"172.21.15.27:6829/3647667975","heartbeat_back_addr":"172.21.15.27:6833/3647667975","heartbeat_front_addr":"172.21.15.27:6831/3647667975","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6809","nonce":223130519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6811","nonce":223130519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6815","nonce":223130519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":223130519},{"type":"v1","addr":"172.21.15.171:6813","nonce":223130519}]},"public_addr":"172.21.15.171:6809/223130519","cluster_addr":"172.21.15.171:6811/223130519","heartbeat_back_addr":"172.21.15.171:6815/223130519","heartbeat_front_addr":"172.21.15.171:6813/223130519","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/921286138":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2657305076":"2024-08-30T14:58:14.775345+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/1523003109":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.171:0/2152868901":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2828185086":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:6801/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/258529454":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6800/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.171:0/2912002540":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2955701830":"2024-08-30T14:58:14.775345+0000","172.21.15.171:6834/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.171:6835/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.171:0/3744606577":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/92805749":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6800/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.171:0/1480641142":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.171:0/3978338528":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/1565341554":"2024-08-30T15:01:28.195875+0000","172.21.15.171:0/1064847764":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T15:09:17.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:16 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/496302951' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:09:17.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:16 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/496302951' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:09:17.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:16 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/496302951' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:09:17.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-29T15:09:17.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-29T15:09:17.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-29T15:09:17.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-29T15:09:17.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-29T15:09:17.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-29T15:09:17.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-29T15:09:17.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-29T15:09:17.839 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:17.848 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:17.868 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:17.877 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:17 smithi171 ceph-mon[90031]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:17 smithi027 ceph-mon[93921]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:17 smithi027 ceph-mon[98970]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:18.209 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:18.212 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:18.212 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:18.848 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:19.276 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:09:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:09:19] "GET /metrics HTTP/1.1" 200 33935 "" "Prometheus/2.51.0" 2024-08-29T15:09:19.988 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:19 smithi027 ceph-mon[98970]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:19.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:19 smithi027 ceph-mon[93921]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:19 smithi171 ceph-mon[90031]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:21 smithi171 ceph-mon[90031]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:21 smithi027 ceph-mon[93921]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:21 smithi027 ceph-mon[98970]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:23 smithi171 ceph-mon[90031]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:24.028 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:23 smithi027 ceph-mon[93921]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:24.028 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:23 smithi027 ceph-mon[98970]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:25 smithi171 ceph-mon[90031]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:26.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:25 smithi027 ceph-mon[98970]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:25 smithi027 ceph-mon[93921]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:27.519 INFO:teuthology.orchestra.run.smithi027.stdout:163208757449 2024-08-29T15:09:27.519 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-29T15:09:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:27 smithi027 ceph-mon[93921]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:28.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:27 smithi027 ceph-mon[98970]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:28.182 INFO:teuthology.orchestra.run.smithi027.stdout:468151435271 2024-08-29T15:09:28.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-29T15:09:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:27 smithi171 ceph-mon[90031]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:29.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:29.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:09:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:09:29] "GET /metrics HTTP/1.1" 200 33934 "" "Prometheus/2.51.0" 2024-08-29T15:09:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:29 smithi027 ceph-mon[93921]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:29 smithi027 ceph-mon[98970]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:29 smithi171 ceph-mon[90031]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:31.017 INFO:teuthology.orchestra.run.smithi027.stdout:446676598798 2024-08-29T15:09:31.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-29T15:09:31.420 INFO:teuthology.orchestra.run.smithi027.stdout:403726925852 2024-08-29T15:09:31.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-29T15:09:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:31 smithi027 ceph-mon[93921]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:31 smithi027 ceph-mon[98970]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:32.142 INFO:teuthology.orchestra.run.smithi027.stdout:128849019087 2024-08-29T15:09:32.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-29T15:09:32.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:31 smithi171 ceph-mon[90031]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:34.033 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:33 smithi027 ceph-mon[98970]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:34.034 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:33 smithi027 ceph-mon[93921]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:34.056 INFO:teuthology.orchestra.run.smithi027.stdout:231928234178 2024-08-29T15:09:34.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-29T15:09:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:33 smithi171 ceph-mon[90031]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:34.518 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:34.519 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:34.544 INFO:teuthology.orchestra.run.smithi027.stdout:425201762326 2024-08-29T15:09:34.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-29T15:09:35.032 INFO:teuthology.orchestra.run.smithi027.stdout:197568495814 2024-08-29T15:09:35.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-29T15:09:35.148 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:35.149 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:35.255 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:35.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:35 smithi027 ceph-mon[93921]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:35.872 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:35 smithi027 ceph-mon[98970]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:35 smithi171 ceph-mon[90031]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:38.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:37 smithi027 ceph-mon[98970]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:38.062 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:37 smithi027 ceph-mon[93921]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:37 smithi171 ceph-mon[90031]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:09:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:09:39] "GET /metrics HTTP/1.1" 200 33934 "" "Prometheus/2.51.0" 2024-08-29T15:09:39.958 INFO:teuthology.orchestra.run.smithi027.stdout:403726925853 2024-08-29T15:09:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:39 smithi027 ceph-mon[98970]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:39 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/313235056' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-29T15:09:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:39 smithi027 ceph-mon[93921]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:40.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:39 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/313235056' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-29T15:09:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:39 smithi171 ceph-mon[90031]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:39 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/313235056' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-29T15:09:40.513 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:40.523 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925852 got 403726925853 for osd.0 2024-08-29T15:09:40.523 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:41.706 INFO:teuthology.orchestra.run.smithi027.stdout:468151435274 2024-08-29T15:09:41.888 INFO:teuthology.orchestra.run.smithi027.stdout:446676598801 2024-08-29T15:09:41.894 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:41.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:41 smithi027 ceph-mon[98970]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:41.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3807533407' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-29T15:09:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:41 smithi027 ceph-mon[93921]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:41.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3807533407' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-29T15:09:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:41 smithi171 ceph-mon[90031]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3807533407' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-29T15:09:42.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3783002083' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-29T15:09:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3783002083' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-29T15:09:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3783002083' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-29T15:09:44.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:43 smithi027 ceph-mon[98970]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:44.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:44.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:43 smithi027 ceph-mon[93921]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:44.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:44.054 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435271 got 468151435274 for osd.3 2024-08-29T15:09:44.055 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:43 smithi171 ceph-mon[90031]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:44.719 INFO:teuthology.orchestra.run.smithi027.stdout:163208757453 2024-08-29T15:09:44.911 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2098843120' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-29T15:09:44.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2098843120' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-29T15:09:45.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2098843120' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-29T15:09:45.515 INFO:teuthology.orchestra.run.smithi027.stdout:128849019091 2024-08-29T15:09:45.908 INFO:teuthology.orchestra.run.smithi027.stdout:197568495817 2024-08-29T15:09:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:45 smithi027 ceph-mon[93921]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1085021391' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-29T15:09:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:45 smithi027 ceph-mon[98970]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1085021391' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-29T15:09:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:45 smithi171 ceph-mon[90031]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1085021391' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-29T15:09:46.519 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019087 got 128849019091 for osd.4 2024-08-29T15:09:46.520 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1464911484' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-29T15:09:47.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1464911484' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-29T15:09:47.241 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:47.255 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598798 got 446676598801 for osd.2 2024-08-29T15:09:47.255 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1464911484' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-29T15:09:47.840 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757449 got 163208757453 for osd.5 2024-08-29T15:09:47.840 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:47 smithi027 ceph-mon[93921]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:47 smithi027 ceph-mon[98970]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:48.221 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495814 got 197568495817 for osd.6 2024-08-29T15:09:48.221 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:47 smithi171 ceph-mon[90031]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:09:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:09:49] "GET /metrics HTTP/1.1" 200 33935 "" "Prometheus/2.51.0" 2024-08-29T15:09:49.663 INFO:teuthology.orchestra.run.smithi027.stdout:231928234181 2024-08-29T15:09:50.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:49 smithi027 ceph-mon[93921]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:50.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1887016148' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-29T15:09:50.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:49 smithi027 ceph-mon[98970]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:50.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:49 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1887016148' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-29T15:09:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:49 smithi171 ceph-mon[90031]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1887016148' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-29T15:09:51.076 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234178 got 231928234181 for osd.7 2024-08-29T15:09:51.076 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:51.179 INFO:teuthology.orchestra.run.smithi027.stdout:425201762330 2024-08-29T15:09:51.750 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762326 got 425201762330 for osd.1 2024-08-29T15:09:51.750 DEBUG:teuthology.parallel:result is None 2024-08-29T15:09:51.750 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-29T15:09:51.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:09:51.962 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:51 smithi027 ceph-mon[98970]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3396885058' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-29T15:09:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:51 smithi027 ceph-mon[93921]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3396885058' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-29T15:09:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:51 smithi171 ceph-mon[90031]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3396885058' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-29T15:09:53.750 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:09:53.769 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:09:54.253 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:53 smithi027 ceph-mon[93921]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:54.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:53 smithi027 ceph-mon[98970]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:53 smithi171 ceph-mon[90031]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:54.506 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":275,"stamp":"2024-08-29T15:09:52.329971+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":249952,"kb_used_data":23576,"kb_used_omap":34,"kb_used_meta":226205,"kb_avail":749711264,"statfs":{"total":767960285184,"available":767704334336,"internally_reserved":0,"allocated":24141824,"data_stored":19390055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34969,"internal_metadata":231634791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003063"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882496+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:08:58.882496+0000","last_peered":"2024-08-29T15:08:58.882496+0000","last_clean":"2024-08-29T15:08:58.882496+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:08:58.882496+0000","last_undegraded":"2024-08-29T15:08:58.882496+0000","last_fullsized":"2024-08-29T15:08:58.882496+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877466+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:08:58.877466+0000","last_peered":"2024-08-29T15:08:58.877466+0000","last_clean":"2024-08-29T15:08:58.877466+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:08:58.877466+0000","last_undegraded":"2024-08-29T15:08:58.877466+0000","last_fullsized":"2024-08-29T15:08:58.877466+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887518+0000","last_change":"2024-08-29T15:08:58.887518+0000","last_active":"2024-08-29T15:08:58.887518+0000","last_peered":"2024-08-29T15:08:58.887518+0000","last_clean":"2024-08-29T15:08:58.887518+0000","last_became_active":"2024-08-29T15:08:58.887049+0000","last_became_peered":"2024-08-29T15:08:58.887049+0000","last_unstale":"2024-08-29T15:08:58.887518+0000","last_undegraded":"2024-08-29T15:08:58.887518+0000","last_fullsized":"2024-08-29T15:08:58.887518+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:23:07.638407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890091+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:08:58.890091+0000","last_peered":"2024-08-29T15:08:58.890091+0000","last_clean":"2024-08-29T15:08:58.890091+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:08:58.890091+0000","last_undegraded":"2024-08-29T15:08:58.890091+0000","last_fullsized":"2024-08-29T15:08:58.890091+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889545+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:08:58.889545+0000","last_peered":"2024-08-29T15:08:58.889545+0000","last_clean":"2024-08-29T15:08:58.889545+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:08:58.889545+0000","last_undegraded":"2024-08-29T15:08:58.889545+0000","last_fullsized":"2024-08-29T15:08:58.889545+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877550+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T15:08:58.877550+0000","last_peered":"2024-08-29T15:08:58.877550+0000","last_clean":"2024-08-29T15:08:58.877550+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T15:08:58.877550+0000","last_undegraded":"2024-08-29T15:08:58.877550+0000","last_fullsized":"2024-08-29T15:08:58.877550+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884766+0000","last_change":"2024-08-29T15:08:58.884675+0000","last_active":"2024-08-29T15:08:58.884766+0000","last_peered":"2024-08-29T15:08:58.884766+0000","last_clean":"2024-08-29T15:08:58.884766+0000","last_became_active":"2024-08-29T15:08:58.884007+0000","last_became_peered":"2024-08-29T15:08:58.884007+0000","last_unstale":"2024-08-29T15:08:58.884766+0000","last_undegraded":"2024-08-29T15:08:58.884766+0000","last_fullsized":"2024-08-29T15:08:58.884766+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:12:03.629717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885010+0000","last_change":"2024-08-29T15:07:20.812313+0000","last_active":"2024-08-29T15:08:58.885010+0000","last_peered":"2024-08-29T15:08:58.885010+0000","last_clean":"2024-08-29T15:08:58.885010+0000","last_became_active":"2024-08-29T15:07:20.811973+0000","last_became_peered":"2024-08-29T15:07:20.811973+0000","last_unstale":"2024-08-29T15:08:58.885010+0000","last_undegraded":"2024-08-29T15:08:58.885010+0000","last_fullsized":"2024-08-29T15:08:58.885010+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:37.283470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877737+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:08:58.877737+0000","last_peered":"2024-08-29T15:08:58.877737+0000","last_clean":"2024-08-29T15:08:58.877737+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:08:58.877737+0000","last_undegraded":"2024-08-29T15:08:58.877737+0000","last_fullsized":"2024-08-29T15:08:58.877737+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872144+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T15:08:57.872144+0000","last_peered":"2024-08-29T15:08:57.872144+0000","last_clean":"2024-08-29T15:08:57.872144+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T15:08:57.872144+0000","last_undegraded":"2024-08-29T15:08:57.872144+0000","last_fullsized":"2024-08-29T15:08:57.872144+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877594+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:08:58.877594+0000","last_peered":"2024-08-29T15:08:58.877594+0000","last_clean":"2024-08-29T15:08:58.877594+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:08:58.877594+0000","last_undegraded":"2024-08-29T15:08:58.877594+0000","last_fullsized":"2024-08-29T15:08:58.877594+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885413+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:08:58.885413+0000","last_peered":"2024-08-29T15:08:58.885413+0000","last_clean":"2024-08-29T15:08:58.885413+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:08:58.885413+0000","last_undegraded":"2024-08-29T15:08:58.885413+0000","last_fullsized":"2024-08-29T15:08:58.885413+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877537+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T15:08:58.877537+0000","last_peered":"2024-08-29T15:08:58.877537+0000","last_clean":"2024-08-29T15:08:58.877537+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T15:08:58.877537+0000","last_undegraded":"2024-08-29T15:08:58.877537+0000","last_fullsized":"2024-08-29T15:08:58.877537+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876085+0000","last_change":"2024-08-29T15:07:52.136104+0000","last_active":"2024-08-29T15:08:57.876085+0000","last_peered":"2024-08-29T15:08:57.876085+0000","last_clean":"2024-08-29T15:08:57.876085+0000","last_became_active":"2024-08-29T15:07:52.135781+0000","last_became_peered":"2024-08-29T15:07:52.135781+0000","last_unstale":"2024-08-29T15:08:57.876085+0000","last_undegraded":"2024-08-29T15:08:57.876085+0000","last_fullsized":"2024-08-29T15:08:57.876085+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:26.381499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888097+0000","last_change":"2024-08-29T15:08:58.887669+0000","last_active":"2024-08-29T15:08:58.888097+0000","last_peered":"2024-08-29T15:08:58.888097+0000","last_clean":"2024-08-29T15:08:58.888097+0000","last_became_active":"2024-08-29T15:08:58.887343+0000","last_became_peered":"2024-08-29T15:08:58.887343+0000","last_unstale":"2024-08-29T15:08:58.888097+0000","last_undegraded":"2024-08-29T15:08:58.888097+0000","last_fullsized":"2024-08-29T15:08:58.888097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:32.280756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878358+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:08:58.878358+0000","last_peered":"2024-08-29T15:08:58.878358+0000","last_clean":"2024-08-29T15:08:58.878358+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:08:58.878358+0000","last_undegraded":"2024-08-29T15:08:58.878358+0000","last_fullsized":"2024-08-29T15:08:58.878358+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889564+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:08:58.889564+0000","last_peered":"2024-08-29T15:08:58.889564+0000","last_clean":"2024-08-29T15:08:58.889564+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:08:58.889564+0000","last_undegraded":"2024-08-29T15:08:58.889564+0000","last_fullsized":"2024-08-29T15:08:58.889564+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886744+0000","last_change":"2024-08-29T15:08:58.886369+0000","last_active":"2024-08-29T15:08:58.886744+0000","last_peered":"2024-08-29T15:08:58.886744+0000","last_clean":"2024-08-29T15:08:58.886744+0000","last_became_active":"2024-08-29T15:08:58.886022+0000","last_became_peered":"2024-08-29T15:08:58.886022+0000","last_unstale":"2024-08-29T15:08:58.886744+0000","last_undegraded":"2024-08-29T15:08:58.886744+0000","last_fullsized":"2024-08-29T15:08:58.886744+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:36.451849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876498+0000","last_change":"2024-08-29T15:07:52.134960+0000","last_active":"2024-08-29T15:08:57.876498+0000","last_peered":"2024-08-29T15:08:57.876498+0000","last_clean":"2024-08-29T15:08:57.876498+0000","last_became_active":"2024-08-29T15:07:52.134608+0000","last_became_peered":"2024-08-29T15:07:52.134608+0000","last_unstale":"2024-08-29T15:08:57.876498+0000","last_undegraded":"2024-08-29T15:08:57.876498+0000","last_fullsized":"2024-08-29T15:08:57.876498+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:42.154504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875880+0000","last_change":"2024-08-29T15:08:25.191304+0000","last_active":"2024-08-29T15:08:58.875880+0000","last_peered":"2024-08-29T15:08:58.875880+0000","last_clean":"2024-08-29T15:08:58.875880+0000","last_became_active":"2024-08-29T15:08:25.191022+0000","last_became_peered":"2024-08-29T15:08:25.191022+0000","last_unstale":"2024-08-29T15:08:58.875880+0000","last_undegraded":"2024-08-29T15:08:58.875880+0000","last_fullsized":"2024-08-29T15:08:58.875880+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:07.608759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.874763+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:08:57.874763+0000","last_peered":"2024-08-29T15:08:57.874763+0000","last_clean":"2024-08-29T15:08:57.874763+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:08:57.874763+0000","last_undegraded":"2024-08-29T15:08:57.874763+0000","last_fullsized":"2024-08-29T15:08:57.874763+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875459+0000","last_change":"2024-08-29T15:07:52.142297+0000","last_active":"2024-08-29T15:08:58.875459+0000","last_peered":"2024-08-29T15:08:58.875459+0000","last_clean":"2024-08-29T15:08:58.875459+0000","last_became_active":"2024-08-29T15:07:52.141836+0000","last_became_peered":"2024-08-29T15:07:52.141836+0000","last_unstale":"2024-08-29T15:08:58.875459+0000","last_undegraded":"2024-08-29T15:08:58.875459+0000","last_fullsized":"2024-08-29T15:08:58.875459+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:58:01.221672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881991+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:08:58.881991+0000","last_peered":"2024-08-29T15:08:58.881991+0000","last_clean":"2024-08-29T15:08:58.881991+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:08:58.881991+0000","last_undegraded":"2024-08-29T15:08:58.881991+0000","last_fullsized":"2024-08-29T15:08:58.881991+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882231+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:08:58.882231+0000","last_peered":"2024-08-29T15:08:58.882231+0000","last_clean":"2024-08-29T15:08:58.882231+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:08:58.882231+0000","last_undegraded":"2024-08-29T15:08:58.882231+0000","last_fullsized":"2024-08-29T15:08:58.882231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":403,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.879974+0000","last_change":"2024-08-29T15:07:20.817595+0000","last_active":"2024-08-29T15:08:57.879974+0000","last_peered":"2024-08-29T15:08:57.879974+0000","last_clean":"2024-08-29T15:08:57.879974+0000","last_became_active":"2024-08-29T15:07:20.817130+0000","last_became_peered":"2024-08-29T15:07:20.817130+0000","last_unstale":"2024-08-29T15:08:57.879974+0000","last_undegraded":"2024-08-29T15:08:57.879974+0000","last_fullsized":"2024-08-29T15:08:57.879974+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:28:33.151864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":595,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874858+0000","last_change":"2024-08-29T15:08:25.185728+0000","last_active":"2024-08-29T15:08:58.874858+0000","last_peered":"2024-08-29T15:08:58.874858+0000","last_clean":"2024-08-29T15:08:58.874858+0000","last_became_active":"2024-08-29T15:08:25.185219+0000","last_became_peered":"2024-08-29T15:08:25.185219+0000","last_unstale":"2024-08-29T15:08:58.874858+0000","last_undegraded":"2024-08-29T15:08:58.874858+0000","last_fullsized":"2024-08-29T15:08:58.874858+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:16:25.013137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890162+0000","last_change":"2024-08-29T15:08:58.890060+0000","last_active":"2024-08-29T15:08:58.890162+0000","last_peered":"2024-08-29T15:08:58.890162+0000","last_clean":"2024-08-29T15:08:58.890162+0000","last_became_active":"2024-08-29T15:08:58.889714+0000","last_became_peered":"2024-08-29T15:08:58.889714+0000","last_unstale":"2024-08-29T15:08:58.890162+0000","last_undegraded":"2024-08-29T15:08:58.890162+0000","last_fullsized":"2024-08-29T15:08:58.890162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:50.659662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876262+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T15:08:57.876262+0000","last_peered":"2024-08-29T15:08:57.876262+0000","last_clean":"2024-08-29T15:08:57.876262+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T15:08:57.876262+0000","last_undegraded":"2024-08-29T15:08:57.876262+0000","last_fullsized":"2024-08-29T15:08:57.876262+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":800,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.879076+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:08:57.879076+0000","last_peered":"2024-08-29T15:08:57.879076+0000","last_clean":"2024-08-29T15:08:57.879076+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:08:57.879076+0000","last_undegraded":"2024-08-29T15:08:57.879076+0000","last_fullsized":"2024-08-29T15:08:57.879076+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889019+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:08:58.889019+0000","last_peered":"2024-08-29T15:08:58.889019+0000","last_clean":"2024-08-29T15:08:58.889019+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:08:58.889019+0000","last_undegraded":"2024-08-29T15:08:58.889019+0000","last_fullsized":"2024-08-29T15:08:58.889019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.879442+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:08:58.879442+0000","last_peered":"2024-08-29T15:08:58.879442+0000","last_clean":"2024-08-29T15:08:58.879442+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:08:58.879442+0000","last_undegraded":"2024-08-29T15:08:58.879442+0000","last_fullsized":"2024-08-29T15:08:58.879442+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":540,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881830+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:08:58.881830+0000","last_peered":"2024-08-29T15:08:58.881830+0000","last_clean":"2024-08-29T15:08:58.881830+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:08:58.881830+0000","last_undegraded":"2024-08-29T15:08:58.881830+0000","last_fullsized":"2024-08-29T15:08:58.881830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873790+0000","last_change":"2024-08-29T15:08:25.188186+0000","last_active":"2024-08-29T15:08:58.873790+0000","last_peered":"2024-08-29T15:08:58.873790+0000","last_clean":"2024-08-29T15:08:58.873790+0000","last_became_active":"2024-08-29T15:08:25.187830+0000","last_became_peered":"2024-08-29T15:08:25.187830+0000","last_unstale":"2024-08-29T15:08:58.873790+0000","last_undegraded":"2024-08-29T15:08:58.873790+0000","last_fullsized":"2024-08-29T15:08:58.873790+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:00:12.780470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873782+0000","last_change":"2024-08-29T15:07:52.139273+0000","last_active":"2024-08-29T15:08:58.873782+0000","last_peered":"2024-08-29T15:08:58.873782+0000","last_clean":"2024-08-29T15:08:58.873782+0000","last_became_active":"2024-08-29T15:07:52.138052+0000","last_became_peered":"2024-08-29T15:07:52.138052+0000","last_unstale":"2024-08-29T15:08:58.873782+0000","last_undegraded":"2024-08-29T15:08:58.873782+0000","last_fullsized":"2024-08-29T15:08:58.873782+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:01.705167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":526,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872946+0000","last_change":"2024-08-29T15:07:52.138323+0000","last_active":"2024-08-29T15:08:57.872946+0000","last_peered":"2024-08-29T15:08:57.872946+0000","last_clean":"2024-08-29T15:08:57.872946+0000","last_became_active":"2024-08-29T15:07:52.137781+0000","last_became_peered":"2024-08-29T15:07:52.137781+0000","last_unstale":"2024-08-29T15:08:57.872946+0000","last_undegraded":"2024-08-29T15:08:57.872946+0000","last_fullsized":"2024-08-29T15:08:57.872946+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:11:27.075260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":579,"num_read_kb":2582,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875043+0000","last_change":"2024-08-29T15:07:20.816485+0000","last_active":"2024-08-29T15:08:57.875043+0000","last_peered":"2024-08-29T15:08:57.875043+0000","last_clean":"2024-08-29T15:08:57.875043+0000","last_became_active":"2024-08-29T15:07:20.816190+0000","last_became_peered":"2024-08-29T15:07:20.816190+0000","last_unstale":"2024-08-29T15:08:57.875043+0000","last_undegraded":"2024-08-29T15:08:57.875043+0000","last_fullsized":"2024-08-29T15:08:57.875043+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:06.731598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.883782+0000","last_change":"2024-08-29T15:08:58.883318+0000","last_active":"2024-08-29T15:08:58.883782+0000","last_peered":"2024-08-29T15:08:58.883782+0000","last_clean":"2024-08-29T15:08:58.883782+0000","last_became_active":"2024-08-29T15:08:58.882978+0000","last_became_peered":"2024-08-29T15:08:58.882978+0000","last_unstale":"2024-08-29T15:08:58.883782+0000","last_undegraded":"2024-08-29T15:08:58.883782+0000","last_fullsized":"2024-08-29T15:08:58.883782+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:08.675766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887791+0000","last_change":"2024-08-29T15:08:58.887791+0000","last_active":"2024-08-29T15:08:58.887791+0000","last_peered":"2024-08-29T15:08:58.887791+0000","last_clean":"2024-08-29T15:08:58.887791+0000","last_became_active":"2024-08-29T15:08:58.887472+0000","last_became_peered":"2024-08-29T15:08:58.887472+0000","last_unstale":"2024-08-29T15:08:58.887791+0000","last_undegraded":"2024-08-29T15:08:58.887791+0000","last_fullsized":"2024-08-29T15:08:58.887791+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:35:58.690131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":506,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874296+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:08:58.874296+0000","last_peered":"2024-08-29T15:08:58.874296+0000","last_clean":"2024-08-29T15:08:58.874296+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:08:58.874296+0000","last_undegraded":"2024-08-29T15:08:58.874296+0000","last_fullsized":"2024-08-29T15:08:58.874296+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873739+0000","last_change":"2024-08-29T15:07:52.139543+0000","last_active":"2024-08-29T15:08:58.873739+0000","last_peered":"2024-08-29T15:08:58.873739+0000","last_clean":"2024-08-29T15:08:58.873739+0000","last_became_active":"2024-08-29T15:07:52.138499+0000","last_became_peered":"2024-08-29T15:07:52.138499+0000","last_unstale":"2024-08-29T15:08:58.873739+0000","last_undegraded":"2024-08-29T15:08:58.873739+0000","last_fullsized":"2024-08-29T15:08:58.873739+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:41:05.588954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874015+0000","last_change":"2024-08-29T15:08:25.186307+0000","last_active":"2024-08-29T15:08:58.874015+0000","last_peered":"2024-08-29T15:08:58.874015+0000","last_clean":"2024-08-29T15:08:58.874015+0000","last_became_active":"2024-08-29T15:08:25.185788+0000","last_became_peered":"2024-08-29T15:08:25.185788+0000","last_unstale":"2024-08-29T15:08:58.874015+0000","last_undegraded":"2024-08-29T15:08:58.874015+0000","last_fullsized":"2024-08-29T15:08:58.874015+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:58:08.511525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":497,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877648+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:08:58.877648+0000","last_peered":"2024-08-29T15:08:58.877648+0000","last_clean":"2024-08-29T15:08:58.877648+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:08:58.877648+0000","last_undegraded":"2024-08-29T15:08:58.877648+0000","last_fullsized":"2024-08-29T15:08:58.877648+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878271+0000","last_change":"2024-08-29T15:07:20.813592+0000","last_active":"2024-08-29T15:08:58.878271+0000","last_peered":"2024-08-29T15:08:58.878271+0000","last_clean":"2024-08-29T15:08:58.878271+0000","last_became_active":"2024-08-29T15:07:20.812970+0000","last_became_peered":"2024-08-29T15:07:20.812970+0000","last_unstale":"2024-08-29T15:08:58.878271+0000","last_undegraded":"2024-08-29T15:08:58.878271+0000","last_fullsized":"2024-08-29T15:08:58.878271+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:54.850894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888546+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:08:58.888546+0000","last_peered":"2024-08-29T15:08:58.888546+0000","last_clean":"2024-08-29T15:08:58.888546+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:08:58.888546+0000","last_undegraded":"2024-08-29T15:08:58.888546+0000","last_fullsized":"2024-08-29T15:08:58.888546+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":597,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874088+0000","last_change":"2024-08-29T15:07:52.139196+0000","last_active":"2024-08-29T15:08:58.874088+0000","last_peered":"2024-08-29T15:08:58.874088+0000","last_clean":"2024-08-29T15:08:58.874088+0000","last_became_active":"2024-08-29T15:07:52.138669+0000","last_became_peered":"2024-08-29T15:07:52.138669+0000","last_unstale":"2024-08-29T15:08:58.874088+0000","last_undegraded":"2024-08-29T15:08:58.874088+0000","last_fullsized":"2024-08-29T15:08:58.874088+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:43:54.949715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888619+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:08:58.888619+0000","last_peered":"2024-08-29T15:08:58.888619+0000","last_clean":"2024-08-29T15:08:58.888619+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:08:58.888619+0000","last_undegraded":"2024-08-29T15:08:58.888619+0000","last_fullsized":"2024-08-29T15:08:58.888619+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878638+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:08:58.878638+0000","last_peered":"2024-08-29T15:08:58.878638+0000","last_clean":"2024-08-29T15:08:58.878638+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:08:58.878638+0000","last_undegraded":"2024-08-29T15:08:58.878638+0000","last_fullsized":"2024-08-29T15:08:58.878638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":337,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885216+0000","last_change":"2024-08-29T15:08:25.186819+0000","last_active":"2024-08-29T15:08:58.885216+0000","last_peered":"2024-08-29T15:08:58.885216+0000","last_clean":"2024-08-29T15:08:58.885216+0000","last_became_active":"2024-08-29T15:08:25.186401+0000","last_became_peered":"2024-08-29T15:08:25.186401+0000","last_unstale":"2024-08-29T15:08:58.885216+0000","last_undegraded":"2024-08-29T15:08:58.885216+0000","last_fullsized":"2024-08-29T15:08:58.885216+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:45.101824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.873204+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T15:08:57.873204+0000","last_peered":"2024-08-29T15:08:57.873204+0000","last_clean":"2024-08-29T15:08:57.873204+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T15:08:57.873204+0000","last_undegraded":"2024-08-29T15:08:57.873204+0000","last_fullsized":"2024-08-29T15:08:57.873204+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875638+0000","last_change":"2024-08-29T15:08:25.185452+0000","last_active":"2024-08-29T15:08:58.875638+0000","last_peered":"2024-08-29T15:08:58.875638+0000","last_clean":"2024-08-29T15:08:58.875638+0000","last_became_active":"2024-08-29T15:08:25.185095+0000","last_became_peered":"2024-08-29T15:08:25.185095+0000","last_unstale":"2024-08-29T15:08:58.875638+0000","last_undegraded":"2024-08-29T15:08:58.875638+0000","last_fullsized":"2024-08-29T15:08:58.875638+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:19:03.238217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875238+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T15:08:58.875238+0000","last_peered":"2024-08-29T15:08:58.875238+0000","last_clean":"2024-08-29T15:08:58.875238+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T15:08:58.875238+0000","last_undegraded":"2024-08-29T15:08:58.875238+0000","last_fullsized":"2024-08-29T15:08:58.875238+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873562+0000","last_change":"2024-08-29T15:08:25.186368+0000","last_active":"2024-08-29T15:08:58.873562+0000","last_peered":"2024-08-29T15:08:58.873562+0000","last_clean":"2024-08-29T15:08:58.873562+0000","last_became_active":"2024-08-29T15:08:25.185912+0000","last_became_peered":"2024-08-29T15:08:25.185912+0000","last_unstale":"2024-08-29T15:08:58.873562+0000","last_undegraded":"2024-08-29T15:08:58.873562+0000","last_fullsized":"2024-08-29T15:08:58.873562+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:23.124995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886502+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:08:58.886502+0000","last_peered":"2024-08-29T15:08:58.886502+0000","last_clean":"2024-08-29T15:08:58.886502+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:08:58.886502+0000","last_undegraded":"2024-08-29T15:08:58.886502+0000","last_fullsized":"2024-08-29T15:08:58.886502+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872257+0000","last_change":"2024-08-29T15:07:52.136541+0000","last_active":"2024-08-29T15:08:57.872257+0000","last_peered":"2024-08-29T15:08:57.872257+0000","last_clean":"2024-08-29T15:08:57.872257+0000","last_became_active":"2024-08-29T15:07:52.136202+0000","last_became_peered":"2024-08-29T15:07:52.136202+0000","last_unstale":"2024-08-29T15:08:57.872257+0000","last_undegraded":"2024-08-29T15:08:57.872257+0000","last_fullsized":"2024-08-29T15:08:57.872257+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:01.747963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888935+0000","last_change":"2024-08-29T15:08:58.888848+0000","last_active":"2024-08-29T15:08:58.888935+0000","last_peered":"2024-08-29T15:08:58.888935+0000","last_clean":"2024-08-29T15:08:58.888935+0000","last_became_active":"2024-08-29T15:08:58.886702+0000","last_became_peered":"2024-08-29T15:08:58.886702+0000","last_unstale":"2024-08-29T15:08:58.888935+0000","last_undegraded":"2024-08-29T15:08:58.888935+0000","last_fullsized":"2024-08-29T15:08:58.888935+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:56.189439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887636+0000","last_change":"2024-08-29T15:08:58.887151+0000","last_active":"2024-08-29T15:08:58.887636+0000","last_peered":"2024-08-29T15:08:58.887636+0000","last_clean":"2024-08-29T15:08:58.887636+0000","last_became_active":"2024-08-29T15:08:58.886918+0000","last_became_peered":"2024-08-29T15:08:58.886918+0000","last_unstale":"2024-08-29T15:08:58.887636+0000","last_undegraded":"2024-08-29T15:08:58.887636+0000","last_fullsized":"2024-08-29T15:08:58.887636+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:37.557892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889787+0000","last_change":"2024-08-29T15:08:58.889700+0000","last_active":"2024-08-29T15:08:58.889787+0000","last_peered":"2024-08-29T15:08:58.889787+0000","last_clean":"2024-08-29T15:08:58.889787+0000","last_became_active":"2024-08-29T15:08:58.887521+0000","last_became_peered":"2024-08-29T15:08:58.887521+0000","last_unstale":"2024-08-29T15:08:58.889787+0000","last_undegraded":"2024-08-29T15:08:58.889787+0000","last_fullsized":"2024-08-29T15:08:58.889787+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:32.997619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.877208+0000","last_change":"2024-08-29T15:07:52.135092+0000","last_active":"2024-08-29T15:08:57.877208+0000","last_peered":"2024-08-29T15:08:57.877208+0000","last_clean":"2024-08-29T15:08:57.877208+0000","last_became_active":"2024-08-29T15:07:52.134625+0000","last_became_peered":"2024-08-29T15:07:52.134625+0000","last_unstale":"2024-08-29T15:08:57.877208+0000","last_undegraded":"2024-08-29T15:08:57.877208+0000","last_fullsized":"2024-08-29T15:08:57.877208+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:22:35.250433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885989+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:08:58.885989+0000","last_peered":"2024-08-29T15:08:58.885989+0000","last_clean":"2024-08-29T15:08:58.885989+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:08:58.885989+0000","last_undegraded":"2024-08-29T15:08:58.885989+0000","last_fullsized":"2024-08-29T15:08:58.885989+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889927+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:08:58.889927+0000","last_peered":"2024-08-29T15:08:58.889927+0000","last_clean":"2024-08-29T15:08:58.889927+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:08:58.889927+0000","last_undegraded":"2024-08-29T15:08:58.889927+0000","last_fullsized":"2024-08-29T15:08:58.889927+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.883653+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:08:58.883653+0000","last_peered":"2024-08-29T15:08:58.883653+0000","last_clean":"2024-08-29T15:08:58.883653+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:08:58.883653+0000","last_undegraded":"2024-08-29T15:08:58.883653+0000","last_fullsized":"2024-08-29T15:08:58.883653+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888863+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:08:58.888863+0000","last_peered":"2024-08-29T15:08:58.888863+0000","last_clean":"2024-08-29T15:08:58.888863+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:08:58.888863+0000","last_undegraded":"2024-08-29T15:08:58.888863+0000","last_fullsized":"2024-08-29T15:08:58.888863+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.873347+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:08:57.873347+0000","last_peered":"2024-08-29T15:08:57.873347+0000","last_clean":"2024-08-29T15:08:57.873347+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:08:57.873347+0000","last_undegraded":"2024-08-29T15:08:57.873347+0000","last_fullsized":"2024-08-29T15:08:57.873347+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884205+0000","last_change":"2024-08-29T15:08:58.884121+0000","last_active":"2024-08-29T15:08:58.884205+0000","last_peered":"2024-08-29T15:08:58.884205+0000","last_clean":"2024-08-29T15:08:58.884205+0000","last_became_active":"2024-08-29T15:08:58.883669+0000","last_became_peered":"2024-08-29T15:08:58.883669+0000","last_unstale":"2024-08-29T15:08:58.884205+0000","last_undegraded":"2024-08-29T15:08:58.884205+0000","last_fullsized":"2024-08-29T15:08:58.884205+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:32.165590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878176+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:08:58.878176+0000","last_peered":"2024-08-29T15:08:58.878176+0000","last_clean":"2024-08-29T15:08:58.878176+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:08:58.878176+0000","last_undegraded":"2024-08-29T15:08:58.878176+0000","last_fullsized":"2024-08-29T15:08:58.878176+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884996+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:08:58.884996+0000","last_peered":"2024-08-29T15:08:58.884996+0000","last_clean":"2024-08-29T15:08:58.884996+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:08:58.884996+0000","last_undegraded":"2024-08-29T15:08:58.884996+0000","last_fullsized":"2024-08-29T15:08:58.884996+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872326+0000","last_change":"2024-08-29T15:07:52.134597+0000","last_active":"2024-08-29T15:08:57.872326+0000","last_peered":"2024-08-29T15:08:57.872326+0000","last_clean":"2024-08-29T15:08:57.872326+0000","last_became_active":"2024-08-29T15:07:52.133893+0000","last_became_peered":"2024-08-29T15:07:52.133893+0000","last_unstale":"2024-08-29T15:08:57.872326+0000","last_undegraded":"2024-08-29T15:08:57.872326+0000","last_fullsized":"2024-08-29T15:08:57.872326+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:14.240866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884919+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:08:58.884919+0000","last_peered":"2024-08-29T15:08:58.884919+0000","last_clean":"2024-08-29T15:08:58.884919+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:08:58.884919+0000","last_undegraded":"2024-08-29T15:08:58.884919+0000","last_fullsized":"2024-08-29T15:08:58.884919+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875599+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T15:08:57.875599+0000","last_peered":"2024-08-29T15:08:57.875599+0000","last_clean":"2024-08-29T15:08:57.875599+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T15:08:57.875599+0000","last_undegraded":"2024-08-29T15:08:57.875599+0000","last_fullsized":"2024-08-29T15:08:57.875599+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888235+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:08:58.888235+0000","last_peered":"2024-08-29T15:08:58.888235+0000","last_clean":"2024-08-29T15:08:58.888235+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:08:58.888235+0000","last_undegraded":"2024-08-29T15:08:58.888235+0000","last_fullsized":"2024-08-29T15:08:58.888235+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888750+0000","last_change":"2024-08-29T15:08:58.888653+0000","last_active":"2024-08-29T15:08:58.888750+0000","last_peered":"2024-08-29T15:08:58.888750+0000","last_clean":"2024-08-29T15:08:58.888750+0000","last_became_active":"2024-08-29T15:08:58.888133+0000","last_became_peered":"2024-08-29T15:08:58.888133+0000","last_unstale":"2024-08-29T15:08:58.888750+0000","last_undegraded":"2024-08-29T15:08:58.888750+0000","last_fullsized":"2024-08-29T15:08:58.888750+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:24.144974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872210+0000","last_change":"2024-08-29T15:07:52.135649+0000","last_active":"2024-08-29T15:08:57.872210+0000","last_peered":"2024-08-29T15:08:57.872210+0000","last_clean":"2024-08-29T15:08:57.872210+0000","last_became_active":"2024-08-29T15:07:52.135327+0000","last_became_peered":"2024-08-29T15:07:52.135327+0000","last_unstale":"2024-08-29T15:08:57.872210+0000","last_undegraded":"2024-08-29T15:08:57.872210+0000","last_fullsized":"2024-08-29T15:08:57.872210+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:05.402462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878257+0000","last_change":"2024-08-29T15:08:25.187422+0000","last_active":"2024-08-29T15:08:58.878257+0000","last_peered":"2024-08-29T15:08:58.878257+0000","last_clean":"2024-08-29T15:08:58.878257+0000","last_became_active":"2024-08-29T15:08:25.187062+0000","last_became_peered":"2024-08-29T15:08:25.187062+0000","last_unstale":"2024-08-29T15:08:58.878257+0000","last_undegraded":"2024-08-29T15:08:58.878257+0000","last_fullsized":"2024-08-29T15:08:58.878257+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:18:20.745348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234182,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37004,"kb_used_data":5888,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708148,"statfs":{"total":95995035648,"available":95957143552,"internally_reserved":0,"allocated":6029312,"data_stored":5417499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495818,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28808,"kb_used_data":1780,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716344,"statfs":{"total":95995035648,"available":95965536256,"internally_reserved":0,"allocated":1822720,"data_stored":1222795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757455,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1220608,"data_stored":628195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019092,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36420,"kb_used_data":5300,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708732,"statfs":{"total":95995035648,"available":95957741568,"internally_reserved":0,"allocated":5427200,"data_stored":4826788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1188,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1216512,"data_stored":627957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":28110089},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598803,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1180,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1208320,"data_stored":627233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":28172623},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762330,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28728,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716424,"statfs":{"total":95995035648,"available":95965618176,"internally_reserved":0,"allocated":1216512,"data_stored":627886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":28172642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925856,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33396,"kb_used_data":5860,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711756,"statfs":{"total":95995035648,"available":95960838144,"internally_reserved":0,"allocated":6000640,"data_stored":5411702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":28172431},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:09:54.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:09:54.717 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:55.144 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:54 smithi027 ceph-mon[98970]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:09:55.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:54 smithi027 ceph-mon[93921]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:09:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:54 smithi171 ceph-mon[90031]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:09:56.151 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:55 smithi027 ceph-mon[98970]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:56.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:55 smithi027 ceph-mon[93921]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:55 smithi171 ceph-mon[90031]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:56.432 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:09:56.433 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:09:57.095 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":277,"stamp":"2024-08-29T15:09:56.330941+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":249952,"kb_used_data":23576,"kb_used_omap":34,"kb_used_meta":226205,"kb_avail":749711264,"statfs":{"total":767960285184,"available":767704334336,"internally_reserved":0,"allocated":24141824,"data_stored":19390055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34969,"internal_metadata":231634791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002899"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882496+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:08:58.882496+0000","last_peered":"2024-08-29T15:08:58.882496+0000","last_clean":"2024-08-29T15:08:58.882496+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:08:58.882496+0000","last_undegraded":"2024-08-29T15:08:58.882496+0000","last_fullsized":"2024-08-29T15:08:58.882496+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877466+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:08:58.877466+0000","last_peered":"2024-08-29T15:08:58.877466+0000","last_clean":"2024-08-29T15:08:58.877466+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:08:58.877466+0000","last_undegraded":"2024-08-29T15:08:58.877466+0000","last_fullsized":"2024-08-29T15:08:58.877466+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887518+0000","last_change":"2024-08-29T15:08:58.887518+0000","last_active":"2024-08-29T15:08:58.887518+0000","last_peered":"2024-08-29T15:08:58.887518+0000","last_clean":"2024-08-29T15:08:58.887518+0000","last_became_active":"2024-08-29T15:08:58.887049+0000","last_became_peered":"2024-08-29T15:08:58.887049+0000","last_unstale":"2024-08-29T15:08:58.887518+0000","last_undegraded":"2024-08-29T15:08:58.887518+0000","last_fullsized":"2024-08-29T15:08:58.887518+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:23:07.638407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890091+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:08:58.890091+0000","last_peered":"2024-08-29T15:08:58.890091+0000","last_clean":"2024-08-29T15:08:58.890091+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:08:58.890091+0000","last_undegraded":"2024-08-29T15:08:58.890091+0000","last_fullsized":"2024-08-29T15:08:58.890091+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889545+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:08:58.889545+0000","last_peered":"2024-08-29T15:08:58.889545+0000","last_clean":"2024-08-29T15:08:58.889545+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:08:58.889545+0000","last_undegraded":"2024-08-29T15:08:58.889545+0000","last_fullsized":"2024-08-29T15:08:58.889545+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877550+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T15:08:58.877550+0000","last_peered":"2024-08-29T15:08:58.877550+0000","last_clean":"2024-08-29T15:08:58.877550+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T15:08:58.877550+0000","last_undegraded":"2024-08-29T15:08:58.877550+0000","last_fullsized":"2024-08-29T15:08:58.877550+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884766+0000","last_change":"2024-08-29T15:08:58.884675+0000","last_active":"2024-08-29T15:08:58.884766+0000","last_peered":"2024-08-29T15:08:58.884766+0000","last_clean":"2024-08-29T15:08:58.884766+0000","last_became_active":"2024-08-29T15:08:58.884007+0000","last_became_peered":"2024-08-29T15:08:58.884007+0000","last_unstale":"2024-08-29T15:08:58.884766+0000","last_undegraded":"2024-08-29T15:08:58.884766+0000","last_fullsized":"2024-08-29T15:08:58.884766+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:12:03.629717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885010+0000","last_change":"2024-08-29T15:07:20.812313+0000","last_active":"2024-08-29T15:08:58.885010+0000","last_peered":"2024-08-29T15:08:58.885010+0000","last_clean":"2024-08-29T15:08:58.885010+0000","last_became_active":"2024-08-29T15:07:20.811973+0000","last_became_peered":"2024-08-29T15:07:20.811973+0000","last_unstale":"2024-08-29T15:08:58.885010+0000","last_undegraded":"2024-08-29T15:08:58.885010+0000","last_fullsized":"2024-08-29T15:08:58.885010+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:37.283470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877737+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:08:58.877737+0000","last_peered":"2024-08-29T15:08:58.877737+0000","last_clean":"2024-08-29T15:08:58.877737+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:08:58.877737+0000","last_undegraded":"2024-08-29T15:08:58.877737+0000","last_fullsized":"2024-08-29T15:08:58.877737+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872144+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T15:08:57.872144+0000","last_peered":"2024-08-29T15:08:57.872144+0000","last_clean":"2024-08-29T15:08:57.872144+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T15:08:57.872144+0000","last_undegraded":"2024-08-29T15:08:57.872144+0000","last_fullsized":"2024-08-29T15:08:57.872144+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877594+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:08:58.877594+0000","last_peered":"2024-08-29T15:08:58.877594+0000","last_clean":"2024-08-29T15:08:58.877594+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:08:58.877594+0000","last_undegraded":"2024-08-29T15:08:58.877594+0000","last_fullsized":"2024-08-29T15:08:58.877594+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885413+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:08:58.885413+0000","last_peered":"2024-08-29T15:08:58.885413+0000","last_clean":"2024-08-29T15:08:58.885413+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:08:58.885413+0000","last_undegraded":"2024-08-29T15:08:58.885413+0000","last_fullsized":"2024-08-29T15:08:58.885413+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877537+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T15:08:58.877537+0000","last_peered":"2024-08-29T15:08:58.877537+0000","last_clean":"2024-08-29T15:08:58.877537+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T15:08:58.877537+0000","last_undegraded":"2024-08-29T15:08:58.877537+0000","last_fullsized":"2024-08-29T15:08:58.877537+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876085+0000","last_change":"2024-08-29T15:07:52.136104+0000","last_active":"2024-08-29T15:08:57.876085+0000","last_peered":"2024-08-29T15:08:57.876085+0000","last_clean":"2024-08-29T15:08:57.876085+0000","last_became_active":"2024-08-29T15:07:52.135781+0000","last_became_peered":"2024-08-29T15:07:52.135781+0000","last_unstale":"2024-08-29T15:08:57.876085+0000","last_undegraded":"2024-08-29T15:08:57.876085+0000","last_fullsized":"2024-08-29T15:08:57.876085+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:26.381499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888097+0000","last_change":"2024-08-29T15:08:58.887669+0000","last_active":"2024-08-29T15:08:58.888097+0000","last_peered":"2024-08-29T15:08:58.888097+0000","last_clean":"2024-08-29T15:08:58.888097+0000","last_became_active":"2024-08-29T15:08:58.887343+0000","last_became_peered":"2024-08-29T15:08:58.887343+0000","last_unstale":"2024-08-29T15:08:58.888097+0000","last_undegraded":"2024-08-29T15:08:58.888097+0000","last_fullsized":"2024-08-29T15:08:58.888097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:32.280756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878358+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:08:58.878358+0000","last_peered":"2024-08-29T15:08:58.878358+0000","last_clean":"2024-08-29T15:08:58.878358+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:08:58.878358+0000","last_undegraded":"2024-08-29T15:08:58.878358+0000","last_fullsized":"2024-08-29T15:08:58.878358+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889564+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:08:58.889564+0000","last_peered":"2024-08-29T15:08:58.889564+0000","last_clean":"2024-08-29T15:08:58.889564+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:08:58.889564+0000","last_undegraded":"2024-08-29T15:08:58.889564+0000","last_fullsized":"2024-08-29T15:08:58.889564+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886744+0000","last_change":"2024-08-29T15:08:58.886369+0000","last_active":"2024-08-29T15:08:58.886744+0000","last_peered":"2024-08-29T15:08:58.886744+0000","last_clean":"2024-08-29T15:08:58.886744+0000","last_became_active":"2024-08-29T15:08:58.886022+0000","last_became_peered":"2024-08-29T15:08:58.886022+0000","last_unstale":"2024-08-29T15:08:58.886744+0000","last_undegraded":"2024-08-29T15:08:58.886744+0000","last_fullsized":"2024-08-29T15:08:58.886744+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:36.451849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876498+0000","last_change":"2024-08-29T15:07:52.134960+0000","last_active":"2024-08-29T15:08:57.876498+0000","last_peered":"2024-08-29T15:08:57.876498+0000","last_clean":"2024-08-29T15:08:57.876498+0000","last_became_active":"2024-08-29T15:07:52.134608+0000","last_became_peered":"2024-08-29T15:07:52.134608+0000","last_unstale":"2024-08-29T15:08:57.876498+0000","last_undegraded":"2024-08-29T15:08:57.876498+0000","last_fullsized":"2024-08-29T15:08:57.876498+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:42.154504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875880+0000","last_change":"2024-08-29T15:08:25.191304+0000","last_active":"2024-08-29T15:08:58.875880+0000","last_peered":"2024-08-29T15:08:58.875880+0000","last_clean":"2024-08-29T15:08:58.875880+0000","last_became_active":"2024-08-29T15:08:25.191022+0000","last_became_peered":"2024-08-29T15:08:25.191022+0000","last_unstale":"2024-08-29T15:08:58.875880+0000","last_undegraded":"2024-08-29T15:08:58.875880+0000","last_fullsized":"2024-08-29T15:08:58.875880+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:07.608759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.874763+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:08:57.874763+0000","last_peered":"2024-08-29T15:08:57.874763+0000","last_clean":"2024-08-29T15:08:57.874763+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:08:57.874763+0000","last_undegraded":"2024-08-29T15:08:57.874763+0000","last_fullsized":"2024-08-29T15:08:57.874763+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875459+0000","last_change":"2024-08-29T15:07:52.142297+0000","last_active":"2024-08-29T15:08:58.875459+0000","last_peered":"2024-08-29T15:08:58.875459+0000","last_clean":"2024-08-29T15:08:58.875459+0000","last_became_active":"2024-08-29T15:07:52.141836+0000","last_became_peered":"2024-08-29T15:07:52.141836+0000","last_unstale":"2024-08-29T15:08:58.875459+0000","last_undegraded":"2024-08-29T15:08:58.875459+0000","last_fullsized":"2024-08-29T15:08:58.875459+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:58:01.221672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881991+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:08:58.881991+0000","last_peered":"2024-08-29T15:08:58.881991+0000","last_clean":"2024-08-29T15:08:58.881991+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:08:58.881991+0000","last_undegraded":"2024-08-29T15:08:58.881991+0000","last_fullsized":"2024-08-29T15:08:58.881991+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882231+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:08:58.882231+0000","last_peered":"2024-08-29T15:08:58.882231+0000","last_clean":"2024-08-29T15:08:58.882231+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:08:58.882231+0000","last_undegraded":"2024-08-29T15:08:58.882231+0000","last_fullsized":"2024-08-29T15:08:58.882231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":404,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877283+0000","last_change":"2024-08-29T15:07:20.817595+0000","last_active":"2024-08-29T15:08:58.877283+0000","last_peered":"2024-08-29T15:08:58.877283+0000","last_clean":"2024-08-29T15:08:58.877283+0000","last_became_active":"2024-08-29T15:07:20.817130+0000","last_became_peered":"2024-08-29T15:07:20.817130+0000","last_unstale":"2024-08-29T15:08:58.877283+0000","last_undegraded":"2024-08-29T15:08:58.877283+0000","last_fullsized":"2024-08-29T15:08:58.877283+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:28:33.151864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":595,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874858+0000","last_change":"2024-08-29T15:08:25.185728+0000","last_active":"2024-08-29T15:08:58.874858+0000","last_peered":"2024-08-29T15:08:58.874858+0000","last_clean":"2024-08-29T15:08:58.874858+0000","last_became_active":"2024-08-29T15:08:25.185219+0000","last_became_peered":"2024-08-29T15:08:25.185219+0000","last_unstale":"2024-08-29T15:08:58.874858+0000","last_undegraded":"2024-08-29T15:08:58.874858+0000","last_fullsized":"2024-08-29T15:08:58.874858+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:16:25.013137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890162+0000","last_change":"2024-08-29T15:08:58.890060+0000","last_active":"2024-08-29T15:08:58.890162+0000","last_peered":"2024-08-29T15:08:58.890162+0000","last_clean":"2024-08-29T15:08:58.890162+0000","last_became_active":"2024-08-29T15:08:58.889714+0000","last_became_peered":"2024-08-29T15:08:58.889714+0000","last_unstale":"2024-08-29T15:08:58.890162+0000","last_undegraded":"2024-08-29T15:08:58.890162+0000","last_fullsized":"2024-08-29T15:08:58.890162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:50.659662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876262+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T15:08:57.876262+0000","last_peered":"2024-08-29T15:08:57.876262+0000","last_clean":"2024-08-29T15:08:57.876262+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T15:08:57.876262+0000","last_undegraded":"2024-08-29T15:08:57.876262+0000","last_fullsized":"2024-08-29T15:08:57.876262+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":800,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.879076+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:08:57.879076+0000","last_peered":"2024-08-29T15:08:57.879076+0000","last_clean":"2024-08-29T15:08:57.879076+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:08:57.879076+0000","last_undegraded":"2024-08-29T15:08:57.879076+0000","last_fullsized":"2024-08-29T15:08:57.879076+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889019+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:08:58.889019+0000","last_peered":"2024-08-29T15:08:58.889019+0000","last_clean":"2024-08-29T15:08:58.889019+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:08:58.889019+0000","last_undegraded":"2024-08-29T15:08:58.889019+0000","last_fullsized":"2024-08-29T15:08:58.889019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.879442+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:08:58.879442+0000","last_peered":"2024-08-29T15:08:58.879442+0000","last_clean":"2024-08-29T15:08:58.879442+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:08:58.879442+0000","last_undegraded":"2024-08-29T15:08:58.879442+0000","last_fullsized":"2024-08-29T15:08:58.879442+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":540,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881830+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:08:58.881830+0000","last_peered":"2024-08-29T15:08:58.881830+0000","last_clean":"2024-08-29T15:08:58.881830+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:08:58.881830+0000","last_undegraded":"2024-08-29T15:08:58.881830+0000","last_fullsized":"2024-08-29T15:08:58.881830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873790+0000","last_change":"2024-08-29T15:08:25.188186+0000","last_active":"2024-08-29T15:08:58.873790+0000","last_peered":"2024-08-29T15:08:58.873790+0000","last_clean":"2024-08-29T15:08:58.873790+0000","last_became_active":"2024-08-29T15:08:25.187830+0000","last_became_peered":"2024-08-29T15:08:25.187830+0000","last_unstale":"2024-08-29T15:08:58.873790+0000","last_undegraded":"2024-08-29T15:08:58.873790+0000","last_fullsized":"2024-08-29T15:08:58.873790+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:00:12.780470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873782+0000","last_change":"2024-08-29T15:07:52.139273+0000","last_active":"2024-08-29T15:08:58.873782+0000","last_peered":"2024-08-29T15:08:58.873782+0000","last_clean":"2024-08-29T15:08:58.873782+0000","last_became_active":"2024-08-29T15:07:52.138052+0000","last_became_peered":"2024-08-29T15:07:52.138052+0000","last_unstale":"2024-08-29T15:08:58.873782+0000","last_undegraded":"2024-08-29T15:08:58.873782+0000","last_fullsized":"2024-08-29T15:08:58.873782+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:01.705167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":526,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872946+0000","last_change":"2024-08-29T15:07:52.138323+0000","last_active":"2024-08-29T15:08:57.872946+0000","last_peered":"2024-08-29T15:08:57.872946+0000","last_clean":"2024-08-29T15:08:57.872946+0000","last_became_active":"2024-08-29T15:07:52.137781+0000","last_became_peered":"2024-08-29T15:07:52.137781+0000","last_unstale":"2024-08-29T15:08:57.872946+0000","last_undegraded":"2024-08-29T15:08:57.872946+0000","last_fullsized":"2024-08-29T15:08:57.872946+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:11:27.075260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":579,"num_read_kb":2582,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875043+0000","last_change":"2024-08-29T15:07:20.816485+0000","last_active":"2024-08-29T15:08:57.875043+0000","last_peered":"2024-08-29T15:08:57.875043+0000","last_clean":"2024-08-29T15:08:57.875043+0000","last_became_active":"2024-08-29T15:07:20.816190+0000","last_became_peered":"2024-08-29T15:07:20.816190+0000","last_unstale":"2024-08-29T15:08:57.875043+0000","last_undegraded":"2024-08-29T15:08:57.875043+0000","last_fullsized":"2024-08-29T15:08:57.875043+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:06.731598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884316+0000","last_change":"2024-08-29T15:08:58.883318+0000","last_active":"2024-08-29T15:08:58.884316+0000","last_peered":"2024-08-29T15:08:58.884316+0000","last_clean":"2024-08-29T15:08:58.884316+0000","last_became_active":"2024-08-29T15:08:58.882978+0000","last_became_peered":"2024-08-29T15:08:58.882978+0000","last_unstale":"2024-08-29T15:08:58.884316+0000","last_undegraded":"2024-08-29T15:08:58.884316+0000","last_fullsized":"2024-08-29T15:08:58.884316+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:08.675766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887791+0000","last_change":"2024-08-29T15:08:58.887791+0000","last_active":"2024-08-29T15:08:58.887791+0000","last_peered":"2024-08-29T15:08:58.887791+0000","last_clean":"2024-08-29T15:08:58.887791+0000","last_became_active":"2024-08-29T15:08:58.887472+0000","last_became_peered":"2024-08-29T15:08:58.887472+0000","last_unstale":"2024-08-29T15:08:58.887791+0000","last_undegraded":"2024-08-29T15:08:58.887791+0000","last_fullsized":"2024-08-29T15:08:58.887791+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:35:58.690131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":507,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874346+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:08:58.874346+0000","last_peered":"2024-08-29T15:08:58.874346+0000","last_clean":"2024-08-29T15:08:58.874346+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:08:58.874346+0000","last_undegraded":"2024-08-29T15:08:58.874346+0000","last_fullsized":"2024-08-29T15:08:58.874346+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873788+0000","last_change":"2024-08-29T15:07:52.139543+0000","last_active":"2024-08-29T15:08:58.873788+0000","last_peered":"2024-08-29T15:08:58.873788+0000","last_clean":"2024-08-29T15:08:58.873788+0000","last_became_active":"2024-08-29T15:07:52.138499+0000","last_became_peered":"2024-08-29T15:07:52.138499+0000","last_unstale":"2024-08-29T15:08:58.873788+0000","last_undegraded":"2024-08-29T15:08:58.873788+0000","last_fullsized":"2024-08-29T15:08:58.873788+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:41:05.588954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874122+0000","last_change":"2024-08-29T15:08:25.186307+0000","last_active":"2024-08-29T15:08:58.874122+0000","last_peered":"2024-08-29T15:08:58.874122+0000","last_clean":"2024-08-29T15:08:58.874122+0000","last_became_active":"2024-08-29T15:08:25.185788+0000","last_became_peered":"2024-08-29T15:08:25.185788+0000","last_unstale":"2024-08-29T15:08:58.874122+0000","last_undegraded":"2024-08-29T15:08:58.874122+0000","last_fullsized":"2024-08-29T15:08:58.874122+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:58:08.511525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":497,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877648+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:08:58.877648+0000","last_peered":"2024-08-29T15:08:58.877648+0000","last_clean":"2024-08-29T15:08:58.877648+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:08:58.877648+0000","last_undegraded":"2024-08-29T15:08:58.877648+0000","last_fullsized":"2024-08-29T15:08:58.877648+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878271+0000","last_change":"2024-08-29T15:07:20.813592+0000","last_active":"2024-08-29T15:08:58.878271+0000","last_peered":"2024-08-29T15:08:58.878271+0000","last_clean":"2024-08-29T15:08:58.878271+0000","last_became_active":"2024-08-29T15:07:20.812970+0000","last_became_peered":"2024-08-29T15:07:20.812970+0000","last_unstale":"2024-08-29T15:08:58.878271+0000","last_undegraded":"2024-08-29T15:08:58.878271+0000","last_fullsized":"2024-08-29T15:08:58.878271+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:54.850894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888546+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:08:58.888546+0000","last_peered":"2024-08-29T15:08:58.888546+0000","last_clean":"2024-08-29T15:08:58.888546+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:08:58.888546+0000","last_undegraded":"2024-08-29T15:08:58.888546+0000","last_fullsized":"2024-08-29T15:08:58.888546+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":597,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874088+0000","last_change":"2024-08-29T15:07:52.139196+0000","last_active":"2024-08-29T15:08:58.874088+0000","last_peered":"2024-08-29T15:08:58.874088+0000","last_clean":"2024-08-29T15:08:58.874088+0000","last_became_active":"2024-08-29T15:07:52.138669+0000","last_became_peered":"2024-08-29T15:07:52.138669+0000","last_unstale":"2024-08-29T15:08:58.874088+0000","last_undegraded":"2024-08-29T15:08:58.874088+0000","last_fullsized":"2024-08-29T15:08:58.874088+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:43:54.949715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888619+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:08:58.888619+0000","last_peered":"2024-08-29T15:08:58.888619+0000","last_clean":"2024-08-29T15:08:58.888619+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:08:58.888619+0000","last_undegraded":"2024-08-29T15:08:58.888619+0000","last_fullsized":"2024-08-29T15:08:58.888619+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878638+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:08:58.878638+0000","last_peered":"2024-08-29T15:08:58.878638+0000","last_clean":"2024-08-29T15:08:58.878638+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:08:58.878638+0000","last_undegraded":"2024-08-29T15:08:58.878638+0000","last_fullsized":"2024-08-29T15:08:58.878638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":337,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885216+0000","last_change":"2024-08-29T15:08:25.186819+0000","last_active":"2024-08-29T15:08:58.885216+0000","last_peered":"2024-08-29T15:08:58.885216+0000","last_clean":"2024-08-29T15:08:58.885216+0000","last_became_active":"2024-08-29T15:08:25.186401+0000","last_became_peered":"2024-08-29T15:08:25.186401+0000","last_unstale":"2024-08-29T15:08:58.885216+0000","last_undegraded":"2024-08-29T15:08:58.885216+0000","last_fullsized":"2024-08-29T15:08:58.885216+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:45.101824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.873204+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T15:08:57.873204+0000","last_peered":"2024-08-29T15:08:57.873204+0000","last_clean":"2024-08-29T15:08:57.873204+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T15:08:57.873204+0000","last_undegraded":"2024-08-29T15:08:57.873204+0000","last_fullsized":"2024-08-29T15:08:57.873204+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875638+0000","last_change":"2024-08-29T15:08:25.185452+0000","last_active":"2024-08-29T15:08:58.875638+0000","last_peered":"2024-08-29T15:08:58.875638+0000","last_clean":"2024-08-29T15:08:58.875638+0000","last_became_active":"2024-08-29T15:08:25.185095+0000","last_became_peered":"2024-08-29T15:08:25.185095+0000","last_unstale":"2024-08-29T15:08:58.875638+0000","last_undegraded":"2024-08-29T15:08:58.875638+0000","last_fullsized":"2024-08-29T15:08:58.875638+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:19:03.238217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875238+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T15:08:58.875238+0000","last_peered":"2024-08-29T15:08:58.875238+0000","last_clean":"2024-08-29T15:08:58.875238+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T15:08:58.875238+0000","last_undegraded":"2024-08-29T15:08:58.875238+0000","last_fullsized":"2024-08-29T15:08:58.875238+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873562+0000","last_change":"2024-08-29T15:08:25.186368+0000","last_active":"2024-08-29T15:08:58.873562+0000","last_peered":"2024-08-29T15:08:58.873562+0000","last_clean":"2024-08-29T15:08:58.873562+0000","last_became_active":"2024-08-29T15:08:25.185912+0000","last_became_peered":"2024-08-29T15:08:25.185912+0000","last_unstale":"2024-08-29T15:08:58.873562+0000","last_undegraded":"2024-08-29T15:08:58.873562+0000","last_fullsized":"2024-08-29T15:08:58.873562+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:23.124995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886830+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:08:58.886830+0000","last_peered":"2024-08-29T15:08:58.886830+0000","last_clean":"2024-08-29T15:08:58.886830+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:08:58.886830+0000","last_undegraded":"2024-08-29T15:08:58.886830+0000","last_fullsized":"2024-08-29T15:08:58.886830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872257+0000","last_change":"2024-08-29T15:07:52.136541+0000","last_active":"2024-08-29T15:08:57.872257+0000","last_peered":"2024-08-29T15:08:57.872257+0000","last_clean":"2024-08-29T15:08:57.872257+0000","last_became_active":"2024-08-29T15:07:52.136202+0000","last_became_peered":"2024-08-29T15:07:52.136202+0000","last_unstale":"2024-08-29T15:08:57.872257+0000","last_undegraded":"2024-08-29T15:08:57.872257+0000","last_fullsized":"2024-08-29T15:08:57.872257+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:01.747963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888935+0000","last_change":"2024-08-29T15:08:58.888848+0000","last_active":"2024-08-29T15:08:58.888935+0000","last_peered":"2024-08-29T15:08:58.888935+0000","last_clean":"2024-08-29T15:08:58.888935+0000","last_became_active":"2024-08-29T15:08:58.886702+0000","last_became_peered":"2024-08-29T15:08:58.886702+0000","last_unstale":"2024-08-29T15:08:58.888935+0000","last_undegraded":"2024-08-29T15:08:58.888935+0000","last_fullsized":"2024-08-29T15:08:58.888935+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:56.189439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887636+0000","last_change":"2024-08-29T15:08:58.887151+0000","last_active":"2024-08-29T15:08:58.887636+0000","last_peered":"2024-08-29T15:08:58.887636+0000","last_clean":"2024-08-29T15:08:58.887636+0000","last_became_active":"2024-08-29T15:08:58.886918+0000","last_became_peered":"2024-08-29T15:08:58.886918+0000","last_unstale":"2024-08-29T15:08:58.887636+0000","last_undegraded":"2024-08-29T15:08:58.887636+0000","last_fullsized":"2024-08-29T15:08:58.887636+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:37.557892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889787+0000","last_change":"2024-08-29T15:08:58.889700+0000","last_active":"2024-08-29T15:08:58.889787+0000","last_peered":"2024-08-29T15:08:58.889787+0000","last_clean":"2024-08-29T15:08:58.889787+0000","last_became_active":"2024-08-29T15:08:58.887521+0000","last_became_peered":"2024-08-29T15:08:58.887521+0000","last_unstale":"2024-08-29T15:08:58.889787+0000","last_undegraded":"2024-08-29T15:08:58.889787+0000","last_fullsized":"2024-08-29T15:08:58.889787+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:32.997619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.877208+0000","last_change":"2024-08-29T15:07:52.135092+0000","last_active":"2024-08-29T15:08:57.877208+0000","last_peered":"2024-08-29T15:08:57.877208+0000","last_clean":"2024-08-29T15:08:57.877208+0000","last_became_active":"2024-08-29T15:07:52.134625+0000","last_became_peered":"2024-08-29T15:07:52.134625+0000","last_unstale":"2024-08-29T15:08:57.877208+0000","last_undegraded":"2024-08-29T15:08:57.877208+0000","last_fullsized":"2024-08-29T15:08:57.877208+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:22:35.250433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885989+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:08:58.885989+0000","last_peered":"2024-08-29T15:08:58.885989+0000","last_clean":"2024-08-29T15:08:58.885989+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:08:58.885989+0000","last_undegraded":"2024-08-29T15:08:58.885989+0000","last_fullsized":"2024-08-29T15:08:58.885989+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889927+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:08:58.889927+0000","last_peered":"2024-08-29T15:08:58.889927+0000","last_clean":"2024-08-29T15:08:58.889927+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:08:58.889927+0000","last_undegraded":"2024-08-29T15:08:58.889927+0000","last_fullsized":"2024-08-29T15:08:58.889927+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.883653+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:08:58.883653+0000","last_peered":"2024-08-29T15:08:58.883653+0000","last_clean":"2024-08-29T15:08:58.883653+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:08:58.883653+0000","last_undegraded":"2024-08-29T15:08:58.883653+0000","last_fullsized":"2024-08-29T15:08:58.883653+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888863+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:08:58.888863+0000","last_peered":"2024-08-29T15:08:58.888863+0000","last_clean":"2024-08-29T15:08:58.888863+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:08:58.888863+0000","last_undegraded":"2024-08-29T15:08:58.888863+0000","last_fullsized":"2024-08-29T15:08:58.888863+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.873347+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:08:57.873347+0000","last_peered":"2024-08-29T15:08:57.873347+0000","last_clean":"2024-08-29T15:08:57.873347+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:08:57.873347+0000","last_undegraded":"2024-08-29T15:08:57.873347+0000","last_fullsized":"2024-08-29T15:08:57.873347+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884205+0000","last_change":"2024-08-29T15:08:58.884121+0000","last_active":"2024-08-29T15:08:58.884205+0000","last_peered":"2024-08-29T15:08:58.884205+0000","last_clean":"2024-08-29T15:08:58.884205+0000","last_became_active":"2024-08-29T15:08:58.883669+0000","last_became_peered":"2024-08-29T15:08:58.883669+0000","last_unstale":"2024-08-29T15:08:58.884205+0000","last_undegraded":"2024-08-29T15:08:58.884205+0000","last_fullsized":"2024-08-29T15:08:58.884205+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:32.165590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878176+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:08:58.878176+0000","last_peered":"2024-08-29T15:08:58.878176+0000","last_clean":"2024-08-29T15:08:58.878176+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:08:58.878176+0000","last_undegraded":"2024-08-29T15:08:58.878176+0000","last_fullsized":"2024-08-29T15:08:58.878176+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884996+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:08:58.884996+0000","last_peered":"2024-08-29T15:08:58.884996+0000","last_clean":"2024-08-29T15:08:58.884996+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:08:58.884996+0000","last_undegraded":"2024-08-29T15:08:58.884996+0000","last_fullsized":"2024-08-29T15:08:58.884996+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872326+0000","last_change":"2024-08-29T15:07:52.134597+0000","last_active":"2024-08-29T15:08:57.872326+0000","last_peered":"2024-08-29T15:08:57.872326+0000","last_clean":"2024-08-29T15:08:57.872326+0000","last_became_active":"2024-08-29T15:07:52.133893+0000","last_became_peered":"2024-08-29T15:07:52.133893+0000","last_unstale":"2024-08-29T15:08:57.872326+0000","last_undegraded":"2024-08-29T15:08:57.872326+0000","last_fullsized":"2024-08-29T15:08:57.872326+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:14.240866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884919+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:08:58.884919+0000","last_peered":"2024-08-29T15:08:58.884919+0000","last_clean":"2024-08-29T15:08:58.884919+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:08:58.884919+0000","last_undegraded":"2024-08-29T15:08:58.884919+0000","last_fullsized":"2024-08-29T15:08:58.884919+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875599+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T15:08:57.875599+0000","last_peered":"2024-08-29T15:08:57.875599+0000","last_clean":"2024-08-29T15:08:57.875599+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T15:08:57.875599+0000","last_undegraded":"2024-08-29T15:08:57.875599+0000","last_fullsized":"2024-08-29T15:08:57.875599+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888235+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:08:58.888235+0000","last_peered":"2024-08-29T15:08:58.888235+0000","last_clean":"2024-08-29T15:08:58.888235+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:08:58.888235+0000","last_undegraded":"2024-08-29T15:08:58.888235+0000","last_fullsized":"2024-08-29T15:08:58.888235+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888750+0000","last_change":"2024-08-29T15:08:58.888653+0000","last_active":"2024-08-29T15:08:58.888750+0000","last_peered":"2024-08-29T15:08:58.888750+0000","last_clean":"2024-08-29T15:08:58.888750+0000","last_became_active":"2024-08-29T15:08:58.888133+0000","last_became_peered":"2024-08-29T15:08:58.888133+0000","last_unstale":"2024-08-29T15:08:58.888750+0000","last_undegraded":"2024-08-29T15:08:58.888750+0000","last_fullsized":"2024-08-29T15:08:58.888750+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:24.144974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872210+0000","last_change":"2024-08-29T15:07:52.135649+0000","last_active":"2024-08-29T15:08:57.872210+0000","last_peered":"2024-08-29T15:08:57.872210+0000","last_clean":"2024-08-29T15:08:57.872210+0000","last_became_active":"2024-08-29T15:07:52.135327+0000","last_became_peered":"2024-08-29T15:07:52.135327+0000","last_unstale":"2024-08-29T15:08:57.872210+0000","last_undegraded":"2024-08-29T15:08:57.872210+0000","last_fullsized":"2024-08-29T15:08:57.872210+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:05.402462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878257+0000","last_change":"2024-08-29T15:08:25.187422+0000","last_active":"2024-08-29T15:08:58.878257+0000","last_peered":"2024-08-29T15:08:58.878257+0000","last_clean":"2024-08-29T15:08:58.878257+0000","last_became_active":"2024-08-29T15:08:25.187062+0000","last_became_peered":"2024-08-29T15:08:25.187062+0000","last_unstale":"2024-08-29T15:08:58.878257+0000","last_undegraded":"2024-08-29T15:08:58.878257+0000","last_fullsized":"2024-08-29T15:08:58.878257+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:18:20.745348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234183,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37004,"kb_used_data":5888,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708148,"statfs":{"total":95995035648,"available":95957143552,"internally_reserved":0,"allocated":6029312,"data_stored":5417499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495819,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28808,"kb_used_data":1780,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716344,"statfs":{"total":95995035648,"available":95965536256,"internally_reserved":0,"allocated":1822720,"data_stored":1222795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757456,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1220608,"data_stored":628195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019093,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36420,"kb_used_data":5300,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708732,"statfs":{"total":95995035648,"available":95957741568,"internally_reserved":0,"allocated":5427200,"data_stored":4826788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435277,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1188,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1216512,"data_stored":627957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":28110089},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598804,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1180,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1208320,"data_stored":627233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":28172623},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762331,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28728,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716424,"statfs":{"total":95995035648,"available":95965618176,"internally_reserved":0,"allocated":1216512,"data_stored":627886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":28172642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925857,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33396,"kb_used_data":5860,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711756,"statfs":{"total":95995035648,"available":95960838144,"internally_reserved":0,"allocated":6000640,"data_stored":5411702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":28172431},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:09:57.098 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-29T15:09:57.098 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-29T15:09:57.098 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-29T15:09:57.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph health --format=json 2024-08-29T15:09:57.305 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:09:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:57 smithi027 ceph-mon[93921]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:57 smithi027 ceph-mon[93921]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:09:58.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:57 smithi027 ceph-mon[98970]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:58.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:57 smithi027 ceph-mon[98970]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:09:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:57 smithi171 ceph-mon[90031]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:09:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:57 smithi171 ceph-mon[90031]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:09:59.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:59.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:09:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:09:59.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:59.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:09:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:09:59.173 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:09:59.173 INFO:teuthology.orchestra.run.smithi027.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-29T15:09:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:09:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:09:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:09:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:09:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:09:59] "GET /metrics HTTP/1.1" 200 33935 "" "Prometheus/2.51.0" 2024-08-29T15:09:59.770 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-29T15:09:59.770 INFO:teuthology.run_tasks:Running task print... 2024-08-29T15:09:59.779 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-08-29T15:09:59.780 INFO:teuthology.run_tasks:Running task parallel... 2024-08-29T15:09:59.801 INFO:teuthology.task.parallel:starting parallel... 2024-08-29T15:09:59.802 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-29T15:09:59.802 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-08-29T15:09:59.808 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-29T15:09:59.809 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'wip-yuri4-testing-2024-08-28-1359'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']}, 'workunit': {'branch': 'wip-yuri4-testing-2024-08-28-1359', 'sha1': '1f4056d968930e12926c25a8f13ee8ed6047c791'}} 2024-08-29T15:09:59.809 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-29T15:09:59.809 INFO:tasks.thrashosds:Beginning thrashosds... 2024-08-29T15:09:59.809 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-29T15:09:59.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:10:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[93921]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3181240539' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-29T15:10:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[93921]: overall HEALTH_OK 2024-08-29T15:10:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[98970]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3181240539' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-29T15:10:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:00 smithi027 ceph-mon[98970]: overall HEALTH_OK 2024-08-29T15:10:00.329 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:00 smithi171 ceph-mon[90031]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:00.329 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:00.329 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:00.329 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3181240539' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-29T15:10:00.329 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:00 smithi171 ceph-mon[90031]: overall HEALTH_OK 2024-08-29T15:10:00.725 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:02 smithi027 ceph-mon[93921]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:02 smithi027 ceph-mon[98970]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:02.371 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:02 smithi171 ceph-mon[90031]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:02.523 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:10:02.523 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:10:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[93921]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[93921]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:10:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[98970]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[98970]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:10:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:04.178 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":280,"stamp":"2024-08-29T15:10:02.332272+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":249952,"kb_used_data":23576,"kb_used_omap":34,"kb_used_meta":226205,"kb_avail":749711264,"statfs":{"total":767960285184,"available":767704334336,"internally_reserved":0,"allocated":24141824,"data_stored":19390055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34969,"internal_metadata":231634791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002875"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882496+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:08:58.882496+0000","last_peered":"2024-08-29T15:08:58.882496+0000","last_clean":"2024-08-29T15:08:58.882496+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:08:58.882496+0000","last_undegraded":"2024-08-29T15:08:58.882496+0000","last_fullsized":"2024-08-29T15:08:58.882496+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877466+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:08:58.877466+0000","last_peered":"2024-08-29T15:08:58.877466+0000","last_clean":"2024-08-29T15:08:58.877466+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:08:58.877466+0000","last_undegraded":"2024-08-29T15:08:58.877466+0000","last_fullsized":"2024-08-29T15:08:58.877466+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887905+0000","last_change":"2024-08-29T15:08:58.887518+0000","last_active":"2024-08-29T15:08:58.887905+0000","last_peered":"2024-08-29T15:08:58.887905+0000","last_clean":"2024-08-29T15:08:58.887905+0000","last_became_active":"2024-08-29T15:08:58.887049+0000","last_became_peered":"2024-08-29T15:08:58.887049+0000","last_unstale":"2024-08-29T15:08:58.887905+0000","last_undegraded":"2024-08-29T15:08:58.887905+0000","last_fullsized":"2024-08-29T15:08:58.887905+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:23:07.638407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890091+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:08:58.890091+0000","last_peered":"2024-08-29T15:08:58.890091+0000","last_clean":"2024-08-29T15:08:58.890091+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:08:58.890091+0000","last_undegraded":"2024-08-29T15:08:58.890091+0000","last_fullsized":"2024-08-29T15:08:58.890091+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889545+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:08:58.889545+0000","last_peered":"2024-08-29T15:08:58.889545+0000","last_clean":"2024-08-29T15:08:58.889545+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:08:58.889545+0000","last_undegraded":"2024-08-29T15:08:58.889545+0000","last_fullsized":"2024-08-29T15:08:58.889545+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877550+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T15:08:58.877550+0000","last_peered":"2024-08-29T15:08:58.877550+0000","last_clean":"2024-08-29T15:08:58.877550+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T15:08:58.877550+0000","last_undegraded":"2024-08-29T15:08:58.877550+0000","last_fullsized":"2024-08-29T15:08:58.877550+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884766+0000","last_change":"2024-08-29T15:08:58.884675+0000","last_active":"2024-08-29T15:08:58.884766+0000","last_peered":"2024-08-29T15:08:58.884766+0000","last_clean":"2024-08-29T15:08:58.884766+0000","last_became_active":"2024-08-29T15:08:58.884007+0000","last_became_peered":"2024-08-29T15:08:58.884007+0000","last_unstale":"2024-08-29T15:08:58.884766+0000","last_undegraded":"2024-08-29T15:08:58.884766+0000","last_fullsized":"2024-08-29T15:08:58.884766+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:12:03.629717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885010+0000","last_change":"2024-08-29T15:07:20.812313+0000","last_active":"2024-08-29T15:08:58.885010+0000","last_peered":"2024-08-29T15:08:58.885010+0000","last_clean":"2024-08-29T15:08:58.885010+0000","last_became_active":"2024-08-29T15:07:20.811973+0000","last_became_peered":"2024-08-29T15:07:20.811973+0000","last_unstale":"2024-08-29T15:08:58.885010+0000","last_undegraded":"2024-08-29T15:08:58.885010+0000","last_fullsized":"2024-08-29T15:08:58.885010+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:37.283470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877737+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:08:58.877737+0000","last_peered":"2024-08-29T15:08:58.877737+0000","last_clean":"2024-08-29T15:08:58.877737+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:08:58.877737+0000","last_undegraded":"2024-08-29T15:08:58.877737+0000","last_fullsized":"2024-08-29T15:08:58.877737+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872144+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T15:08:57.872144+0000","last_peered":"2024-08-29T15:08:57.872144+0000","last_clean":"2024-08-29T15:08:57.872144+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T15:08:57.872144+0000","last_undegraded":"2024-08-29T15:08:57.872144+0000","last_fullsized":"2024-08-29T15:08:57.872144+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877594+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:08:58.877594+0000","last_peered":"2024-08-29T15:08:58.877594+0000","last_clean":"2024-08-29T15:08:58.877594+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:08:58.877594+0000","last_undegraded":"2024-08-29T15:08:58.877594+0000","last_fullsized":"2024-08-29T15:08:58.877594+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885413+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:08:58.885413+0000","last_peered":"2024-08-29T15:08:58.885413+0000","last_clean":"2024-08-29T15:08:58.885413+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:08:58.885413+0000","last_undegraded":"2024-08-29T15:08:58.885413+0000","last_fullsized":"2024-08-29T15:08:58.885413+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877537+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T15:08:58.877537+0000","last_peered":"2024-08-29T15:08:58.877537+0000","last_clean":"2024-08-29T15:08:58.877537+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T15:08:58.877537+0000","last_undegraded":"2024-08-29T15:08:58.877537+0000","last_fullsized":"2024-08-29T15:08:58.877537+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876085+0000","last_change":"2024-08-29T15:07:52.136104+0000","last_active":"2024-08-29T15:08:57.876085+0000","last_peered":"2024-08-29T15:08:57.876085+0000","last_clean":"2024-08-29T15:08:57.876085+0000","last_became_active":"2024-08-29T15:07:52.135781+0000","last_became_peered":"2024-08-29T15:07:52.135781+0000","last_unstale":"2024-08-29T15:08:57.876085+0000","last_undegraded":"2024-08-29T15:08:57.876085+0000","last_fullsized":"2024-08-29T15:08:57.876085+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:26.381499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888097+0000","last_change":"2024-08-29T15:08:58.887669+0000","last_active":"2024-08-29T15:08:58.888097+0000","last_peered":"2024-08-29T15:08:58.888097+0000","last_clean":"2024-08-29T15:08:58.888097+0000","last_became_active":"2024-08-29T15:08:58.887343+0000","last_became_peered":"2024-08-29T15:08:58.887343+0000","last_unstale":"2024-08-29T15:08:58.888097+0000","last_undegraded":"2024-08-29T15:08:58.888097+0000","last_fullsized":"2024-08-29T15:08:58.888097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:32.280756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878358+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:08:58.878358+0000","last_peered":"2024-08-29T15:08:58.878358+0000","last_clean":"2024-08-29T15:08:58.878358+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:08:58.878358+0000","last_undegraded":"2024-08-29T15:08:58.878358+0000","last_fullsized":"2024-08-29T15:08:58.878358+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889564+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:08:58.889564+0000","last_peered":"2024-08-29T15:08:58.889564+0000","last_clean":"2024-08-29T15:08:58.889564+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:08:58.889564+0000","last_undegraded":"2024-08-29T15:08:58.889564+0000","last_fullsized":"2024-08-29T15:08:58.889564+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886744+0000","last_change":"2024-08-29T15:08:58.886369+0000","last_active":"2024-08-29T15:08:58.886744+0000","last_peered":"2024-08-29T15:08:58.886744+0000","last_clean":"2024-08-29T15:08:58.886744+0000","last_became_active":"2024-08-29T15:08:58.886022+0000","last_became_peered":"2024-08-29T15:08:58.886022+0000","last_unstale":"2024-08-29T15:08:58.886744+0000","last_undegraded":"2024-08-29T15:08:58.886744+0000","last_fullsized":"2024-08-29T15:08:58.886744+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:36.451849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876498+0000","last_change":"2024-08-29T15:07:52.134960+0000","last_active":"2024-08-29T15:08:57.876498+0000","last_peered":"2024-08-29T15:08:57.876498+0000","last_clean":"2024-08-29T15:08:57.876498+0000","last_became_active":"2024-08-29T15:07:52.134608+0000","last_became_peered":"2024-08-29T15:07:52.134608+0000","last_unstale":"2024-08-29T15:08:57.876498+0000","last_undegraded":"2024-08-29T15:08:57.876498+0000","last_fullsized":"2024-08-29T15:08:57.876498+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:42.154504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875880+0000","last_change":"2024-08-29T15:08:25.191304+0000","last_active":"2024-08-29T15:08:58.875880+0000","last_peered":"2024-08-29T15:08:58.875880+0000","last_clean":"2024-08-29T15:08:58.875880+0000","last_became_active":"2024-08-29T15:08:25.191022+0000","last_became_peered":"2024-08-29T15:08:25.191022+0000","last_unstale":"2024-08-29T15:08:58.875880+0000","last_undegraded":"2024-08-29T15:08:58.875880+0000","last_fullsized":"2024-08-29T15:08:58.875880+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:07.608759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.874763+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:08:57.874763+0000","last_peered":"2024-08-29T15:08:57.874763+0000","last_clean":"2024-08-29T15:08:57.874763+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:08:57.874763+0000","last_undegraded":"2024-08-29T15:08:57.874763+0000","last_fullsized":"2024-08-29T15:08:57.874763+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875459+0000","last_change":"2024-08-29T15:07:52.142297+0000","last_active":"2024-08-29T15:08:58.875459+0000","last_peered":"2024-08-29T15:08:58.875459+0000","last_clean":"2024-08-29T15:08:58.875459+0000","last_became_active":"2024-08-29T15:07:52.141836+0000","last_became_peered":"2024-08-29T15:07:52.141836+0000","last_unstale":"2024-08-29T15:08:58.875459+0000","last_undegraded":"2024-08-29T15:08:58.875459+0000","last_fullsized":"2024-08-29T15:08:58.875459+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:58:01.221672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881991+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:08:58.881991+0000","last_peered":"2024-08-29T15:08:58.881991+0000","last_clean":"2024-08-29T15:08:58.881991+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:08:58.881991+0000","last_undegraded":"2024-08-29T15:08:58.881991+0000","last_fullsized":"2024-08-29T15:08:58.881991+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882231+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:08:58.882231+0000","last_peered":"2024-08-29T15:08:58.882231+0000","last_clean":"2024-08-29T15:08:58.882231+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:08:58.882231+0000","last_undegraded":"2024-08-29T15:08:58.882231+0000","last_fullsized":"2024-08-29T15:08:58.882231+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":404,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877283+0000","last_change":"2024-08-29T15:07:20.817595+0000","last_active":"2024-08-29T15:08:58.877283+0000","last_peered":"2024-08-29T15:08:58.877283+0000","last_clean":"2024-08-29T15:08:58.877283+0000","last_became_active":"2024-08-29T15:07:20.817130+0000","last_became_peered":"2024-08-29T15:07:20.817130+0000","last_unstale":"2024-08-29T15:08:58.877283+0000","last_undegraded":"2024-08-29T15:08:58.877283+0000","last_fullsized":"2024-08-29T15:08:58.877283+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:28:33.151864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":595,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874858+0000","last_change":"2024-08-29T15:08:25.185728+0000","last_active":"2024-08-29T15:08:58.874858+0000","last_peered":"2024-08-29T15:08:58.874858+0000","last_clean":"2024-08-29T15:08:58.874858+0000","last_became_active":"2024-08-29T15:08:25.185219+0000","last_became_peered":"2024-08-29T15:08:25.185219+0000","last_unstale":"2024-08-29T15:08:58.874858+0000","last_undegraded":"2024-08-29T15:08:58.874858+0000","last_fullsized":"2024-08-29T15:08:58.874858+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:16:25.013137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890162+0000","last_change":"2024-08-29T15:08:58.890060+0000","last_active":"2024-08-29T15:08:58.890162+0000","last_peered":"2024-08-29T15:08:58.890162+0000","last_clean":"2024-08-29T15:08:58.890162+0000","last_became_active":"2024-08-29T15:08:58.889714+0000","last_became_peered":"2024-08-29T15:08:58.889714+0000","last_unstale":"2024-08-29T15:08:58.890162+0000","last_undegraded":"2024-08-29T15:08:58.890162+0000","last_fullsized":"2024-08-29T15:08:58.890162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:50.659662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876262+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T15:08:57.876262+0000","last_peered":"2024-08-29T15:08:57.876262+0000","last_clean":"2024-08-29T15:08:57.876262+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T15:08:57.876262+0000","last_undegraded":"2024-08-29T15:08:57.876262+0000","last_fullsized":"2024-08-29T15:08:57.876262+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":800,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.879076+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:08:57.879076+0000","last_peered":"2024-08-29T15:08:57.879076+0000","last_clean":"2024-08-29T15:08:57.879076+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:08:57.879076+0000","last_undegraded":"2024-08-29T15:08:57.879076+0000","last_fullsized":"2024-08-29T15:08:57.879076+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889019+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:08:58.889019+0000","last_peered":"2024-08-29T15:08:58.889019+0000","last_clean":"2024-08-29T15:08:58.889019+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:08:58.889019+0000","last_undegraded":"2024-08-29T15:08:58.889019+0000","last_fullsized":"2024-08-29T15:08:58.889019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.879442+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:08:58.879442+0000","last_peered":"2024-08-29T15:08:58.879442+0000","last_clean":"2024-08-29T15:08:58.879442+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:08:58.879442+0000","last_undegraded":"2024-08-29T15:08:58.879442+0000","last_fullsized":"2024-08-29T15:08:58.879442+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":540,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881830+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:08:58.881830+0000","last_peered":"2024-08-29T15:08:58.881830+0000","last_clean":"2024-08-29T15:08:58.881830+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:08:58.881830+0000","last_undegraded":"2024-08-29T15:08:58.881830+0000","last_fullsized":"2024-08-29T15:08:58.881830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873790+0000","last_change":"2024-08-29T15:08:25.188186+0000","last_active":"2024-08-29T15:08:58.873790+0000","last_peered":"2024-08-29T15:08:58.873790+0000","last_clean":"2024-08-29T15:08:58.873790+0000","last_became_active":"2024-08-29T15:08:25.187830+0000","last_became_peered":"2024-08-29T15:08:25.187830+0000","last_unstale":"2024-08-29T15:08:58.873790+0000","last_undegraded":"2024-08-29T15:08:58.873790+0000","last_fullsized":"2024-08-29T15:08:58.873790+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:00:12.780470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873838+0000","last_change":"2024-08-29T15:07:52.139273+0000","last_active":"2024-08-29T15:08:58.873838+0000","last_peered":"2024-08-29T15:08:58.873838+0000","last_clean":"2024-08-29T15:08:58.873838+0000","last_became_active":"2024-08-29T15:07:52.138052+0000","last_became_peered":"2024-08-29T15:07:52.138052+0000","last_unstale":"2024-08-29T15:08:58.873838+0000","last_undegraded":"2024-08-29T15:08:58.873838+0000","last_fullsized":"2024-08-29T15:08:58.873838+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:01.705167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":526,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872946+0000","last_change":"2024-08-29T15:07:52.138323+0000","last_active":"2024-08-29T15:08:57.872946+0000","last_peered":"2024-08-29T15:08:57.872946+0000","last_clean":"2024-08-29T15:08:57.872946+0000","last_became_active":"2024-08-29T15:07:52.137781+0000","last_became_peered":"2024-08-29T15:07:52.137781+0000","last_unstale":"2024-08-29T15:08:57.872946+0000","last_undegraded":"2024-08-29T15:08:57.872946+0000","last_fullsized":"2024-08-29T15:08:57.872946+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:11:27.075260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":579,"num_read_kb":2582,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875043+0000","last_change":"2024-08-29T15:07:20.816485+0000","last_active":"2024-08-29T15:08:57.875043+0000","last_peered":"2024-08-29T15:08:57.875043+0000","last_clean":"2024-08-29T15:08:57.875043+0000","last_became_active":"2024-08-29T15:07:20.816190+0000","last_became_peered":"2024-08-29T15:07:20.816190+0000","last_unstale":"2024-08-29T15:08:57.875043+0000","last_undegraded":"2024-08-29T15:08:57.875043+0000","last_fullsized":"2024-08-29T15:08:57.875043+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:06.731598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884316+0000","last_change":"2024-08-29T15:08:58.883318+0000","last_active":"2024-08-29T15:08:58.884316+0000","last_peered":"2024-08-29T15:08:58.884316+0000","last_clean":"2024-08-29T15:08:58.884316+0000","last_became_active":"2024-08-29T15:08:58.882978+0000","last_became_peered":"2024-08-29T15:08:58.882978+0000","last_unstale":"2024-08-29T15:08:58.884316+0000","last_undegraded":"2024-08-29T15:08:58.884316+0000","last_fullsized":"2024-08-29T15:08:58.884316+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:08.675766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887791+0000","last_change":"2024-08-29T15:08:58.887791+0000","last_active":"2024-08-29T15:08:58.887791+0000","last_peered":"2024-08-29T15:08:58.887791+0000","last_clean":"2024-08-29T15:08:58.887791+0000","last_became_active":"2024-08-29T15:08:58.887472+0000","last_became_peered":"2024-08-29T15:08:58.887472+0000","last_unstale":"2024-08-29T15:08:58.887791+0000","last_undegraded":"2024-08-29T15:08:58.887791+0000","last_fullsized":"2024-08-29T15:08:58.887791+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:35:58.690131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":507,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874346+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:08:58.874346+0000","last_peered":"2024-08-29T15:08:58.874346+0000","last_clean":"2024-08-29T15:08:58.874346+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:08:58.874346+0000","last_undegraded":"2024-08-29T15:08:58.874346+0000","last_fullsized":"2024-08-29T15:08:58.874346+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873788+0000","last_change":"2024-08-29T15:07:52.139543+0000","last_active":"2024-08-29T15:08:58.873788+0000","last_peered":"2024-08-29T15:08:58.873788+0000","last_clean":"2024-08-29T15:08:58.873788+0000","last_became_active":"2024-08-29T15:07:52.138499+0000","last_became_peered":"2024-08-29T15:07:52.138499+0000","last_unstale":"2024-08-29T15:08:58.873788+0000","last_undegraded":"2024-08-29T15:08:58.873788+0000","last_fullsized":"2024-08-29T15:08:58.873788+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:41:05.588954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874122+0000","last_change":"2024-08-29T15:08:25.186307+0000","last_active":"2024-08-29T15:08:58.874122+0000","last_peered":"2024-08-29T15:08:58.874122+0000","last_clean":"2024-08-29T15:08:58.874122+0000","last_became_active":"2024-08-29T15:08:25.185788+0000","last_became_peered":"2024-08-29T15:08:25.185788+0000","last_unstale":"2024-08-29T15:08:58.874122+0000","last_undegraded":"2024-08-29T15:08:58.874122+0000","last_fullsized":"2024-08-29T15:08:58.874122+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:58:08.511525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":497,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877648+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:08:58.877648+0000","last_peered":"2024-08-29T15:08:58.877648+0000","last_clean":"2024-08-29T15:08:58.877648+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:08:58.877648+0000","last_undegraded":"2024-08-29T15:08:58.877648+0000","last_fullsized":"2024-08-29T15:08:58.877648+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878271+0000","last_change":"2024-08-29T15:07:20.813592+0000","last_active":"2024-08-29T15:08:58.878271+0000","last_peered":"2024-08-29T15:08:58.878271+0000","last_clean":"2024-08-29T15:08:58.878271+0000","last_became_active":"2024-08-29T15:07:20.812970+0000","last_became_peered":"2024-08-29T15:07:20.812970+0000","last_unstale":"2024-08-29T15:08:58.878271+0000","last_undegraded":"2024-08-29T15:08:58.878271+0000","last_fullsized":"2024-08-29T15:08:58.878271+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:54.850894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888546+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:08:58.888546+0000","last_peered":"2024-08-29T15:08:58.888546+0000","last_clean":"2024-08-29T15:08:58.888546+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:08:58.888546+0000","last_undegraded":"2024-08-29T15:08:58.888546+0000","last_fullsized":"2024-08-29T15:08:58.888546+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":598,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874127+0000","last_change":"2024-08-29T15:07:52.139196+0000","last_active":"2024-08-29T15:08:58.874127+0000","last_peered":"2024-08-29T15:08:58.874127+0000","last_clean":"2024-08-29T15:08:58.874127+0000","last_became_active":"2024-08-29T15:07:52.138669+0000","last_became_peered":"2024-08-29T15:07:52.138669+0000","last_unstale":"2024-08-29T15:08:58.874127+0000","last_undegraded":"2024-08-29T15:08:58.874127+0000","last_fullsized":"2024-08-29T15:08:58.874127+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:43:54.949715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888619+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:08:58.888619+0000","last_peered":"2024-08-29T15:08:58.888619+0000","last_clean":"2024-08-29T15:08:58.888619+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:08:58.888619+0000","last_undegraded":"2024-08-29T15:08:58.888619+0000","last_fullsized":"2024-08-29T15:08:58.888619+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878638+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:08:58.878638+0000","last_peered":"2024-08-29T15:08:58.878638+0000","last_clean":"2024-08-29T15:08:58.878638+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:08:58.878638+0000","last_undegraded":"2024-08-29T15:08:58.878638+0000","last_fullsized":"2024-08-29T15:08:58.878638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":337,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885216+0000","last_change":"2024-08-29T15:08:25.186819+0000","last_active":"2024-08-29T15:08:58.885216+0000","last_peered":"2024-08-29T15:08:58.885216+0000","last_clean":"2024-08-29T15:08:58.885216+0000","last_became_active":"2024-08-29T15:08:25.186401+0000","last_became_peered":"2024-08-29T15:08:25.186401+0000","last_unstale":"2024-08-29T15:08:58.885216+0000","last_undegraded":"2024-08-29T15:08:58.885216+0000","last_fullsized":"2024-08-29T15:08:58.885216+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:45.101824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.873204+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T15:08:57.873204+0000","last_peered":"2024-08-29T15:08:57.873204+0000","last_clean":"2024-08-29T15:08:57.873204+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T15:08:57.873204+0000","last_undegraded":"2024-08-29T15:08:57.873204+0000","last_fullsized":"2024-08-29T15:08:57.873204+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875638+0000","last_change":"2024-08-29T15:08:25.185452+0000","last_active":"2024-08-29T15:08:58.875638+0000","last_peered":"2024-08-29T15:08:58.875638+0000","last_clean":"2024-08-29T15:08:58.875638+0000","last_became_active":"2024-08-29T15:08:25.185095+0000","last_became_peered":"2024-08-29T15:08:25.185095+0000","last_unstale":"2024-08-29T15:08:58.875638+0000","last_undegraded":"2024-08-29T15:08:58.875638+0000","last_fullsized":"2024-08-29T15:08:58.875638+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:19:03.238217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875238+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T15:08:58.875238+0000","last_peered":"2024-08-29T15:08:58.875238+0000","last_clean":"2024-08-29T15:08:58.875238+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T15:08:58.875238+0000","last_undegraded":"2024-08-29T15:08:58.875238+0000","last_fullsized":"2024-08-29T15:08:58.875238+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873562+0000","last_change":"2024-08-29T15:08:25.186368+0000","last_active":"2024-08-29T15:08:58.873562+0000","last_peered":"2024-08-29T15:08:58.873562+0000","last_clean":"2024-08-29T15:08:58.873562+0000","last_became_active":"2024-08-29T15:08:25.185912+0000","last_became_peered":"2024-08-29T15:08:25.185912+0000","last_unstale":"2024-08-29T15:08:58.873562+0000","last_undegraded":"2024-08-29T15:08:58.873562+0000","last_fullsized":"2024-08-29T15:08:58.873562+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:23.124995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886830+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:08:58.886830+0000","last_peered":"2024-08-29T15:08:58.886830+0000","last_clean":"2024-08-29T15:08:58.886830+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:08:58.886830+0000","last_undegraded":"2024-08-29T15:08:58.886830+0000","last_fullsized":"2024-08-29T15:08:58.886830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872257+0000","last_change":"2024-08-29T15:07:52.136541+0000","last_active":"2024-08-29T15:08:57.872257+0000","last_peered":"2024-08-29T15:08:57.872257+0000","last_clean":"2024-08-29T15:08:57.872257+0000","last_became_active":"2024-08-29T15:07:52.136202+0000","last_became_peered":"2024-08-29T15:07:52.136202+0000","last_unstale":"2024-08-29T15:08:57.872257+0000","last_undegraded":"2024-08-29T15:08:57.872257+0000","last_fullsized":"2024-08-29T15:08:57.872257+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:01.747963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888935+0000","last_change":"2024-08-29T15:08:58.888848+0000","last_active":"2024-08-29T15:08:58.888935+0000","last_peered":"2024-08-29T15:08:58.888935+0000","last_clean":"2024-08-29T15:08:58.888935+0000","last_became_active":"2024-08-29T15:08:58.886702+0000","last_became_peered":"2024-08-29T15:08:58.886702+0000","last_unstale":"2024-08-29T15:08:58.888935+0000","last_undegraded":"2024-08-29T15:08:58.888935+0000","last_fullsized":"2024-08-29T15:08:58.888935+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:56.189439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887636+0000","last_change":"2024-08-29T15:08:58.887151+0000","last_active":"2024-08-29T15:08:58.887636+0000","last_peered":"2024-08-29T15:08:58.887636+0000","last_clean":"2024-08-29T15:08:58.887636+0000","last_became_active":"2024-08-29T15:08:58.886918+0000","last_became_peered":"2024-08-29T15:08:58.886918+0000","last_unstale":"2024-08-29T15:08:58.887636+0000","last_undegraded":"2024-08-29T15:08:58.887636+0000","last_fullsized":"2024-08-29T15:08:58.887636+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:37.557892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889787+0000","last_change":"2024-08-29T15:08:58.889700+0000","last_active":"2024-08-29T15:08:58.889787+0000","last_peered":"2024-08-29T15:08:58.889787+0000","last_clean":"2024-08-29T15:08:58.889787+0000","last_became_active":"2024-08-29T15:08:58.887521+0000","last_became_peered":"2024-08-29T15:08:58.887521+0000","last_unstale":"2024-08-29T15:08:58.889787+0000","last_undegraded":"2024-08-29T15:08:58.889787+0000","last_fullsized":"2024-08-29T15:08:58.889787+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:32.997619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.877208+0000","last_change":"2024-08-29T15:07:52.135092+0000","last_active":"2024-08-29T15:08:57.877208+0000","last_peered":"2024-08-29T15:08:57.877208+0000","last_clean":"2024-08-29T15:08:57.877208+0000","last_became_active":"2024-08-29T15:07:52.134625+0000","last_became_peered":"2024-08-29T15:07:52.134625+0000","last_unstale":"2024-08-29T15:08:57.877208+0000","last_undegraded":"2024-08-29T15:08:57.877208+0000","last_fullsized":"2024-08-29T15:08:57.877208+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:22:35.250433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885989+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:08:58.885989+0000","last_peered":"2024-08-29T15:08:58.885989+0000","last_clean":"2024-08-29T15:08:58.885989+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:08:58.885989+0000","last_undegraded":"2024-08-29T15:08:58.885989+0000","last_fullsized":"2024-08-29T15:08:58.885989+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889927+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:08:58.889927+0000","last_peered":"2024-08-29T15:08:58.889927+0000","last_clean":"2024-08-29T15:08:58.889927+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:08:58.889927+0000","last_undegraded":"2024-08-29T15:08:58.889927+0000","last_fullsized":"2024-08-29T15:08:58.889927+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.883653+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:08:58.883653+0000","last_peered":"2024-08-29T15:08:58.883653+0000","last_clean":"2024-08-29T15:08:58.883653+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:08:58.883653+0000","last_undegraded":"2024-08-29T15:08:58.883653+0000","last_fullsized":"2024-08-29T15:08:58.883653+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888863+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:08:58.888863+0000","last_peered":"2024-08-29T15:08:58.888863+0000","last_clean":"2024-08-29T15:08:58.888863+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:08:58.888863+0000","last_undegraded":"2024-08-29T15:08:58.888863+0000","last_fullsized":"2024-08-29T15:08:58.888863+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.873347+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:08:57.873347+0000","last_peered":"2024-08-29T15:08:57.873347+0000","last_clean":"2024-08-29T15:08:57.873347+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:08:57.873347+0000","last_undegraded":"2024-08-29T15:08:57.873347+0000","last_fullsized":"2024-08-29T15:08:57.873347+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884205+0000","last_change":"2024-08-29T15:08:58.884121+0000","last_active":"2024-08-29T15:08:58.884205+0000","last_peered":"2024-08-29T15:08:58.884205+0000","last_clean":"2024-08-29T15:08:58.884205+0000","last_became_active":"2024-08-29T15:08:58.883669+0000","last_became_peered":"2024-08-29T15:08:58.883669+0000","last_unstale":"2024-08-29T15:08:58.884205+0000","last_undegraded":"2024-08-29T15:08:58.884205+0000","last_fullsized":"2024-08-29T15:08:58.884205+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:32.165590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878176+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:08:58.878176+0000","last_peered":"2024-08-29T15:08:58.878176+0000","last_clean":"2024-08-29T15:08:58.878176+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:08:58.878176+0000","last_undegraded":"2024-08-29T15:08:58.878176+0000","last_fullsized":"2024-08-29T15:08:58.878176+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884996+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:08:58.884996+0000","last_peered":"2024-08-29T15:08:58.884996+0000","last_clean":"2024-08-29T15:08:58.884996+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:08:58.884996+0000","last_undegraded":"2024-08-29T15:08:58.884996+0000","last_fullsized":"2024-08-29T15:08:58.884996+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872326+0000","last_change":"2024-08-29T15:07:52.134597+0000","last_active":"2024-08-29T15:08:57.872326+0000","last_peered":"2024-08-29T15:08:57.872326+0000","last_clean":"2024-08-29T15:08:57.872326+0000","last_became_active":"2024-08-29T15:07:52.133893+0000","last_became_peered":"2024-08-29T15:07:52.133893+0000","last_unstale":"2024-08-29T15:08:57.872326+0000","last_undegraded":"2024-08-29T15:08:57.872326+0000","last_fullsized":"2024-08-29T15:08:57.872326+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:14.240866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884919+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:08:58.884919+0000","last_peered":"2024-08-29T15:08:58.884919+0000","last_clean":"2024-08-29T15:08:58.884919+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:08:58.884919+0000","last_undegraded":"2024-08-29T15:08:58.884919+0000","last_fullsized":"2024-08-29T15:08:58.884919+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875599+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T15:08:57.875599+0000","last_peered":"2024-08-29T15:08:57.875599+0000","last_clean":"2024-08-29T15:08:57.875599+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T15:08:57.875599+0000","last_undegraded":"2024-08-29T15:08:57.875599+0000","last_fullsized":"2024-08-29T15:08:57.875599+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888235+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:08:58.888235+0000","last_peered":"2024-08-29T15:08:58.888235+0000","last_clean":"2024-08-29T15:08:58.888235+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:08:58.888235+0000","last_undegraded":"2024-08-29T15:08:58.888235+0000","last_fullsized":"2024-08-29T15:08:58.888235+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888750+0000","last_change":"2024-08-29T15:08:58.888653+0000","last_active":"2024-08-29T15:08:58.888750+0000","last_peered":"2024-08-29T15:08:58.888750+0000","last_clean":"2024-08-29T15:08:58.888750+0000","last_became_active":"2024-08-29T15:08:58.888133+0000","last_became_peered":"2024-08-29T15:08:58.888133+0000","last_unstale":"2024-08-29T15:08:58.888750+0000","last_undegraded":"2024-08-29T15:08:58.888750+0000","last_fullsized":"2024-08-29T15:08:58.888750+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:24.144974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872210+0000","last_change":"2024-08-29T15:07:52.135649+0000","last_active":"2024-08-29T15:08:57.872210+0000","last_peered":"2024-08-29T15:08:57.872210+0000","last_clean":"2024-08-29T15:08:57.872210+0000","last_became_active":"2024-08-29T15:07:52.135327+0000","last_became_peered":"2024-08-29T15:07:52.135327+0000","last_unstale":"2024-08-29T15:08:57.872210+0000","last_undegraded":"2024-08-29T15:08:57.872210+0000","last_fullsized":"2024-08-29T15:08:57.872210+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:05.402462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878257+0000","last_change":"2024-08-29T15:08:25.187422+0000","last_active":"2024-08-29T15:08:58.878257+0000","last_peered":"2024-08-29T15:08:58.878257+0000","last_clean":"2024-08-29T15:08:58.878257+0000","last_became_active":"2024-08-29T15:08:25.187062+0000","last_became_peered":"2024-08-29T15:08:25.187062+0000","last_unstale":"2024-08-29T15:08:58.878257+0000","last_undegraded":"2024-08-29T15:08:58.878257+0000","last_fullsized":"2024-08-29T15:08:58.878257+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:18:20.745348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234184,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37004,"kb_used_data":5888,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708148,"statfs":{"total":95995035648,"available":95957143552,"internally_reserved":0,"allocated":6029312,"data_stored":5417499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495820,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28808,"kb_used_data":1780,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716344,"statfs":{"total":95995035648,"available":95965536256,"internally_reserved":0,"allocated":1822720,"data_stored":1222795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757457,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1220608,"data_stored":628195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019094,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36420,"kb_used_data":5300,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708732,"statfs":{"total":95995035648,"available":95957741568,"internally_reserved":0,"allocated":5427200,"data_stored":4826788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435279,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1188,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1216512,"data_stored":627957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":28110089},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598805,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1180,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1208320,"data_stored":627233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":28172623},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762332,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28728,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716424,"statfs":{"total":95995035648,"available":95965618176,"internally_reserved":0,"allocated":1216512,"data_stored":627886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":28172642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925858,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33396,"kb_used_data":5860,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711756,"statfs":{"total":95995035648,"available":95960838144,"internally_reserved":0,"allocated":6000640,"data_stored":5411702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":28172431},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:10:04.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:10:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:03 smithi171 ceph-mon[90031]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:03 smithi171 ceph-mon[90031]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:10:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:05.000 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:06.100 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[98970]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[93921]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.101 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.102 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:10:06.102 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:10:06.102 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:06 smithi171 ceph-mon[90031]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:10:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:10:06.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:06.833 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:10:06.834 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:10:07.495 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":282,"stamp":"2024-08-29T15:10:06.333195+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6390,"num_read_kb":13845,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":249952,"kb_used_data":23576,"kb_used_omap":34,"kb_used_meta":226205,"kb_avail":749711264,"statfs":{"total":767960285184,"available":767704334336,"internally_reserved":0,"allocated":24141824,"data_stored":19390055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34969,"internal_metadata":231634791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002688"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882496+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:08:58.882496+0000","last_peered":"2024-08-29T15:08:58.882496+0000","last_clean":"2024-08-29T15:08:58.882496+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:08:58.882496+0000","last_undegraded":"2024-08-29T15:08:58.882496+0000","last_fullsized":"2024-08-29T15:08:58.882496+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877466+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:08:58.877466+0000","last_peered":"2024-08-29T15:08:58.877466+0000","last_clean":"2024-08-29T15:08:58.877466+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:08:58.877466+0000","last_undegraded":"2024-08-29T15:08:58.877466+0000","last_fullsized":"2024-08-29T15:08:58.877466+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887905+0000","last_change":"2024-08-29T15:08:58.887518+0000","last_active":"2024-08-29T15:08:58.887905+0000","last_peered":"2024-08-29T15:08:58.887905+0000","last_clean":"2024-08-29T15:08:58.887905+0000","last_became_active":"2024-08-29T15:08:58.887049+0000","last_became_peered":"2024-08-29T15:08:58.887049+0000","last_unstale":"2024-08-29T15:08:58.887905+0000","last_undegraded":"2024-08-29T15:08:58.887905+0000","last_fullsized":"2024-08-29T15:08:58.887905+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:23:07.638407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890091+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:08:58.890091+0000","last_peered":"2024-08-29T15:08:58.890091+0000","last_clean":"2024-08-29T15:08:58.890091+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:08:58.890091+0000","last_undegraded":"2024-08-29T15:08:58.890091+0000","last_fullsized":"2024-08-29T15:08:58.890091+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889545+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:08:58.889545+0000","last_peered":"2024-08-29T15:08:58.889545+0000","last_clean":"2024-08-29T15:08:58.889545+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:08:58.889545+0000","last_undegraded":"2024-08-29T15:08:58.889545+0000","last_fullsized":"2024-08-29T15:08:58.889545+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877550+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T15:08:58.877550+0000","last_peered":"2024-08-29T15:08:58.877550+0000","last_clean":"2024-08-29T15:08:58.877550+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T15:08:58.877550+0000","last_undegraded":"2024-08-29T15:08:58.877550+0000","last_fullsized":"2024-08-29T15:08:58.877550+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884766+0000","last_change":"2024-08-29T15:08:58.884675+0000","last_active":"2024-08-29T15:08:58.884766+0000","last_peered":"2024-08-29T15:08:58.884766+0000","last_clean":"2024-08-29T15:08:58.884766+0000","last_became_active":"2024-08-29T15:08:58.884007+0000","last_became_peered":"2024-08-29T15:08:58.884007+0000","last_unstale":"2024-08-29T15:08:58.884766+0000","last_undegraded":"2024-08-29T15:08:58.884766+0000","last_fullsized":"2024-08-29T15:08:58.884766+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:12:03.629717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":81,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885010+0000","last_change":"2024-08-29T15:07:20.812313+0000","last_active":"2024-08-29T15:08:58.885010+0000","last_peered":"2024-08-29T15:08:58.885010+0000","last_clean":"2024-08-29T15:08:58.885010+0000","last_became_active":"2024-08-29T15:07:20.811973+0000","last_became_peered":"2024-08-29T15:07:20.811973+0000","last_unstale":"2024-08-29T15:08:58.885010+0000","last_undegraded":"2024-08-29T15:08:58.885010+0000","last_fullsized":"2024-08-29T15:08:58.885010+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:37.283470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877737+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:08:58.877737+0000","last_peered":"2024-08-29T15:08:58.877737+0000","last_clean":"2024-08-29T15:08:58.877737+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:08:58.877737+0000","last_undegraded":"2024-08-29T15:08:58.877737+0000","last_fullsized":"2024-08-29T15:08:58.877737+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872144+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T15:08:57.872144+0000","last_peered":"2024-08-29T15:08:57.872144+0000","last_clean":"2024-08-29T15:08:57.872144+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T15:08:57.872144+0000","last_undegraded":"2024-08-29T15:08:57.872144+0000","last_fullsized":"2024-08-29T15:08:57.872144+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877594+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:08:58.877594+0000","last_peered":"2024-08-29T15:08:58.877594+0000","last_clean":"2024-08-29T15:08:58.877594+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:08:58.877594+0000","last_undegraded":"2024-08-29T15:08:58.877594+0000","last_fullsized":"2024-08-29T15:08:58.877594+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885413+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:08:58.885413+0000","last_peered":"2024-08-29T15:08:58.885413+0000","last_clean":"2024-08-29T15:08:58.885413+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:08:58.885413+0000","last_undegraded":"2024-08-29T15:08:58.885413+0000","last_fullsized":"2024-08-29T15:08:58.885413+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877537+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T15:08:58.877537+0000","last_peered":"2024-08-29T15:08:58.877537+0000","last_clean":"2024-08-29T15:08:58.877537+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T15:08:58.877537+0000","last_undegraded":"2024-08-29T15:08:58.877537+0000","last_fullsized":"2024-08-29T15:08:58.877537+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876085+0000","last_change":"2024-08-29T15:07:52.136104+0000","last_active":"2024-08-29T15:08:57.876085+0000","last_peered":"2024-08-29T15:08:57.876085+0000","last_clean":"2024-08-29T15:08:57.876085+0000","last_became_active":"2024-08-29T15:07:52.135781+0000","last_became_peered":"2024-08-29T15:07:52.135781+0000","last_unstale":"2024-08-29T15:08:57.876085+0000","last_undegraded":"2024-08-29T15:08:57.876085+0000","last_fullsized":"2024-08-29T15:08:57.876085+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:26.381499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888097+0000","last_change":"2024-08-29T15:08:58.887669+0000","last_active":"2024-08-29T15:08:58.888097+0000","last_peered":"2024-08-29T15:08:58.888097+0000","last_clean":"2024-08-29T15:08:58.888097+0000","last_became_active":"2024-08-29T15:08:58.887343+0000","last_became_peered":"2024-08-29T15:08:58.887343+0000","last_unstale":"2024-08-29T15:08:58.888097+0000","last_undegraded":"2024-08-29T15:08:58.888097+0000","last_fullsized":"2024-08-29T15:08:58.888097+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:32.280756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878358+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:08:58.878358+0000","last_peered":"2024-08-29T15:08:58.878358+0000","last_clean":"2024-08-29T15:08:58.878358+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:08:58.878358+0000","last_undegraded":"2024-08-29T15:08:58.878358+0000","last_fullsized":"2024-08-29T15:08:58.878358+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889564+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:08:58.889564+0000","last_peered":"2024-08-29T15:08:58.889564+0000","last_clean":"2024-08-29T15:08:58.889564+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:08:58.889564+0000","last_undegraded":"2024-08-29T15:08:58.889564+0000","last_fullsized":"2024-08-29T15:08:58.889564+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886744+0000","last_change":"2024-08-29T15:08:58.886369+0000","last_active":"2024-08-29T15:08:58.886744+0000","last_peered":"2024-08-29T15:08:58.886744+0000","last_clean":"2024-08-29T15:08:58.886744+0000","last_became_active":"2024-08-29T15:08:58.886022+0000","last_became_peered":"2024-08-29T15:08:58.886022+0000","last_unstale":"2024-08-29T15:08:58.886744+0000","last_undegraded":"2024-08-29T15:08:58.886744+0000","last_fullsized":"2024-08-29T15:08:58.886744+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:36.451849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876498+0000","last_change":"2024-08-29T15:07:52.134960+0000","last_active":"2024-08-29T15:08:57.876498+0000","last_peered":"2024-08-29T15:08:57.876498+0000","last_clean":"2024-08-29T15:08:57.876498+0000","last_became_active":"2024-08-29T15:07:52.134608+0000","last_became_peered":"2024-08-29T15:07:52.134608+0000","last_unstale":"2024-08-29T15:08:57.876498+0000","last_undegraded":"2024-08-29T15:08:57.876498+0000","last_fullsized":"2024-08-29T15:08:57.876498+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:42.154504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875880+0000","last_change":"2024-08-29T15:08:25.191304+0000","last_active":"2024-08-29T15:08:58.875880+0000","last_peered":"2024-08-29T15:08:58.875880+0000","last_clean":"2024-08-29T15:08:58.875880+0000","last_became_active":"2024-08-29T15:08:25.191022+0000","last_became_peered":"2024-08-29T15:08:25.191022+0000","last_unstale":"2024-08-29T15:08:58.875880+0000","last_undegraded":"2024-08-29T15:08:58.875880+0000","last_fullsized":"2024-08-29T15:08:58.875880+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:07.608759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.874763+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:08:57.874763+0000","last_peered":"2024-08-29T15:08:57.874763+0000","last_clean":"2024-08-29T15:08:57.874763+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:08:57.874763+0000","last_undegraded":"2024-08-29T15:08:57.874763+0000","last_fullsized":"2024-08-29T15:08:57.874763+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875459+0000","last_change":"2024-08-29T15:07:52.142297+0000","last_active":"2024-08-29T15:08:58.875459+0000","last_peered":"2024-08-29T15:08:58.875459+0000","last_clean":"2024-08-29T15:08:58.875459+0000","last_became_active":"2024-08-29T15:07:52.141836+0000","last_became_peered":"2024-08-29T15:07:52.141836+0000","last_unstale":"2024-08-29T15:08:58.875459+0000","last_undegraded":"2024-08-29T15:08:58.875459+0000","last_fullsized":"2024-08-29T15:08:58.875459+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:58:01.221672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881991+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:08:58.881991+0000","last_peered":"2024-08-29T15:08:58.881991+0000","last_clean":"2024-08-29T15:08:58.881991+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:08:58.881991+0000","last_undegraded":"2024-08-29T15:08:58.881991+0000","last_fullsized":"2024-08-29T15:08:58.881991+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.882511+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:08:58.882511+0000","last_peered":"2024-08-29T15:08:58.882511+0000","last_clean":"2024-08-29T15:08:58.882511+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:08:58.882511+0000","last_undegraded":"2024-08-29T15:08:58.882511+0000","last_fullsized":"2024-08-29T15:08:58.882511+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":404,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877283+0000","last_change":"2024-08-29T15:07:20.817595+0000","last_active":"2024-08-29T15:08:58.877283+0000","last_peered":"2024-08-29T15:08:58.877283+0000","last_clean":"2024-08-29T15:08:58.877283+0000","last_became_active":"2024-08-29T15:07:20.817130+0000","last_became_peered":"2024-08-29T15:07:20.817130+0000","last_unstale":"2024-08-29T15:08:58.877283+0000","last_undegraded":"2024-08-29T15:08:58.877283+0000","last_fullsized":"2024-08-29T15:08:58.877283+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:28:33.151864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":595,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874858+0000","last_change":"2024-08-29T15:08:25.185728+0000","last_active":"2024-08-29T15:08:58.874858+0000","last_peered":"2024-08-29T15:08:58.874858+0000","last_clean":"2024-08-29T15:08:58.874858+0000","last_became_active":"2024-08-29T15:08:25.185219+0000","last_became_peered":"2024-08-29T15:08:25.185219+0000","last_unstale":"2024-08-29T15:08:58.874858+0000","last_undegraded":"2024-08-29T15:08:58.874858+0000","last_fullsized":"2024-08-29T15:08:58.874858+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:16:25.013137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.890162+0000","last_change":"2024-08-29T15:08:58.890060+0000","last_active":"2024-08-29T15:08:58.890162+0000","last_peered":"2024-08-29T15:08:58.890162+0000","last_clean":"2024-08-29T15:08:58.890162+0000","last_became_active":"2024-08-29T15:08:58.889714+0000","last_became_peered":"2024-08-29T15:08:58.889714+0000","last_unstale":"2024-08-29T15:08:58.890162+0000","last_undegraded":"2024-08-29T15:08:58.890162+0000","last_fullsized":"2024-08-29T15:08:58.890162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:50.659662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.876262+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T15:08:57.876262+0000","last_peered":"2024-08-29T15:08:57.876262+0000","last_clean":"2024-08-29T15:08:57.876262+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T15:08:57.876262+0000","last_undegraded":"2024-08-29T15:08:57.876262+0000","last_fullsized":"2024-08-29T15:08:57.876262+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":800,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.879076+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:08:57.879076+0000","last_peered":"2024-08-29T15:08:57.879076+0000","last_clean":"2024-08-29T15:08:57.879076+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:08:57.879076+0000","last_undegraded":"2024-08-29T15:08:57.879076+0000","last_fullsized":"2024-08-29T15:08:57.879076+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889019+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:08:58.889019+0000","last_peered":"2024-08-29T15:08:58.889019+0000","last_clean":"2024-08-29T15:08:58.889019+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:08:58.889019+0000","last_undegraded":"2024-08-29T15:08:58.889019+0000","last_fullsized":"2024-08-29T15:08:58.889019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.879442+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:08:58.879442+0000","last_peered":"2024-08-29T15:08:58.879442+0000","last_clean":"2024-08-29T15:08:58.879442+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:08:58.879442+0000","last_undegraded":"2024-08-29T15:08:58.879442+0000","last_fullsized":"2024-08-29T15:08:58.879442+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":540,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.881830+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:08:58.881830+0000","last_peered":"2024-08-29T15:08:58.881830+0000","last_clean":"2024-08-29T15:08:58.881830+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:08:58.881830+0000","last_undegraded":"2024-08-29T15:08:58.881830+0000","last_fullsized":"2024-08-29T15:08:58.881830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873790+0000","last_change":"2024-08-29T15:08:25.188186+0000","last_active":"2024-08-29T15:08:58.873790+0000","last_peered":"2024-08-29T15:08:58.873790+0000","last_clean":"2024-08-29T15:08:58.873790+0000","last_became_active":"2024-08-29T15:08:25.187830+0000","last_became_peered":"2024-08-29T15:08:25.187830+0000","last_unstale":"2024-08-29T15:08:58.873790+0000","last_undegraded":"2024-08-29T15:08:58.873790+0000","last_fullsized":"2024-08-29T15:08:58.873790+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:00:12.780470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873838+0000","last_change":"2024-08-29T15:07:52.139273+0000","last_active":"2024-08-29T15:08:58.873838+0000","last_peered":"2024-08-29T15:08:58.873838+0000","last_clean":"2024-08-29T15:08:58.873838+0000","last_became_active":"2024-08-29T15:07:52.138052+0000","last_became_peered":"2024-08-29T15:07:52.138052+0000","last_unstale":"2024-08-29T15:08:58.873838+0000","last_undegraded":"2024-08-29T15:08:58.873838+0000","last_fullsized":"2024-08-29T15:08:58.873838+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:01.705167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":526,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872946+0000","last_change":"2024-08-29T15:07:52.138323+0000","last_active":"2024-08-29T15:08:57.872946+0000","last_peered":"2024-08-29T15:08:57.872946+0000","last_clean":"2024-08-29T15:08:57.872946+0000","last_became_active":"2024-08-29T15:07:52.137781+0000","last_became_peered":"2024-08-29T15:07:52.137781+0000","last_unstale":"2024-08-29T15:08:57.872946+0000","last_undegraded":"2024-08-29T15:08:57.872946+0000","last_fullsized":"2024-08-29T15:08:57.872946+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:11:27.075260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":579,"num_read_kb":2582,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875043+0000","last_change":"2024-08-29T15:07:20.816485+0000","last_active":"2024-08-29T15:08:57.875043+0000","last_peered":"2024-08-29T15:08:57.875043+0000","last_clean":"2024-08-29T15:08:57.875043+0000","last_became_active":"2024-08-29T15:07:20.816190+0000","last_became_peered":"2024-08-29T15:07:20.816190+0000","last_unstale":"2024-08-29T15:08:57.875043+0000","last_undegraded":"2024-08-29T15:08:57.875043+0000","last_fullsized":"2024-08-29T15:08:57.875043+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:06.731598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884316+0000","last_change":"2024-08-29T15:08:58.883318+0000","last_active":"2024-08-29T15:08:58.884316+0000","last_peered":"2024-08-29T15:08:58.884316+0000","last_clean":"2024-08-29T15:08:58.884316+0000","last_became_active":"2024-08-29T15:08:58.882978+0000","last_became_peered":"2024-08-29T15:08:58.882978+0000","last_unstale":"2024-08-29T15:08:58.884316+0000","last_undegraded":"2024-08-29T15:08:58.884316+0000","last_fullsized":"2024-08-29T15:08:58.884316+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:08.675766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887791+0000","last_change":"2024-08-29T15:08:58.887791+0000","last_active":"2024-08-29T15:08:58.887791+0000","last_peered":"2024-08-29T15:08:58.887791+0000","last_clean":"2024-08-29T15:08:58.887791+0000","last_became_active":"2024-08-29T15:08:58.887472+0000","last_became_peered":"2024-08-29T15:08:58.887472+0000","last_unstale":"2024-08-29T15:08:58.887791+0000","last_undegraded":"2024-08-29T15:08:58.887791+0000","last_fullsized":"2024-08-29T15:08:58.887791+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:35:58.690131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":507,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874346+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:08:58.874346+0000","last_peered":"2024-08-29T15:08:58.874346+0000","last_clean":"2024-08-29T15:08:58.874346+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:08:58.874346+0000","last_undegraded":"2024-08-29T15:08:58.874346+0000","last_fullsized":"2024-08-29T15:08:58.874346+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":45,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873788+0000","last_change":"2024-08-29T15:07:52.139543+0000","last_active":"2024-08-29T15:08:58.873788+0000","last_peered":"2024-08-29T15:08:58.873788+0000","last_clean":"2024-08-29T15:08:58.873788+0000","last_became_active":"2024-08-29T15:07:52.138499+0000","last_became_peered":"2024-08-29T15:07:52.138499+0000","last_unstale":"2024-08-29T15:08:58.873788+0000","last_undegraded":"2024-08-29T15:08:58.873788+0000","last_fullsized":"2024-08-29T15:08:58.873788+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:41:05.588954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874122+0000","last_change":"2024-08-29T15:08:25.186307+0000","last_active":"2024-08-29T15:08:58.874122+0000","last_peered":"2024-08-29T15:08:58.874122+0000","last_clean":"2024-08-29T15:08:58.874122+0000","last_became_active":"2024-08-29T15:08:25.185788+0000","last_became_peered":"2024-08-29T15:08:25.185788+0000","last_unstale":"2024-08-29T15:08:58.874122+0000","last_undegraded":"2024-08-29T15:08:58.874122+0000","last_fullsized":"2024-08-29T15:08:58.874122+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:58:08.511525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":497,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.877648+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:08:58.877648+0000","last_peered":"2024-08-29T15:08:58.877648+0000","last_clean":"2024-08-29T15:08:58.877648+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:08:58.877648+0000","last_undegraded":"2024-08-29T15:08:58.877648+0000","last_fullsized":"2024-08-29T15:08:58.877648+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878271+0000","last_change":"2024-08-29T15:07:20.813592+0000","last_active":"2024-08-29T15:08:58.878271+0000","last_peered":"2024-08-29T15:08:58.878271+0000","last_clean":"2024-08-29T15:08:58.878271+0000","last_became_active":"2024-08-29T15:07:20.812970+0000","last_became_peered":"2024-08-29T15:07:20.812970+0000","last_unstale":"2024-08-29T15:08:58.878271+0000","last_undegraded":"2024-08-29T15:08:58.878271+0000","last_fullsized":"2024-08-29T15:08:58.878271+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:54.850894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888546+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:08:58.888546+0000","last_peered":"2024-08-29T15:08:58.888546+0000","last_clean":"2024-08-29T15:08:58.888546+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:08:58.888546+0000","last_undegraded":"2024-08-29T15:08:58.888546+0000","last_fullsized":"2024-08-29T15:08:58.888546+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":598,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874127+0000","last_change":"2024-08-29T15:07:52.139196+0000","last_active":"2024-08-29T15:08:58.874127+0000","last_peered":"2024-08-29T15:08:58.874127+0000","last_clean":"2024-08-29T15:08:58.874127+0000","last_became_active":"2024-08-29T15:07:52.138669+0000","last_became_peered":"2024-08-29T15:07:52.138669+0000","last_unstale":"2024-08-29T15:08:58.874127+0000","last_undegraded":"2024-08-29T15:08:58.874127+0000","last_fullsized":"2024-08-29T15:08:58.874127+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:43:54.949715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888619+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:08:58.888619+0000","last_peered":"2024-08-29T15:08:58.888619+0000","last_clean":"2024-08-29T15:08:58.888619+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:08:58.888619+0000","last_undegraded":"2024-08-29T15:08:58.888619+0000","last_fullsized":"2024-08-29T15:08:58.888619+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878638+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:08:58.878638+0000","last_peered":"2024-08-29T15:08:58.878638+0000","last_clean":"2024-08-29T15:08:58.878638+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:08:58.878638+0000","last_undegraded":"2024-08-29T15:08:58.878638+0000","last_fullsized":"2024-08-29T15:08:58.878638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":337,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885216+0000","last_change":"2024-08-29T15:08:25.186819+0000","last_active":"2024-08-29T15:08:58.885216+0000","last_peered":"2024-08-29T15:08:58.885216+0000","last_clean":"2024-08-29T15:08:58.885216+0000","last_became_active":"2024-08-29T15:08:25.186401+0000","last_became_peered":"2024-08-29T15:08:25.186401+0000","last_unstale":"2024-08-29T15:08:58.885216+0000","last_undegraded":"2024-08-29T15:08:58.885216+0000","last_fullsized":"2024-08-29T15:08:58.885216+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:45.101824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875687+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T15:08:58.875687+0000","last_peered":"2024-08-29T15:08:58.875687+0000","last_clean":"2024-08-29T15:08:58.875687+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T15:08:58.875687+0000","last_undegraded":"2024-08-29T15:08:58.875687+0000","last_fullsized":"2024-08-29T15:08:58.875687+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875638+0000","last_change":"2024-08-29T15:08:25.185452+0000","last_active":"2024-08-29T15:08:58.875638+0000","last_peered":"2024-08-29T15:08:58.875638+0000","last_clean":"2024-08-29T15:08:58.875638+0000","last_became_active":"2024-08-29T15:08:25.185095+0000","last_became_peered":"2024-08-29T15:08:25.185095+0000","last_unstale":"2024-08-29T15:08:58.875638+0000","last_undegraded":"2024-08-29T15:08:58.875638+0000","last_fullsized":"2024-08-29T15:08:58.875638+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:19:03.238217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":80,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.875238+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T15:08:58.875238+0000","last_peered":"2024-08-29T15:08:58.875238+0000","last_clean":"2024-08-29T15:08:58.875238+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T15:08:58.875238+0000","last_undegraded":"2024-08-29T15:08:58.875238+0000","last_fullsized":"2024-08-29T15:08:58.875238+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.873562+0000","last_change":"2024-08-29T15:08:25.186368+0000","last_active":"2024-08-29T15:08:58.873562+0000","last_peered":"2024-08-29T15:08:58.873562+0000","last_clean":"2024-08-29T15:08:58.873562+0000","last_became_active":"2024-08-29T15:08:25.185912+0000","last_became_peered":"2024-08-29T15:08:25.185912+0000","last_unstale":"2024-08-29T15:08:58.873562+0000","last_undegraded":"2024-08-29T15:08:58.873562+0000","last_fullsized":"2024-08-29T15:08:58.873562+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:23.124995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.886830+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:08:58.886830+0000","last_peered":"2024-08-29T15:08:58.886830+0000","last_clean":"2024-08-29T15:08:58.886830+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:08:58.886830+0000","last_undegraded":"2024-08-29T15:08:58.886830+0000","last_fullsized":"2024-08-29T15:08:58.886830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.874998+0000","last_change":"2024-08-29T15:07:52.136541+0000","last_active":"2024-08-29T15:08:58.874998+0000","last_peered":"2024-08-29T15:08:58.874998+0000","last_clean":"2024-08-29T15:08:58.874998+0000","last_became_active":"2024-08-29T15:07:52.136202+0000","last_became_peered":"2024-08-29T15:07:52.136202+0000","last_unstale":"2024-08-29T15:08:58.874998+0000","last_undegraded":"2024-08-29T15:08:58.874998+0000","last_fullsized":"2024-08-29T15:08:58.874998+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:01.747963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888935+0000","last_change":"2024-08-29T15:08:58.888848+0000","last_active":"2024-08-29T15:08:58.888935+0000","last_peered":"2024-08-29T15:08:58.888935+0000","last_clean":"2024-08-29T15:08:58.888935+0000","last_became_active":"2024-08-29T15:08:58.886702+0000","last_became_peered":"2024-08-29T15:08:58.886702+0000","last_unstale":"2024-08-29T15:08:58.888935+0000","last_undegraded":"2024-08-29T15:08:58.888935+0000","last_fullsized":"2024-08-29T15:08:58.888935+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:56.189439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.887636+0000","last_change":"2024-08-29T15:08:58.887151+0000","last_active":"2024-08-29T15:08:58.887636+0000","last_peered":"2024-08-29T15:08:58.887636+0000","last_clean":"2024-08-29T15:08:58.887636+0000","last_became_active":"2024-08-29T15:08:58.886918+0000","last_became_peered":"2024-08-29T15:08:58.886918+0000","last_unstale":"2024-08-29T15:08:58.887636+0000","last_undegraded":"2024-08-29T15:08:58.887636+0000","last_fullsized":"2024-08-29T15:08:58.887636+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:37.557892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889787+0000","last_change":"2024-08-29T15:08:58.889700+0000","last_active":"2024-08-29T15:08:58.889787+0000","last_peered":"2024-08-29T15:08:58.889787+0000","last_clean":"2024-08-29T15:08:58.889787+0000","last_became_active":"2024-08-29T15:08:58.887521+0000","last_became_peered":"2024-08-29T15:08:58.887521+0000","last_unstale":"2024-08-29T15:08:58.889787+0000","last_undegraded":"2024-08-29T15:08:58.889787+0000","last_fullsized":"2024-08-29T15:08:58.889787+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:32.997619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885139+0000","last_change":"2024-08-29T15:07:52.135092+0000","last_active":"2024-08-29T15:08:58.885139+0000","last_peered":"2024-08-29T15:08:58.885139+0000","last_clean":"2024-08-29T15:08:58.885139+0000","last_became_active":"2024-08-29T15:07:52.134625+0000","last_became_peered":"2024-08-29T15:07:52.134625+0000","last_unstale":"2024-08-29T15:08:58.885139+0000","last_undegraded":"2024-08-29T15:08:58.885139+0000","last_fullsized":"2024-08-29T15:08:58.885139+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:22:35.250433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.885989+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:08:58.885989+0000","last_peered":"2024-08-29T15:08:58.885989+0000","last_clean":"2024-08-29T15:08:58.885989+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:08:58.885989+0000","last_undegraded":"2024-08-29T15:08:58.885989+0000","last_fullsized":"2024-08-29T15:08:58.885989+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.889927+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:08:58.889927+0000","last_peered":"2024-08-29T15:08:58.889927+0000","last_clean":"2024-08-29T15:08:58.889927+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:08:58.889927+0000","last_undegraded":"2024-08-29T15:08:58.889927+0000","last_fullsized":"2024-08-29T15:08:58.889927+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.883653+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:08:58.883653+0000","last_peered":"2024-08-29T15:08:58.883653+0000","last_clean":"2024-08-29T15:08:58.883653+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:08:58.883653+0000","last_undegraded":"2024-08-29T15:08:58.883653+0000","last_fullsized":"2024-08-29T15:08:58.883653+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888863+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:08:58.888863+0000","last_peered":"2024-08-29T15:08:58.888863+0000","last_clean":"2024-08-29T15:08:58.888863+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:08:58.888863+0000","last_undegraded":"2024-08-29T15:08:58.888863+0000","last_fullsized":"2024-08-29T15:08:58.888863+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":87,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.873347+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:08:57.873347+0000","last_peered":"2024-08-29T15:08:57.873347+0000","last_clean":"2024-08-29T15:08:57.873347+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:08:57.873347+0000","last_undegraded":"2024-08-29T15:08:57.873347+0000","last_fullsized":"2024-08-29T15:08:57.873347+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884205+0000","last_change":"2024-08-29T15:08:58.884121+0000","last_active":"2024-08-29T15:08:58.884205+0000","last_peered":"2024-08-29T15:08:58.884205+0000","last_clean":"2024-08-29T15:08:58.884205+0000","last_became_active":"2024-08-29T15:08:58.883669+0000","last_became_peered":"2024-08-29T15:08:58.883669+0000","last_unstale":"2024-08-29T15:08:58.884205+0000","last_undegraded":"2024-08-29T15:08:58.884205+0000","last_fullsized":"2024-08-29T15:08:58.884205+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:32.165590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":59,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878176+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:08:58.878176+0000","last_peered":"2024-08-29T15:08:58.878176+0000","last_clean":"2024-08-29T15:08:58.878176+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:08:58.878176+0000","last_undegraded":"2024-08-29T15:08:58.878176+0000","last_fullsized":"2024-08-29T15:08:58.878176+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884996+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:08:58.884996+0000","last_peered":"2024-08-29T15:08:58.884996+0000","last_clean":"2024-08-29T15:08:58.884996+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:08:58.884996+0000","last_undegraded":"2024-08-29T15:08:58.884996+0000","last_fullsized":"2024-08-29T15:08:58.884996+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872326+0000","last_change":"2024-08-29T15:07:52.134597+0000","last_active":"2024-08-29T15:08:57.872326+0000","last_peered":"2024-08-29T15:08:57.872326+0000","last_clean":"2024-08-29T15:08:57.872326+0000","last_became_active":"2024-08-29T15:07:52.133893+0000","last_became_peered":"2024-08-29T15:07:52.133893+0000","last_unstale":"2024-08-29T15:08:57.872326+0000","last_undegraded":"2024-08-29T15:08:57.872326+0000","last_fullsized":"2024-08-29T15:08:57.872326+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:14.240866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.884919+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:08:58.884919+0000","last_peered":"2024-08-29T15:08:58.884919+0000","last_clean":"2024-08-29T15:08:58.884919+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:08:58.884919+0000","last_undegraded":"2024-08-29T15:08:58.884919+0000","last_fullsized":"2024-08-29T15:08:58.884919+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.875599+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T15:08:57.875599+0000","last_peered":"2024-08-29T15:08:57.875599+0000","last_clean":"2024-08-29T15:08:57.875599+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T15:08:57.875599+0000","last_undegraded":"2024-08-29T15:08:57.875599+0000","last_fullsized":"2024-08-29T15:08:57.875599+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888235+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:08:58.888235+0000","last_peered":"2024-08-29T15:08:58.888235+0000","last_clean":"2024-08-29T15:08:58.888235+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:08:58.888235+0000","last_undegraded":"2024-08-29T15:08:58.888235+0000","last_fullsized":"2024-08-29T15:08:58.888235+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":82,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.888750+0000","last_change":"2024-08-29T15:08:58.888653+0000","last_active":"2024-08-29T15:08:58.888750+0000","last_peered":"2024-08-29T15:08:58.888750+0000","last_clean":"2024-08-29T15:08:58.888750+0000","last_became_active":"2024-08-29T15:08:58.888133+0000","last_became_peered":"2024-08-29T15:08:58.888133+0000","last_unstale":"2024-08-29T15:08:58.888750+0000","last_undegraded":"2024-08-29T15:08:58.888750+0000","last_fullsized":"2024-08-29T15:08:58.888750+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:24.144974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-08-29T15:08:57.872210+0000","last_change":"2024-08-29T15:07:52.135649+0000","last_active":"2024-08-29T15:08:57.872210+0000","last_peered":"2024-08-29T15:08:57.872210+0000","last_clean":"2024-08-29T15:08:57.872210+0000","last_became_active":"2024-08-29T15:07:52.135327+0000","last_became_peered":"2024-08-29T15:07:52.135327+0000","last_unstale":"2024-08-29T15:08:57.872210+0000","last_undegraded":"2024-08-29T15:08:57.872210+0000","last_fullsized":"2024-08-29T15:08:57.872210+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:05.402462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2024-08-29T15:08:58.878257+0000","last_change":"2024-08-29T15:08:25.187422+0000","last_active":"2024-08-29T15:08:58.878257+0000","last_peered":"2024-08-29T15:08:58.878257+0000","last_clean":"2024-08-29T15:08:58.878257+0000","last_became_active":"2024-08-29T15:08:25.187062+0000","last_became_peered":"2024-08-29T15:08:25.187062+0000","last_unstale":"2024-08-29T15:08:58.878257+0000","last_undegraded":"2024-08-29T15:08:58.878257+0000","last_fullsized":"2024-08-29T15:08:58.878257+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:18:20.745348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234185,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37004,"kb_used_data":5888,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708148,"statfs":{"total":95995035648,"available":95957143552,"internally_reserved":0,"allocated":6029312,"data_stored":5417499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495821,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28808,"kb_used_data":1780,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716344,"statfs":{"total":95995035648,"available":95965536256,"internally_reserved":0,"allocated":1822720,"data_stored":1222795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757458,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1192,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1220608,"data_stored":628195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019095,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36420,"kb_used_data":5300,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93708732,"statfs":{"total":95995035648,"available":95957741568,"internally_reserved":0,"allocated":5427200,"data_stored":4826788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435279,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28664,"kb_used_data":1188,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716488,"statfs":{"total":95995035648,"available":95965683712,"internally_reserved":0,"allocated":1216512,"data_stored":627957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":28110089},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598806,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28712,"kb_used_data":1180,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716440,"statfs":{"total":95995035648,"available":95965634560,"internally_reserved":0,"allocated":1208320,"data_stored":627233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":28172623},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762333,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28728,"kb_used_data":1188,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716424,"statfs":{"total":95995035648,"available":95965618176,"internally_reserved":0,"allocated":1216512,"data_stored":627886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":28172642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925859,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33396,"kb_used_data":5860,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93711756,"statfs":{"total":95995035648,"available":95960838144,"internally_reserved":0,"allocated":6000640,"data_stored":5411702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":28172431},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:10:07.498 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-29T15:10:07.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump 2024-08-29T15:10:07.713 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:08.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:08 smithi027 ceph-mon[98970]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:08.162 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:08 smithi027 ceph-mon[98970]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:10:08.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:08 smithi027 ceph-mon[93921]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:08.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:08 smithi027 ceph-mon[93921]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:10:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:08 smithi171 ceph-mon[90031]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:08 smithi171 ceph-mon[90031]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:10:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:10:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:10:09] "GET /metrics HTTP/1.1" 200 33935 "" "Prometheus/2.51.0" 2024-08-29T15:10:09.400 INFO:teuthology.orchestra.run.smithi027.stdout:epoch 110 2024-08-29T15:10:09.400 INFO:teuthology.orchestra.run.smithi027.stdout:fsid af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:10:09.400 INFO:teuthology.orchestra.run.smithi027.stdout:created 2024-08-29T14:48:44.626777+0000 2024-08-29T15:10:09.400 INFO:teuthology.orchestra.run.smithi027.stdout:modified 2024-08-29T15:08:58.870443+0000 2024-08-29T15:10:09.400 INFO:teuthology.orchestra.run.smithi027.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:crush_version 22 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:full_ratio 0.95 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:backfillfull_ratio 0.9 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:nearfull_ratio 0.85 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:require_min_compat_client luminous 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:min_compat_client luminous 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:require_osd_release reef 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:stretch_mode_enabled false 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 85 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-08-29T15:10:09.401 INFO:teuthology.orchestra.run.smithi027.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:max_osd 8 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423] [v2:172.21.15.27:6804/297906423,v1:172.21.15.27:6805/297906423] exists,up 170eaad0-5c69-43ff-a4cc-cfdcc1763f52 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131] [v2:172.21.15.27:6812/2860215131,v1:172.21.15.27:6813/2860215131] exists,up 73986270-de54-40cf-969c-ddb7ecba6f81 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233] [v2:172.21.15.27:6820/1131810233,v1:172.21.15.27:6821/1131810233] exists,up d2b83d1d-ecf8-4576-a411-6d05ef2528c9 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975] [v2:172.21.15.27:6828/3647667975,v1:172.21.15.27:6829/3647667975] exists,up afc1c2c8-0579-4020-b460-b3cbef9573d5 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111] [v2:172.21.15.171:6802/3854913111,v1:172.21.15.171:6803/3854913111] exists,up adb26953-31d6-491f-8777-02fbb879b3a6 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519] [v2:172.21.15.171:6810/223130519,v1:172.21.15.171:6811/223130519] exists,up 2b6e927d-bca5-4fe8-b17f-7b83a872afad 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066] [v2:172.21.15.171:6818/2050068066,v1:172.21.15.171:6819/2050068066] exists,up f3c5992d-70c0-4893-9221-b8bc5268d92c 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687] [v2:172.21.15.171:6826/3106954687,v1:172.21.15.171:6827/3106954687] exists,up 3c1227a6-0c5b-4753-b5d3-f9e98383c3c3 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:pg_upmap_items 4.d [1,5] 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:pg_upmap_items 4.13 [1,2] 2024-08-29T15:10:09.402 INFO:teuthology.orchestra.run.smithi027.stdout:pg_upmap_items 4.14 [1,2] 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/258529454 expires 2024-08-30T15:01:28.195875+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/92805749 expires 2024-08-30T15:01:28.195875+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6801/301351752 expires 2024-08-30T15:01:28.195875+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6800/301351752 expires 2024-08-30T15:01:28.195875+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/921286138 expires 2024-08-30T14:58:14.775345+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/2218055720 expires 2024-08-30T14:54:15.539878+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/3735938855 expires 2024-08-30T14:49:36.411583+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:0/2152868901 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/2828185086 expires 2024-08-30T15:01:28.195875+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/1323146262 expires 2024-08-30T14:54:15.539878+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/2385046935 expires 2024-08-30T14:49:36.411583+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6801/1661583367 expires 2024-08-30T14:49:36.411583+0000 2024-08-29T15:10:09.403 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6800/1661583367 expires 2024-08-30T14:49:36.411583+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6801/2650682884 expires 2024-08-30T14:58:14.775345+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/2198805353 expires 2024-08-30T14:54:15.539878+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:6835/2863400284 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:0/2912002540 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/2955701830 expires 2024-08-30T14:58:14.775345+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:6834/2863400284 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/3268243398 expires 2024-08-30T14:49:36.411583+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/728365796 expires 2024-08-30T14:49:05.654793+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/275324420 expires 2024-08-30T14:54:15.539878+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/579941994 expires 2024-08-30T14:49:05.654793+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/2006046934 expires 2024-08-30T14:49:05.654793+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:0/3744606577 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.404 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6800/2650682884 expires 2024-08-30T14:58:14.775345+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:0/1480641142 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/1565341554 expires 2024-08-30T15:01:28.195875+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:0/1064847764 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6800/2777626812 expires 2024-08-30T14:49:05.654793+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6801/2777626812 expires 2024-08-30T14:49:05.654793+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/2657305076 expires 2024-08-30T14:58:14.775345+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:0/1523003109 expires 2024-08-30T14:58:14.775345+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6800/1134596600 expires 2024-08-30T14:54:15.539878+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.171:0/3978338528 expires 2024-08-30T14:58:56.932943+0000 2024-08-29T15:10:09.405 INFO:teuthology.orchestra.run.smithi027.stdout:blocklist 172.21.15.27:6801/1134596600 expires 2024-08-30T14:54:15.539878+0000 2024-08-29T15:10:10.018 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.27:6802/297906423,v1:172.21.15.27:6803/297906423] [v2:172.21.15.27:6804/297906423,v1:172.21.15.27:6805/297906423] exists,up 170eaad0-5c69-43ff-a4cc-cfdcc1763f52', 'osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.27:6810/2860215131,v1:172.21.15.27:6811/2860215131] [v2:172.21.15.27:6812/2860215131,v1:172.21.15.27:6813/2860215131] exists,up 73986270-de54-40cf-969c-ddb7ecba6f81', 'osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.27:6818/1131810233,v1:172.21.15.27:6819/1131810233] [v2:172.21.15.27:6820/1131810233,v1:172.21.15.27:6821/1131810233] exists,up d2b83d1d-ecf8-4576-a411-6d05ef2528c9', 'osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.27:6826/3647667975,v1:172.21.15.27:6827/3647667975] [v2:172.21.15.27:6828/3647667975,v1:172.21.15.27:6829/3647667975] exists,up afc1c2c8-0579-4020-b460-b3cbef9573d5', 'osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6800/3854913111,v1:172.21.15.171:6801/3854913111] [v2:172.21.15.171:6802/3854913111,v1:172.21.15.171:6803/3854913111] exists,up adb26953-31d6-491f-8777-02fbb879b3a6', 'osd.5 up in weight 1 up_from 38 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6808/223130519,v1:172.21.15.171:6809/223130519] [v2:172.21.15.171:6810/223130519,v1:172.21.15.171:6811/223130519] exists,up 2b6e927d-bca5-4fe8-b17f-7b83a872afad', 'osd.6 up in weight 1 up_from 46 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6816/2050068066,v1:172.21.15.171:6817/2050068066] [v2:172.21.15.171:6818/2050068066,v1:172.21.15.171:6819/2050068066] exists,up f3c5992d-70c0-4893-9221-b8bc5268d92c', 'osd.7 up in weight 1 up_from 54 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.171:6824/3106954687,v1:172.21.15.171:6825/3106954687] [v2:172.21.15.171:6826/3106954687,v1:172.21.15.171:6827/3106954687] exists,up 3c1227a6-0c5b-4753-b5d3-f9e98383c3c3'] 2024-08-29T15:10:10.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon mon.a config show 2024-08-29T15:10:10.231 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:10 smithi171 ceph-mon[90031]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3745818231' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-29T15:10:10.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:10 smithi027 ceph-mon[98970]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:10.295 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3745818231' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-29T15:10:10.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:10 smithi027 ceph-mon[93921]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:10.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3745818231' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-29T15:10:11.887 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "mon.a", 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "cluster": "ceph", 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "admin_socket_mode": "", 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_client_required": "cephx, none", 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_cluster_required": "cephx", 2024-08-29T15:10:11.888 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_debug": "true", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_service_required": "cephx", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_service_ticket_ttl": "240.000000", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "auth_supported": "", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_aio": "true", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_aio_max_queue_depth": "1024", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_aio_poll_ms": "250", 2024-08-29T15:10:11.889 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_aio_reap_max": "16", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_async_discard_threads": "0", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_block_size": "4096", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_debug_aio": "false", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_debug_inflight_ios": "false", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_enable_discard": "false", 2024-08-29T15:10:11.890 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_flock_retry": "3", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_flock_retry_interval": "0.100000", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_inject_crash": "0", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_inject_crash_flush_delay": "2", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_ioring": "false", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_ioring_hipri": "false", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_ioring_sqthread_poll": "false", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-08-29T15:10:11.891 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_read_buffer_alignment": "4096", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bdev_type": "", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_alloc_size": "1048576", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_allocator": "hybrid", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_buffered_io": "true", 2024-08-29T15:10:11.892 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_check_for_zeros": "false", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_check_volume_selector_often": "false", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_compact_log_sync": "false", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_log_compact_min_size": "16777216", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_log_replay_check_allocations": "true", 2024-08-29T15:10:11.893 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_max_log_runway": "4194304", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_max_prefetch": "1048576", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_min_flush_size": "524288", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_min_log_runway": "1048576", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_replay_recovery": "false", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_shared_alloc_size": "65536", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluefs_sync_write": "false", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-08-29T15:10:11.894 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-08-29T15:10:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-08-29T15:10:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_allocation_from_file": "true", 2024-08-29T15:10:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_allocator": "hybrid", 2024-08-29T15:10:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_async_db_compaction": "true", 2024-08-29T15:10:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-08-29T15:10:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-08-29T15:10:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bdev_label_multi": "true", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bdev_label_require_all": "true", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_blobid_prealloc": "10240", 2024-08-29T15:10:11.896 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_create": "true", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_db_create": "false", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_db_path": "", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_db_size": "0", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_path": "", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_preallocate_file": "false", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_size": "107374182400", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_wal_create": "false", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_wal_path": "", 2024-08-29T15:10:11.897 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_block_wal_size": "100663296", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bluefs": "true", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bluefs_env_mirror": "false", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-08-29T15:10:11.898 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-08-29T15:10:11.899 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-08-29T15:10:11.899 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-08-29T15:10:11.899 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_autotune": "true", 2024-08-29T15:10:11.899 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-08-29T15:10:11.899 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-08-29T15:10:11.900 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-08-29T15:10:11.900 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-08-29T15:10:11.900 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_size": "0", 2024-08-29T15:10:11.900 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-08-29T15:10:11.900 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-08-29T15:10:11.900 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cache_type": "2q", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_clone_cow": "true", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_algorithm": "snappy", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_max_blob_size": "0", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_min_blob_size": "0", 2024-08-29T15:10:11.901 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_mode": "none", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_csum_type": "crc32c", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_enforce_settings": "default", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_freelist": "false", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_fsck_abort": "false", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_inject_read_err": "false", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_legacy_omap": "false", 2024-08-29T15:10:11.902 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_small_allocations": "0", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_default_buffered_read": "true", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_default_buffered_write": "false", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_deferred_batch_ops": "0", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-08-29T15:10:11.903 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_elastic_shared_blobs": "true", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fail_eio": "false", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_on_mkfs": "true", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_on_mount": "false", 2024-08-29T15:10:11.904 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_on_umount": "false", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_ignore_data_csum": "false", 2024-08-29T15:10:11.905 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_kvbackend": "rocksdb", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_log_op_age": "5.000000", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_max_alloc_size": "0", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_max_blob_size": "0", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_max_defer_interval": "3.000000", 2024-08-29T15:10:11.906 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_max_deferred_txc": "32", 2024-08-29T15:10:11.907 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_min_alloc_size": "0", 2024-08-29T15:10:11.907 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-08-29T15:10:11.907 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-08-29T15:10:11.907 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_nid_prealloc": "1024", 2024-08-29T15:10:11.907 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_prefer_deferred_size": "0", 2024-08-29T15:10:11.907 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-08-29T15:10:11.907 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-08-29T15:10:11.908 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_qfsck_on_mount": "true", 2024-08-29T15:10:11.908 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_retry_disk_reads": "3", 2024-08-29T15:10:11.908 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_rocksdb_cf": "true", 2024-08-29T15:10:11.908 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-08-29T15:10:11.908 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-08-29T15:10:11.908 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_rocksdb_options_annex": "", 2024-08-29T15:10:11.909 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-08-29T15:10:11.909 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-08-29T15:10:11.909 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_spdk_coremask": "0x1", 2024-08-29T15:10:11.909 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_spdk_io_sleep": "5", 2024-08-29T15:10:11.909 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_spdk_max_io_completion": "0", 2024-08-29T15:10:11.910 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_spdk_mem": "512", 2024-08-29T15:10:11.910 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_sync_submit_transaction": "false", 2024-08-29T15:10:11.910 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_throttle_bytes": "67108864", 2024-08-29T15:10:11.910 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_throttle_cost_per_io": "0", 2024-08-29T15:10:11.910 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-08-29T15:10:11.910 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_tracing": "false", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_volume_selection_reserved": "0", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-08-29T15:10:11.911 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_write_v2": "false", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_write_v2_random": "false", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "bluestore_zero_block_detection": "false", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_action_update_interval": "2", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_mount_timeout": "10", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-08-29T15:10:11.912 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephx_cluster_require_signatures": "false", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephx_cluster_require_version": "2", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephx_require_signatures": "false", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephx_require_version": "2", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephx_service_require_signatures": "false", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephx_service_require_version": "2", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "cephx_sign_messages": "true", 2024-08-29T15:10:11.913 INFO:teuthology.orchestra.run.smithi027.stdout: "chdir": "", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_acl_type": "", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_asio_thread_count": "2", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_cache_mid": "0.750000", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_cache_size": "16384", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_caps_release_delay": "5", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_check_pool_perm": "true", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_collect_and_send_global_metrics": "false", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_debug_force_sync_read": "false", 2024-08-29T15:10:11.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_debug_getattr_caps": "false", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_debug_inject_features": "", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_debug_inject_tick_delay": "0", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_die_on_failed_remount": "false", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_dirsize_rbytes": "true", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_force_lazyio": "false", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_fs": "", 2024-08-29T15:10:11.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_inject_fixed_oldest_tid": "false", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_inject_release_failure": "false", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_max_inline_size": "4096", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_max_retries_on_remount_failure": "5", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_mds_namespace": "", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_metadata": "", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_mount_gid": "-1", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_mount_timeout": "300", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_mount_uid": "-1", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_mountpoint": "/", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_notify_timeout": "10", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_oc": "true", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_oc_max_dirty": "104857600", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_oc_max_dirty_age": "5.000000", 2024-08-29T15:10:11.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_oc_max_objects": "1000", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_oc_size": "209715200", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_oc_target_dirty": "8388608", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_permissions": "true", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_quota": "true", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_quota_df": "true", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_readahead_max_bytes": "0", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_readahead_max_periods": "4", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_readahead_min": "131072", 2024-08-29T15:10:11.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_reconnect_stale": "false", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "client_shutdown_timeout": "30", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "client_snapdir": ".snap", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "client_tick_interval": "1", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "client_trace": "", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "client_try_dentry_invalidate": "false", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "client_use_faked_inos": "false", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "client_use_random_mds": "false", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "clog_to_graylog": "false", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "clog_to_graylog_port": "12201", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "clog_to_monitors": "default=true", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "clog_to_syslog": "false", 2024-08-29T15:10:11.918 INFO:teuthology.orchestra.run.smithi027.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "clog_to_syslog_level": "info", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "cluster_addr": "-", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "cluster_network": "", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "cluster_network_interface": "", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "compressor_zlib_isal": "false", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "compressor_zlib_level": "5", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "compressor_zlib_winsize": "-15", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "compressor_zstd_level": "1", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "crimson_alien_op_num_threads": "6", 2024-08-29T15:10:11.919 INFO:teuthology.orchestra.run.smithi027.stdout: "crimson_alien_thread_cpu_cores": "", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crimson_osd_obc_lru_size": "512", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crimson_osd_stat_interval": "0", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crimson_seastar_cpu_cores": "", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crimson_seastar_num_threads": "0", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crush_location": "", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crush_location_hook": "", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "crush_location_hook_timeout": "10", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "daemonize": "false", 2024-08-29T15:10:11.920 INFO:teuthology.orchestra.run.smithi027.stdout: "daos_pool": "tank", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "dbstore_db_name_prefix": "dbstore", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_alienstore": "0/5", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_allow_any_pool_priority": "false", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_asok": "1/5", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_asok_assert_abort": "false", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_asserts_on_shutdown": "true", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_auth": "1/5", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_bdev": "1/3", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_bluefs": "1/5", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_bluestore": "1/5", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_buffer": "0/1", 2024-08-29T15:10:11.921 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_ceph_dedup": "0/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_ceph_exporter": "1/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_cephfs_mirror": "0/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_cephsqlite": "0/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_client": "0/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_compressor": "1/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_context": "0/1", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_crimson_interrupt": "0/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_crush": "1/1", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_crypto": "1/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_cyanstore": "0/5", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_deliberately_leak_memory": "false", 2024-08-29T15:10:11.922 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_disable_randomized_ping": "false", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_dpdk": "1/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_eventtrace": "1/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_filer": "0/1", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_filestore": "1/3", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_finisher": "1/1", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_fuse": "1/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_heartbeat_testing_span": "0", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_heartbeatmap": "1/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_immutable_obj_cache": "0/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_javaclient": "1/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_journal": "1/3", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_journaler": "0/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_kstore": "1/5", 2024-08-29T15:10:11.923 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_lockdep": "0/1", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mclock": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mds": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mds_balancer": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mds_locker": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mds_log": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mds_log_expire": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mds_migrator": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mds_quiesce": "3/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_memstore": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mgr": "2/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mgrc": "1/5", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_mon": "20/20", 2024-08-29T15:10:11.924 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_monc": "0/10", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_ms": "1/1", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_none": "0/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_objclass": "0/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_objectcacher": "0/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_objecter": "0/1", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_optracker": "0/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_osd": "1/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_paxos": "20/20", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_perfcounter": "1/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_prioritycache": "1/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rados": "0/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rbd": "0/5", 2024-08-29T15:10:11.925 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rbd_mirror": "0/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rbd_pwl": "0/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rbd_replay": "0/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_refs": "0/0", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_reserver": "1/1", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw_access": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw_datacache": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw_dbstore": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw_flight": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw_lifecycle": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw_notification": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rgw_sync": "1/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_rocksdb": "4/5", 2024-08-29T15:10:11.926 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_backref": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_cache": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_cleaner": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_device": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_epm": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_journal": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_lba": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_odata": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_omap": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_onode": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_t": "0/5", 2024-08-29T15:10:11.927 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_seastore_tm": "0/5", 2024-08-29T15:10:11.928 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_striper": "0/1", 2024-08-29T15:10:11.928 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_test": "0/5", 2024-08-29T15:10:11.928 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_throttle": "1/1", 2024-08-29T15:10:11.928 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_timer": "0/1", 2024-08-29T15:10:11.928 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_tp": "0/5", 2024-08-29T15:10:11.928 INFO:teuthology.orchestra.run.smithi027.stdout: "debug_trace": "1/5", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "device_failure_prediction_mode": "none", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "err_to_graylog": "false", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "err_to_journald": "false", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "err_to_stderr": "true", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "err_to_syslog": "false", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event_tracing": "false", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_addr": "0.0.0.0", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_cert_file": "", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_http_port": "9926", 2024-08-29T15:10:11.929 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_key_file": "", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_prio_limit": "5", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_sort_metrics": "true", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "exporter_stats_period": "5", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "fake_statfs_for_testing": "0", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "fatal_signal_handlers": "true", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "filer_max_purge_ops": "10", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "filer_max_truncate_ops": "128", 2024-08-29T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_apply_finisher_threads": "1", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_blackhole": "false", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_btrfs_clone_range": "true", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_btrfs_snap": "true", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_caller_concurrency": "10", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_collect_device_partition_information": "true", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_commit_timeout": "600.000000", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_debug_inject_read_err": "false", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_debug_omap_check": "false", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_debug_verify_split": "false", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_dump_file": "", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_fadvise": "true", 2024-08-29T15:10:11.931 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_fail_eio": "true", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_fd_cache_shards": "16", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_fd_cache_size": "128", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_fiemap": "false", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_fiemap_threshold": "4096", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_index_retry_probability": "0.000000", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_inject_stall": "0", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_journal_parallel": "false", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_journal_trailing": "false", 2024-08-29T15:10:11.932 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_journal_writeahead": "false", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_kill_at": "0", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattr_size": "0", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattrs": "0", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattrs_other": "2", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_sync_interval": "5.000000", 2024-08-29T15:10:11.933 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_xattr_value_size": "0", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_merge_threshold": "-10", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_min_sync_interval": "0.010000", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_odsync_write": "false", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_omap_backend": "rocksdb", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_omap_backend_path": "", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_omap_header_cache_size": "1024", 2024-08-29T15:10:11.934 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_ondisk_finisher_threads": "1", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_op_thread_timeout": "60", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_op_threads": "2", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_punch_hole": "false", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_max_bytes": "104857600", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-08-29T15:10:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_queue_max_ops": "50", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_seek_data_hole": "false", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_sloppy_crc": "false", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_splice": "false", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_split_multiple": "2", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_split_rand_factor": "20", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_update_to": "1000", 2024-08-29T15:10:11.936 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_enable": "true", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-08-29T15:10:11.937 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_xfs_extsize": "false", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "filestore_zfs_snap": "false", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fio_dir": "/tmp/fio", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fsid": "af03926c-6615-11ef-bcd4-c7b262605968", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_allow_other": "true", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_atomic_o_trunc": "true", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_big_writes": "true", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_debug": "false", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_default_permissions": "false", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_disable_pagecache": "false", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_max_write": "0", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_multithreaded": "true", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_require_active_mds": "true", 2024-08-29T15:10:11.938 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_set_user_groups": "true", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_splice_move": "true", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_splice_read": "true", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_splice_write": "true", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_syncfs_on_mksnap": "true", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "fuse_use_invalidate_cb": "true", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "gss_target_name": "ceph", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "heartbeat_file": "", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "heartbeat_inject_failure": "0", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "heartbeat_interval": "5", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "host": "smithi027", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-08-29T15:10:11.939 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_max_size": "1073741824", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "immutable_object_cache_watermark": "0.900000", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "inject_early_sigterm": "false", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "jaeger_agent_port": "6799", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "jaeger_tracing_enable": "false", 2024-08-29T15:10:11.940 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_aio": "true", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_align_min_size": "65536", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_block_align": "true", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_block_size": "4096", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_dio": "true", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_discard": "false", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_force_aio": "false", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_ignore_corruption": "false", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_max_write_bytes": "10485760", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_max_write_entries": "100", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_replay_from": "0", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_throttle_high_multiple": "0.000000", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-08-29T15:10:11.941 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_throttle_max_multiple": "0.000000", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_write_header_frequency": "0", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "journal_zero_on_create": "false", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "journaler_prefetch_periods": "10", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "journaler_prezero_periods": "5", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "journaler_write_head_interval": "15", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "key": "", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "keyfile": "", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_backend": "rocksdb", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_default_stripe_size": "65536", 2024-08-29T15:10:11.942 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_fsck_on_mount": "false", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_fsck_on_mount_deep": "true", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_max_bytes": "67108864", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_max_ops": "512", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_nid_prealloc": "1024", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_onode_map_size": "1024", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_sync_submit_transaction": "false", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "kstore_sync_transaction": "false", 2024-08-29T15:10:11.944 INFO:teuthology.orchestra.run.smithi027.stdout: "librados_thread_count": "2", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "lockdep": "false", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "lockdep_force_backtrace": "false", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_coarse_timestamps": "true", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_flush_on_exit": "false", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_graylog_host": "127.0.0.1", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_graylog_port": "12201", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_max_new": "1000", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_max_recent": "10000", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_stderr_prefix": "", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_stop_at_utilization": "0.970000", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_to_file": "true", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_to_graylog": "false", 2024-08-29T15:10:11.945 INFO:teuthology.orchestra.run.smithi027.stdout: "log_to_journald": "false", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "log_to_stderr": "false", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "log_to_syslog": "false", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "max_rotating_auth_attempts": "10", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_action_on_write_error": "1", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_alternate_name_max": "8192", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_asio_thread_count": "2", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_export_pin": "true", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_fragment_dirs": "true", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-08-29T15:10:11.946 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_fragment_interval": "5", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_fragment_size_max": "100000", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_idle_threshold": "0.000000", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_interval": "10", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_max": "-1", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_max_until": "-1", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_merge_size": "50", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_midchunk": "0.300000", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_min_rebalance": "0.100000", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_min_start": "0.200000", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_minchunk": "0.001000", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_mode": "0", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_need_max": "1.200000", 2024-08-29T15:10:11.947 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_need_min": "0.800000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_overload_epochs": "2", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_sample_interval": "3.000000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_split_bits": "3", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_split_rd": "25000.000000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_split_size": "10000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_split_wr": "10000.000000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_target_decay": "10.000000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_beacon_grace": "15.000000", 2024-08-29T15:10:11.948 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_beacon_interval": "4.000000", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_beacon_mon_down_grace": "60", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_memory_limit": "4294967296", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_mid": "0.700000", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_quiesce_delay": "0", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_quiesce_sleep": "200", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_quiesce_splitauth": "true", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_quiesce_threshold": "524288", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_release_free_interval": "10", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_reservation": "0.050000", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_trim_interval": "1", 2024-08-29T15:10:11.949 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cache_trim_threshold": "262144", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_client_delegate_inos_pct": "50", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_client_prealloc_inos": "1000", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_connect_bootstrapping": "false", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_damage_table_max_entries": "10000", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_debug_auth_pins": "false", 2024-08-29T15:10:11.950 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_debug_frag": "false", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_debug_scatterstat": "false", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_debug_subtrees": "false", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_decay_halflife": "5.000000", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_default_dir_hash": "2", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_defer_session_stale": "true", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_deny_all_reconnect": "false", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dir_keys_per_op": "16384", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dir_max_commit_size": "10", 2024-08-29T15:10:11.951 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dir_max_entries": "0", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dir_prefetch": "true", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dirstat_min_interval": "1.000000", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dump_cache_after_rejoin": "false", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dump_cache_on_map": "false", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dump_cache_threshold_file": "0", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_early_reply": "true", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_enable_op_tracker": "true", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_enforce_unique_name": "true", 2024-08-29T15:10:11.952 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_export_ephemeral_distributed": "true", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_export_ephemeral_random": "true", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_forward_all_requests_to_auth": "false", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-08-29T15:10:11.953 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_health_cache_threshold": "1.500000", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_health_summarize_threshold": "10", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_heartbeat_grace": "15.000000", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_heartbeat_reset_grace": "1000", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_inject_health_dummy": "false", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_inject_migrator_session_race": "false", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-08-29T15:10:11.954 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_join_fs": "", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_journal_format": "1", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_create_at": "0", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_dirfrag_at": "0", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_export_at": "0", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_import_at": "0", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_journal_at": "0", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_journal_expire_at": "0", 2024-08-29T15:10:11.955 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_journal_replay_at": "0", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_link_at": "0", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_mdstable_at": "0", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_openc_at": "0", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_rename_at": "0", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_kill_shutdown_at": "0", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_event_large_threshold": "524288", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_events_per_segment": "1024", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_major_segment_event_ratio": "12", 2024-08-29T15:10:11.956 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_max_events": "-1", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_max_segments": "128", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_pause": "false", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_segment_size": "0", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_skip_corrupt_events": "false", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_skip_unbounded_events": "false", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_trim_threshold": "128", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-08-29T15:10:11.957 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_log_warn_factor": "2.000000", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_caps_per_client": "1048576", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_completed_flushes": "100000", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_completed_requests": "100000", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_export_size": "20971520", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_file_recover": "32", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_purge_files": "64", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_purge_ops": "8192", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_max_snaps_per_dir": "100", 2024-08-29T15:10:11.958 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_metrics_update_interval": "2", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_min_caps_per_client": "100", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_min_caps_working_set": "10000", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_numa_node": "-1", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_op_complaint_time": "30.000000", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_op_history_duration": "600", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_op_history_size": "20", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_op_history_slow_op_size": "20", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_op_history_slow_op_threshold": "10", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_op_log_threshold": "5", 2024-08-29T15:10:11.959 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_ping_grace": "15", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_ping_interval": "5", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_recall_max_caps": "30000", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_recall_max_decay_threshold": "131072", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_recall_warning_threshold": "262144", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_reconnect_timeout": "45.000000", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_replay_interval": "1.000000", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-08-29T15:10:11.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_root_ino_gid": "0", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_root_ino_uid": "0", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_blocklist_on_evict": "true", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_blocklist_on_timeout": "true", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_session_metadata_threshold": "16777216", 2024-08-29T15:10:11.961 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_shutdown_check": "0", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_skip_ino": "0", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_sleep_rank_change": "0.000000", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_snap_max_uid": "4294967294", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_snap_min_uid": "0", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_snap_rstat": "false", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_standby_replay_damaged": "false", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_symlink_recovery": "true", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_task_status_update_interval": "2.000000", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_thrash_exports": "0", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_thrash_fragments": "0", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_tick_interval": "5.000000", 2024-08-29T15:10:11.962 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_valgrind_exit": "false", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_verify_backtrace": "1", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_verify_scatter": "false", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_wipe_ino_prealloc": "false", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mds_wipe_sessions": "false", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mempool_debug": "false", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "memstore_debug_omit_block_device_write": "false", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "memstore_device_bytes": "1073741824", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "memstore_page_set": "false", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "memstore_page_size": "65536", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_client_bytes": "134217728", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_client_messages": "512", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_connect_retry_interval": "1.000000", 2024-08-29T15:10:11.963 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_disabled_modules": "", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_enable_op_tracker": "true", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_max_pg_creating": "1024", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_max_pg_num_change": "128", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_mds_bytes": "134217728", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_mds_messages": "128", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_mon_bytes": "134217728", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_mon_messages": "128", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_num_op_tracker_shard": "32", 2024-08-29T15:10:11.964 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_op_complaint_time": "30.000000", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_op_history_duration": "600", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_op_history_size": "20", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_op_history_slow_op_size": "20", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_op_log_threshold": "5", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_osd_bytes": "536870912", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_osd_messages": "8192", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_pool": "true", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_service_beacon_grace": "60.000000", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_standby_modules": "true", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_stats_period": "5", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_stats_threshold": "5", 2024-08-29T15:10:11.965 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_tick_period": "2", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_accept_timeout_factor": "2.000000", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_allow_pool_delete": "true", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_allow_pool_size_one": "false", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_auth_validate_all_caps": "true", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_bytes": "104857600", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_directed_command_retry": "2", 2024-08-29T15:10:11.966 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_hunt_interval": "3.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_hunt_parallel": "3", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_log_interval": "1.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_ping_interval": "10.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_ping_timeout": "30.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_client_target_rank": "-1", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_clock_drift_allowed": "1.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-08-29T15:10:11.967 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_level": "debug", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_file": "true", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_graylog": "false", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_journald": "true", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_stderr": "false", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_compact_on_bootstrap": "false", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_compact_on_start": "false", 2024-08-29T15:10:11.968 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_compact_on_trim": "true", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_con_tracker_persist_interval": "10", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_con_tracker_score_halflife": "43200", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_config_key_max_entry_size": "65536", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_cpu_threads": "4", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_crush_min_required_version": "hammer", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_daemon_bytes": "419430400", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_data_avail_crit": "5", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_data_avail_warn": "5", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_data_size_warn": "16106127360", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_block_osdmap_trim": "false", 2024-08-29T15:10:11.969 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_dump_json": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_dump_transactions": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_extra_checks": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_no_require_reef": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_no_require_squid": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_delta_reset_interval": "10.000000", 2024-08-29T15:10:11.970 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_dns_srv_name": "ceph-mon", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_down_mkfs_grace": "60", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_down_uptime_grace": "60", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_election_default_strategy": "1", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_election_timeout": "5.000000", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_elector_ping_divisor": "2", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_elector_ping_timeout": "2.000000", 2024-08-29T15:10:11.971 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_enable_op_tracker": "true", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_fake_pool_delete": "false", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_force_quorum_join": "false", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_fsmap_prune_threshold": "300", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_globalid_prealloc": "10000", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_health_detail_to_clog": "true", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_health_log_update_period": "5", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_health_max_detail": "50", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_health_to_clog": "true", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_health_to_clog_interval": "600", 2024-08-29T15:10:11.972 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_host": "[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] [v2:172.21.15.171:3300/0,v1:172.21.15.171:6789/0] [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_host_override": "", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_initial_members": "", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_keyvaluedb": "rocksdb", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_lease": "5.000000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_log_full_interval": "50", 2024-08-29T15:10:11.973 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_log_max": "10000", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_log_max_summary": "50", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_log_entries_per_event": "4096", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_log_epochs": "500", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_mdsmap_epochs": "500", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_mgrmap_epochs": "500", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_nvmeof_epochs": "500", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_osd": "10000", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_pg_per_osd": "10000", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_pool_pg_num": "65536", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mds_force_trim_to": "0", 2024-08-29T15:10:11.974 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mds_skip_sanity": "true", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_memory_autotune": "true", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_memory_target": "2147483648", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mgr_beacon_grace": "30", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mgr_digest_period": "5", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mgr_inactive_grace": "60", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mgr_mkfs_grace": "240", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_min_osdmap_epochs": "500", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_op_complaint_time": "30", 2024-08-29T15:10:11.975 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_op_history_duration": "600", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_op_history_size": "20", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_op_history_slow_op_size": "20", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_op_history_slow_op_threshold": "10", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_op_log_threshold": "5", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_auto_mark_in": "false", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_auto_mark_new_in": "true", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_cache_size": "500", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_cache_size_min": "134217728", 2024-08-29T15:10:11.976 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_crush_smoke_test": "true", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_destroyed_out_interval": "600", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_down_out_interval": "600", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_force_trim_to": "0", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_full_ratio": "0.950000", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_laggy_halflife": "3600", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_laggy_max_interval": "300", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_laggy_weight": "0.300000", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_max_initial_pgs": "1024", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_min_down_reporters": "2", 2024-08-29T15:10:11.977 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_prime_pg_temp": "true", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_report_timeout": "900", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_warn_num_repaired": "10", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osd_warn_op_age": "32.000000", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osdmap_full_prune_interval": "10", 2024-08-29T15:10:11.978 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osdmap_full_prune_min": "10000", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pg_stuck_threshold": "60", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pg_warn_min_objects": "10000", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pg_warn_min_per_osd": "0", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pool_quota_crit_threshold": "0", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_pool_quota_warn_threshold": "0", 2024-08-29T15:10:11.979 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_probe_timeout": "2.000000", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_reweight_max_change": "0.050000", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_reweight_max_osds": "4", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_scrub_interval": "86400", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_scrub_max_keys": "100", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_scrub_timeout": "300", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_session_timeout": "300", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_smart_report_timeout": "5", 2024-08-29T15:10:11.980 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_stat_smooth_intervals": "6", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_stretch_pool_min_size": "2", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_stretch_pool_size": "4", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_subscribe_interval": "86400.000000", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_sync_debug": "false", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_sync_max_payload_keys": "2000", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_sync_max_payload_size": "1048576", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_sync_provider_kill_at": "0", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_sync_requester_kill_at": "0", 2024-08-29T15:10:11.981 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_sync_timeout": "60.000000", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_target_pg_per_osd": "100", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_tick_interval": "5", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_timecheck_interval": "300.000000", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_use_min_delay_socket": "false", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_older_version_delay": "604800", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_filestore_osds": "true", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-08-29T15:10:11.982 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_misplaced": "false", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_older_version": "true", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_pool_no_app": "true", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-08-29T15:10:11.983 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_on_too_few_osds": "false", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "monmap": "", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "motr_tracing_enabled": "false", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_op_threads": "3", 2024-08-29T15:10:11.984 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_buffer_size": "131072", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_cm": "false", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_device_name": "", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_dscp": "96", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_gid_idx": "0", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_local_gid": "", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_polling_us": "1000", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_port_num": "1", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-08-29T15:10:11.985 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_roce_ver": "1", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_send_buffers": "1024", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_sl": "3", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_support_srq": "true", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_rdma_type": "ib", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_async_reap_threshold": "5", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_before_connect": "false", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_ipv4": "true", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_ipv6": "false", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_msgr1": "true", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_msgr2": "true", 2024-08-29T15:10:11.986 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_port_max": "7568", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_port_min": "6800", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_prefer_ipv4": "false", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_retry_count": "3", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_bind_retry_delay": "5", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_blackhole_client": "false", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_blackhole_mds": "false", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_blackhole_mgr": "false", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_blackhole_mon": "false", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_blackhole_osd": "false", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_client_mode": "crc secure", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-08-29T15:10:11.987 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_cluster_mode": "crc secure", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_cluster_type": "", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_compress_secure": "false", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_connection_idle_timeout": "900", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_connection_ready_timeout": "10", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_crc_data": "true", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_crc_header": "true", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_die_on_bad_msg": "false", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_die_on_bug": "true", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_die_on_old_message": "true", 2024-08-29T15:10:11.988 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_die_on_skipped_message": "false", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_die_on_unhandled_msg": "false", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_coremask": "0xF", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_devs_allowlist": "", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_enable_tso": "true", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_hugepages": "", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_hw_flow_control": "true", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_lro": "true", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_memory_channel": "4", 2024-08-29T15:10:11.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_pmd": "", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_port_id": "0", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dump_corrupt_message_level": "1", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_dump_on_send": "false", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_initial_backoff": "0.200000", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_inject_delay_max": "1.000000", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_inject_delay_probability": "0.000000", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_inject_delay_type": "", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_inject_internal_delays": "0.000000", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_inject_network_congestion": "0", 2024-08-29T15:10:11.990 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_inject_socket_failures": "0", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_learn_addr_from_peer": "true", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_max_accept_failures": "4", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_max_backoff": "15.000000", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_mon_client_mode": "secure crc", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_mon_cluster_mode": "secure crc", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_mon_service_mode": "secure crc", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_osd_compress_min_size": "1024", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_osd_compress_mode": "none", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_osd_compression_algorithm": "snappy", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_pq_min_cost": "65536", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_public_type": "", 2024-08-29T15:10:11.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_service_mode": "crc secure", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_tcp_listen_backlog": "512", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_tcp_nodelay": "true", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_tcp_rcvbuf": "0", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_time_events_min_wait_interval": "1000", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "ms_type": "async+posix", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "no_config_file": "false", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "nvmeof_mon_client_tick_period": "2", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_completion_locks_per_session": "32", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_debug_inject_relock_delay": "false", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_inflight_op_bytes": "104857600", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_inflight_ops": "1024", 2024-08-29T15:10:11.992 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_inject_no_watch_ping": "false", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_tick_interval": "5.000000", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "objecter_timeout": "10.000000", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "objectstore_blackhole": "false", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "openssl_engine_opts": "", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_agent_delay_time": "5.000000", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_agent_hist_halflife": "1000", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_agent_max_low_ops": "2", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_agent_max_ops": "4", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_agent_quantize_effort": "0.100000", 2024-08-29T15:10:11.993 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_agent_slop": "0.020000", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_allow_recovery_below_min_size": "true", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_asio_thread_count": "2", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_async_recovery_min_cost": "100", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_auto_mark_unfound_lost": "false", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_backfill_retry_interval": "30.000000", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_backfill_scan_max": "512", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_backfill_scan_min": "64", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_backoff_on_degraded": "false", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_backoff_on_peering": "false", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_backoff_on_unfound": "true", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_beacon_report_interval": "300", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_bench_duration": "30", 2024-08-29T15:10:11.994 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_bench_max_block_size": "67108864", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_bench_small_size_max_iops": "100", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_blkin_trace_all": "false", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_blocked_scrub_grace_period": "120", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_check_for_log_corruption": "false", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-29T15:10:11.995 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_class_update_on_start": "true", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_client_message_cap": "256", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_client_message_size_cap": "524288000", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_client_op_priority": "63", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_client_watch_timeout": "30", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_command_max_records": "256", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_compact_on_start": "false", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_crush_chooseleaf_type": "1", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_crush_initial_weight": "-1.000000", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_crush_update_on_start": "true", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_crush_update_weight_set": "true", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-08-29T15:10:11.996 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_drop_ping_duration": "0", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_feed_pullee": "-1", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_misdirected_ops": "false", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_no_acting_change": "false", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_no_purge_strays": "false", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_op_order": "false", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_pg_log_writeout": "false", 2024-08-29T15:10:11.997 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_pretend_recovery_active": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_shutdown": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_trim_objects": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_verify_cached_snaps": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_verify_missing_on_start": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_verify_snaps": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-08-29T15:10:11.998 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_deep_scrub_keys": "1024", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_deep_scrub_stride": "524288", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_default_data_pool_replay_window": "45", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_default_notify_timeout": "30", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_delete_sleep": "0.000000", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-08-29T15:10:11.999 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_discard_disconnected_ops": "true", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_ec_partial_reads": "true", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_enable_op_tracker": "true", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_extblkdev_plugins": "vdo", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_fast_info": "true", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_fast_shutdown": "true", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_fast_shutdown_timeout": "15", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-08-29T15:10:12.000 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_function_tracing": "false", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_heartbeat_grace": "20", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_heartbeat_interval": "6", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_heartbeat_min_peers": "10", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_heartbeat_min_size": "2000", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_heartbeat_stale": "600", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_hit_set_max_size": "100000", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_hit_set_min_size": "1000", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-08-29T15:10:12.001 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_journal_flush_on_shutdown": "true", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_journal_size": "5120", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_kill_backfill_at": "0", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_loop_before_reset_tphandle": "64", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_map_cache_size": "50", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_map_dedup": "true", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_map_message_max": "40", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_map_message_max_bytes": "10485760", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_map_share_max_epochs": "40", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_attr_name_len": "100", 2024-08-29T15:10:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_attr_size": "0", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_backfills": "1", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_markdown_count": "5", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_markdown_period": "600", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_object_name_len": "2048", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_object_namespace_len": "256", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_object_size": "134217728", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_omap_entries_per_request": "1024", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_pg_blocked_by": "16", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_pg_log_entries": "10000", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_pgls": "1024", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_push_cost": "8388608", 2024-08-29T15:10:12.003 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_push_objects": "10", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_scrubs": "3", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_trimming_pgs": "2", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_write_op_reply_len": "64", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_max_write_size": "90", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-08-29T15:10:12.004 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_override_recovery_settings": "false", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_profile": "balanced", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mclock_skip_benchmark": "false", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_memory_base": "805306368", 2024-08-29T15:10:12.005 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_memory_cache_min": "134217728", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_memory_target": "4294967296", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_memory_target_autotune": "false", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_min_pg_log_entries": "250", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_min_recovery_priority": "0", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mon_heartbeat_interval": "30", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mon_report_interval": "5", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mon_report_max_in_flight": "2", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-08-29T15:10:12.006 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_num_cache_shards": "32", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_num_op_tracker_shard": "32", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_numa_auto_affinity": "true", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_numa_node": "-1", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_numa_prefer_iface": "true", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_objecter_finishers": "1", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_objectstore": "bluestore", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_objectstore_fuse": "false", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_objectstore_ideal_list_max": "64", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_objectstore_tracing": "false", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_complaint_time": "30.000000", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_history_duration": "600", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_history_size": "20", 2024-08-29T15:10:12.007 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_history_slow_op_size": "20", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_log_threshold": "5", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_num_shards": "0", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_num_shards_hdd": "5", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_num_shards_ssd": "8", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_num_threads_per_shard": "0", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_pq_min_cost": "65536", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_queue": "mclock_scheduler", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_queue_cut_off": "high", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_thread_suicide_timeout": "150", 2024-08-29T15:10:12.008 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_op_thread_timeout": "15", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_open_classes_on_start": "true", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_os_flags": "0", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_peering_op_priority": "255", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_delete_cost": "1048576", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_delete_priority": "5", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_log_dups_tracked": "3000", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_log_trim_max": "10000", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_log_trim_min": "100", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_object_context_cache_count": "64", 2024-08-29T15:10:12.009 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_crimson": "false", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_crush_rule": "-1", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_ec_fast_read": "false", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_flag_bulk": "false", 2024-08-29T15:10:12.010 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_flag_nodelete": "false", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_flags": "0", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_min_size": "0", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_pg_num": "32", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_pgp_num": "0", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_read_ratio": "70", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_size": "3", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_default_type": "replicated", 2024-08-29T15:10:12.011 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_pool_use_gmt_hitset": "true", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_push_per_object_cost": "1000", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_read_ec_check_for_errors": "false", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recover_clone_overlap": "true", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recover_clone_overlap_limit": "10", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_cost": "20971520", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_delay_start": "0.000000", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_max_active": "0", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_max_active_hdd": "3", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_max_active_ssd": "10", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_max_chunk": "8388608", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-08-29T15:10:12.012 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_max_single_start": "1", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_op_priority": "3", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_op_warn_multiple": "16", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_priority": "5", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_retry_interval": "30.000000", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_sleep": "0.000000", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_repair_during_recovery": "false", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_requested_scrub_priority": "120", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_rollback_to_cluster_snap": "", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_auto_repair": "false", 2024-08-29T15:10:12.013 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_begin_hour": "0", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_begin_week_day": "0", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_chunk_max": "25", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_chunk_min": "5", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_cost": "52428800", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_during_recovery": "false", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_end_hour": "0", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_end_week_day": "0", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_event_cost": "4096", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-08-29T15:10:12.014 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_invalid_stats": "true", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_load_threshold": "0.500000", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_max_interval": "604800.000000", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_max_preemptions": "5", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_min_interval": "86400.000000", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_priority": "5", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_scrub_sleep": "0.000000", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_shutdown_pgref_assert": "false", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_skip_data_digest": "false", 2024-08-29T15:10:12.015 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_smart_report_timeout": "5", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_snap_trim_cost": "1048576", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_snap_trim_priority": "5", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_snap_trim_sleep": "0.000000", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_stats_update_period_scrubbing": "15", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_target_transaction_size": "30", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-08-29T15:10:12.016 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_default_cache_mode": "writeback", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_tracing": "false", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_use_stale_snap": "false", 2024-08-29T15:10:12.017 INFO:teuthology.orchestra.run.smithi027.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "osdc_blkin_trace_all": "false", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_kill_at": "0", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_max_join_drift": "10", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_min": "500", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_min_wait": "0.050000", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_propose_interval": "1.000000", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_service_trim_max": "500", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_service_trim_max_multiplier": "20", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_service_trim_min": "250", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_stash_full_interval": "25", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_trim_max": "500", 2024-08-29T15:10:12.018 INFO:teuthology.orchestra.run.smithi027.stdout: "paxos_trim_min": "250", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "perf": "true", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "pid_file": "", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "public_addr": "-", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "public_addrv": "", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "public_bind_addr": "-", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "public_network": "172.21.0.0/20", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "public_network_interface": "", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "qat_compressor_busy_polling": "false", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "qat_compressor_enabled": "false", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "qat_compressor_session_max_number": "256", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "rados_mon_op_timeout": "0", 2024-08-29T15:10:12.019 INFO:teuthology.orchestra.run.smithi027.stdout: "rados_osd_op_timeout": "0", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rados_tracing": "false", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_atime_update_interval": "60", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_balance_parent_reads": "false", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_balance_snap_reads": "false", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_blkin_trace_all": "false", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_blocklist_expire_seconds": "0", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_blocklist_on_break_lock": "true", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache": "true", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_block_writes_upfront": "false", 2024-08-29T15:10:12.020 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_max_dirty": "25165824", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_max_dirty_object": "0", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_policy": "writearound", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_size": "33554432", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_target_dirty": "16777216", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_clone_copy_on_read": "false", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_compression_hint": "none", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_concurrent_management_ops": "10", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_clone_format": "auto", 2024-08-29T15:10:12.021 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_data_pool": "", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_features": "61", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_format": "2", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_map_options": "", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_order": "22", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_pool": "rbd", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_stripe_count": "0", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_default_stripe_unit": "0", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_disable_zero_copy_writes": "true", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_discard_granularity_bytes": "65536", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_enable_alloc_hint": "true", 2024-08-29T15:10:12.022 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_io_scheduler": "simple", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_commit_age": "5.000000", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_object_flush_interval": "0", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-08-29T15:10:12.023 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_order": "24", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_pool": "", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_journal_splay_width": "4", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_localize_parent_reads": "false", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_localize_snap_reads": "false", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-08-29T15:10:12.024 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_image_policy_type": "simple", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-08-29T15:10:12.025 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_memory_autotune": "true", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_memory_base": "805306368", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_memory_target": "4294967296", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirroring_delete_delay": "0", 2024-08-29T15:10:12.026 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirroring_replay_delay": "0", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_move_to_trash_on_remove": "false", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_mtime_update_interval": "60", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_non_blocking_aio": "true", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_op_thread_timeout": "60", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_op_threads": "1", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_parent_cache_enabled": "false", 2024-08-29T15:10:12.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_persistent_cache_mode": "disabled", 2024-08-29T15:10:12.028 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_persistent_cache_path": "/tmp", 2024-08-29T15:10:12.028 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_persistent_cache_size": "1073741824", 2024-08-29T15:10:12.028 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_plugins": "", 2024-08-29T15:10:12.028 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_bps_burst": "0", 2024-08-29T15:10:12.028 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_bps_limit": "0", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_exclude_ops": "0", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_iops_burst": "0", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_iops_limit": "0", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_read_bps_burst": "0", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_read_bps_limit": "0", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_read_iops_burst": "0", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-08-29T15:10:12.029 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_read_iops_limit": "0", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_schedule_tick_min": "50", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_write_bps_burst": "0", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_write_bps_limit": "0", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_write_iops_burst": "0", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_qos_write_iops_limit": "0", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_quiesce_notification_attempts": "10", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_read_from_replica_policy": "default", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_readahead_max_bytes": "524288", 2024-08-29T15:10:12.030 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_readahead_trigger_requests": "10", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_request_timed_out_seconds": "30", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_skip_partial_discard": "true", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_tracing": "false", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_validate_names": "true", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rbd_validate_pool": "true", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "restapi_base_url": "", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "restapi_log_level": "", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_account_default_quota_max_size": "-1", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_acl_grants_max_num": "100", 2024-08-29T15:10:12.031 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_admin_entry": "admin", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_asio_assert_yielding": "false", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_backend_store": "rados", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_barbican_url": "", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_beast_enable_async": "true", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_counters_cache": "false", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_index_max_aio": "128", 2024-08-29T15:10:12.032 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_quota_ttl": "600", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_cache_enabled": "true", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_cache_expiry_interval": "900", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_cache_lru_size": "25000", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_config_store": "rados", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_content_length_compat": "false", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_copy_obj_progress": "true", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_cors_rules_max_num": "100", 2024-08-29T15:10:12.033 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_cross_domain_policy": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_default_encryption_key": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_addr": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_ca_path": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_client_cert": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_client_key": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_password": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_kmip_username": "", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_require_ssl": "true", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-08-29T15:10:12.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_suppress_logs": "true", 2024-08-29T15:10:12.035 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_addr": "", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_auth": "token", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_namespace": "", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_prefix": "", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_token_file": "", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_curl_buffersize": "524288", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_curl_low_speed_limit": "1024", 2024-08-29T15:10:12.036 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_curl_low_speed_time": "300", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_curl_tcp_keepalive": "0", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_l1_fadvise": "4", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-08-29T15:10:12.037 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_l1_fadvise": "4", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data_log_changes_size": "1000", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data_log_num_shards": "128", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data_log_window": "30", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data_notify_interval_msec": "0", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data_sync_poll_interval": "20", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_data_sync_spawn_window": "20", 2024-08-29T15:10:12.038 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_default_data_log_backing": "fifo", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_default_region_info_oid": "default.region", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_defer_to_bucket_acls": "", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_disable_s3select": "false", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-08-29T15:10:12.039 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_data_res": "500.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dns_name": "", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dns_s3website_name": "", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dynamic_resharding": "true", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-08-29T15:10:12.040 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_gc_threads": "true", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_lc_threads": "true", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_mdsearch": "true", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_ops_log": "false", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_quota_threads": "true", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_static_website": "false", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enable_usage_log": "false", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_enforce_swift_acls": "true", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_exit_timeout_secs": "120", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_expose_bucket": "false", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_extended_http_attrs": "", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_filter": "none", 2024-08-29T15:10:12.041 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_frontends": "beast port=7480", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_max_concurrent_io": "10", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_max_deferred": "50", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_max_objs": "32", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_max_queue_size": "134213632", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_max_trim_chunk": "16", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_obj_min_wait": "7200", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_processor_max_time": "3600", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_gc_processor_period": "3600", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_get_obj_window_size": "16777216", 2024-08-29T15:10:12.042 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_healthcheck_disabling_path": "", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ignore_get_invalid_range": "false", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_init_timeout": "300", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_kafka_connection_idle": "300", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_kafka_message_timeout": "5000", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_kafka_sleep_timeout": "10", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-08-29T15:10:12.043 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_domain": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_password": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_password_path": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_project": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_tenant": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_token": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_token_path": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_admin_user": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_barbican_domain": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_barbican_password": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_barbican_project": "", 2024-08-29T15:10:12.044 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_barbican_tenant": "", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_barbican_user": "", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_implicit_tenants": "false", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_service_token_enabled": "false", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_token_cache_size": "10000", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_url": "", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_keystone_verify_ssl": "true", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lc_debug_interval": "-1", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lc_lock_max_time": "90", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lc_max_objs": "32", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lc_max_rules": "1000", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lc_max_worker": "3", 2024-08-29T15:10:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lc_max_wp_worker": "3", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lc_thread_delay": "0", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ldap_dnattr": "uid", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ldap_searchfilter": "", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ldap_uri": "", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lfuda_sync_frequency": "60", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_log_http_headers": "", 2024-08-29T15:10:12.046 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_log_nonexistent_bucket": "false", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_log_object_name_utc": "false", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_attr_name_len": "0", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_attr_size": "0", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_attrs_num_in_req": "0", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_chunk_size": "4194304", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_concurrent_requests": "1024", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_dynamic_shards": "1999", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_listing_results": "1000", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_notify_retries": "10", 2024-08-29T15:10:12.047 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_objs_per_shard": "100000", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_put_param_size": "1048576", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_put_size": "5368709120", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_max_slo_entries": "1000", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_md_log_max_shards": "64", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_md_notify_interval_msec": "200", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_meta_sync_poll_interval": "20", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_meta_sync_spawn_window": "20", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_mp_lock_max_time": "600", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_multipart_min_part_size": "5242880", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-08-29T15:10:12.048 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_fhcache_size": "2017", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_lru_lanes": "5", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_max_gc": "300", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_run_gc_threads": "false", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_run_lc_threads": "false", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_run_quota_threads": "false", 2024-08-29T15:10:12.049 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_run_sync_thread": "false", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_num_async_rados_threads": "32", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_num_control_oids": "8", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_numa_node": "-1", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_obj_stripe_size": "4194304", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_objexp_chunk_size": "100", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_objexp_gc_interval": "600", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_objexp_hints_num_shards": "127", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-08-29T15:10:12.050 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_op_thread_timeout": "600", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_op_tracing": "false", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_opa_token": "", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_opa_url": "", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_opa_verify_ssl": "true", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ops_log_rados": "false", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_ops_log_socket_path": "", 2024-08-29T15:10:12.051 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_period_push_interval": "2.000000", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_period_push_interval_max": "30.000000", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_period_root_pool": ".rgw.root", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_posix_cache_lanes": "3", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_posix_cache_max_buckets": "100", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_posix_cache_partitions": "3", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-08-29T15:10:12.052 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_print_continue": "true", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_print_prohibited_content_length": "false", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_rados_tracing": "false", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_realm": "", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_realm_id": "", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_region": "", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_region_root_pool": ".rgw.root", 2024-08-29T15:10:12.053 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_relaxed_region_enforcement": "false", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_request_uri": "", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_reshard_batch_size": "64", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_reshard_debug_interval": "-1", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_reshard_max_aio": "128", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_reshard_num_logs": "16", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_reshard_thread_interval": "600", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_resolve_cname": "false", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_rest_getusage_op_compat": "false", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_run_sync_thread": "true", 2024-08-29T15:10:12.054 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_auth_use_keystone": "false", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_auth_use_ldap": "false", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_auth_use_rados": "true", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_auth_use_sts": "false", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_s3_success_create_obj_status": "0", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_scheduler_type": "throttler", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_script_uri": "", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_service_provider_name": "", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-08-29T15:10:12.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sts_client_id": "", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sts_client_secret": "", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sts_entry": "sts", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sts_key": "sts", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sts_max_session_duration": "43200", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sts_min_session_duration": "900", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sts_token_introspection_url": "", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_account_in_url": "false", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_auth_entry": "auth", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_auth_url": "", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_custom_header": "", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_enforce_content_length": "false", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_need_stats": "true", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_tenant_name": "", 2024-08-29T15:10:12.056 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_token_expiration": "86400", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_url": "", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_url_prefix": "swift", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_swift_versioning_enabled": "false", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_lease_period": "120", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_log_trim_interval": "1200", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_obj_etag_verify": "false", 2024-08-29T15:10:12.057 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_trace_history_size": "4096", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_thread_pool_size": "512", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_topic_persistency_max_retries": "0", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_topic_require_publish_policy": "false", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_comment": "", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_createby": "", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_encoding": "", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_flag": "false", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_max_size": "5368709120", 2024-08-29T15:10:12.058 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_origin": "", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_sha_unit": "524288", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_torrent_tracker": "", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_trust_forwarded_https": "false", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_usage_log_tick_interval": "30", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_usage_max_shards": "32", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_usage_max_user_shards": "1", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_use_opa_authz": "false", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_counters_cache": "false", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_counters_cache_size": "10000", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_default_quota_max_size": "-1", 2024-08-29T15:10:12.059 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_max_buckets": "1000", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_policies_max_num": "100", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_quota_sync_interval": "86400", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_user_unique_email": "true", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_verify_ssl": "true", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_website_routing_rules_max_num": "50", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_zone": "", 2024-08-29T15:10:12.060 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_zone_id": "", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_zonegroup": "", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_zonegroup_id": "", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rgwlc_auto_session_clear": "true", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rgwlc_skip_bucket_step": "false", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_block_size": "4096", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cache_shard_bits": "4", 2024-08-29T15:10:12.061 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cache_size": "536870912", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cache_type": "binned_lru", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_collect_compaction_stats": "false", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_collect_extended_stats": "false", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_collect_memory_stats": "false", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_index_type": "binary_search", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_log_to_ceph_log": "true", 2024-08-29T15:10:12.062 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_metadata_block_size": "4096", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_partition_filters": "false", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_perf": "false", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "rotating_keys_renewal_timeout": "10", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "run_dir": "/var/run/ceph", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_block_create": "true", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_cache_lru_size": "67108864", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_cbjournal_size": "5368709120", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_data_delta_based_overwrite": "0", 2024-08-29T15:10:12.063 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_default_max_object_size": "16777216", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_device_size": "53687091200", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_full_integrity_check": "false", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_journal_batch_capacity": "16", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_journal_iodepth_limit": "5", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_main_device_type": "SSD", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_max_concurrent_transactions": "8", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_max_data_allocation_size": "32768", 2024-08-29T15:10:12.064 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "seastore_segment_size": "67108864", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "set_keepcaps": "false", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "setgroup": "ceph", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "setuser": "ceph", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "setuser_match_path": "", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "target_max_misplaced_ratio": "0.050000", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "thp": "false", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "threadpool_default_timeout": "60", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "threadpool_empty_queue_max_wait": "2", 2024-08-29T15:10:12.065 INFO:teuthology.orchestra.run.smithi027.stdout: "throttler_perf_counter": "true", 2024-08-29T15:10:12.066 INFO:teuthology.orchestra.run.smithi027.stdout: "tmp_dir": "/tmp", 2024-08-29T15:10:12.066 INFO:teuthology.orchestra.run.smithi027.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-08-29T15:10:12.066 INFO:teuthology.orchestra.run.smithi027.stdout: "uadk_compressor_enabled": "false", 2024-08-29T15:10:12.066 INFO:teuthology.orchestra.run.smithi027.stdout: "uadk_wd_sync_ctx_num": "2" 2024-08-29T15:10:12.066 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:10:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:12 smithi171 ceph-mon[90031]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:12 smithi027 ceph-mon[93921]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:12 smithi027 ceph-mon[98970]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:12.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-08-29T15:10:12.987 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:14 smithi027 ceph-mon[93921]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:14 smithi027 ceph-mon[98970]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:14 smithi171 ceph-mon[90031]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:14.609 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a: {} 2024-08-29T15:10:14.609 INFO:teuthology.orchestra.run.smithi027.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-29T15:10:14.619 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b: {} 2024-08-29T15:10:14.619 INFO:teuthology.orchestra.run.smithi027.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-29T15:10:14.627 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c: {} 2024-08-29T15:10:14.628 INFO:teuthology.orchestra.run.smithi027.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-29T15:10:15.223 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-08-29T15:10:15.223 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-08-29T15:10:15.224 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-29T15:10:15.229 INFO:tasks.radosbench:Beginning radosbench... 2024-08-29T15:10:15.229 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-08-29T15:10:15.230 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-08-29T15:10:15.230 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-08-29T15:10:15.230 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:10:15.230 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:10:15.230 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-08-29T15:10:15.231 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:10:15.231 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:10:15.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-08-29T15:10:15.231 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-08-29T15:10:15.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:15.232 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-08-29T15:10:15.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:10:15.233 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-08-29T15:10:15.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:10:15.233 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-08-29T15:10:15.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:10:15.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:15.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:15.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:15.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:15 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2848461704' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-29T15:10:15.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:15 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2848461704' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-29T15:10:15.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:15.399 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:15.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:15.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:15.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:15 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2848461704' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-29T15:10:15.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:15.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:15.538 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:15.538 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:15.541 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:15.558 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:15.585 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:15.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:15.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:15.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:15.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:15.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:15.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:15.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:15.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:15.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:15.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:16.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:16.037 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:16.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:16.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:16.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:16 smithi027 ceph-mon[98970]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:16.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:16 smithi027 ceph-mon[93921]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:16.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:16.288 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:16.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:16.414 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:16.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:16.517 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:16.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:16 smithi171 ceph-mon[90031]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:16.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:16.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:16.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:16.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:16.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:16.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:16.797 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:16.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:16.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:16.926 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:16.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:17.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:17.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:17.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:17.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:17.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:17.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:17.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:17.304 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:17.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:17.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:17.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:17.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:17.560 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:17.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:17.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:17.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:17.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:17.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:17.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:17.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:17.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:17.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:18.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:18.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:18.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:18.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:18.301 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:18 smithi027 ceph-mon[93921]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:18.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:18 smithi027 ceph-mon[98970]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:18.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:18.429 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:18.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:18 smithi171 ceph-mon[90031]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:18.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:18.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:18.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:18.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:18.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:18.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:18.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:18.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:18.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:18.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:18.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:18.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:19.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:19.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:19.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:19.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:10:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:10:19] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.51.0" 2024-08-29T15:10:19.326 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:19.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:19.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:19.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:19.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:19.611 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:19.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:19.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:19.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:19.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:19.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.867 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:19.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:19.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:19.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:19.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:20.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:20.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:20.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:20.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:20.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:20.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:20.293 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:20 smithi027 ceph-mon[93921]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:20.293 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:20 smithi027 ceph-mon[98970]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:20.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:20.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:20.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:20.474 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:20.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:20.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:10:20.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:10:20.506 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:10:20.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:20 smithi171 ceph-mon[90031]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:20.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:20.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:20.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:20.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:20.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:20.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:20.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:20.879 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:20.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:20.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:20.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.007 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:21.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:21.052 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_0' created 2024-08-29T15:10:21.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:21.133 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:21.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:21.260 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2159170475' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1557316497' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-29T15:10:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:21 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-29T15:10:21.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2159170475' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1557316497' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-29T15:10:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:21 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-29T15:10:21.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:21.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.386 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:21.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:21.488 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:21.514 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:21.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:21.516 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2159170475' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1557316497' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-29T15:10:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:21 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-29T15:10:21.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:21.638 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:21.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:21.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:21.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:21.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-08-29T15:10:21.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:21.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:21.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:21.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:21.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:22.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:22.052 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:10:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[93921]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-29T15:10:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[93921]: osdmap e111: 8 total, 8 up, 8 in 2024-08-29T15:10:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3112668704' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:10:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:10:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[98970]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:22.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-29T15:10:22.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[98970]: osdmap e111: 8 total, 8 up, 8 in 2024-08-29T15:10:22.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3112668704' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:10:22.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:10:22.120 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:22.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:22.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:22.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:22.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:22.373 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:22.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:22.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:22.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:22 smithi171 ceph-mon[90031]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-29T15:10:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:22 smithi171 ceph-mon[90031]: osdmap e111: 8 total, 8 up, 8 in 2024-08-29T15:10:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3112668704' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:10:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:10:22.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:22.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:22.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:22.649 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:22.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:22.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:22.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:22.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:22.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:22.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:22.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:22.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:23.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:23.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:23.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:23.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:23.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:23.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:23.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:23.282 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:23 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:23 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:10:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:23 smithi027 ceph-mon[93921]: osdmap e112: 8 total, 8 up, 8 in 2024-08-29T15:10:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:23 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:10:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:23 smithi027 ceph-mon[98970]: osdmap e112: 8 total, 8 up, 8 in 2024-08-29T15:10:23.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:23.411 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:23.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:23.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:23 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:10:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:23 smithi171 ceph-mon[90031]: osdmap e112: 8 total, 8 up, 8 in 2024-08-29T15:10:23.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:23.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:23.640 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:23.665 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:23.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:23.793 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:23.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:23.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:23.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:23.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:24.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:24.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:24.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:24.172 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:24.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:24.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:24.299 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:24 smithi027 ceph-mon[93921]: pgmap v292: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:24 smithi027 ceph-mon[93921]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:10:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:24 smithi027 ceph-mon[93921]: osdmap e113: 8 total, 8 up, 8 in 2024-08-29T15:10:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:24 smithi027 ceph-mon[98970]: pgmap v292: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:24 smithi027 ceph-mon[98970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:10:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:24 smithi027 ceph-mon[98970]: osdmap e113: 8 total, 8 up, 8 in 2024-08-29T15:10:24.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:24.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:10:24.411 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:24.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:24.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:24 smithi171 ceph-mon[90031]: pgmap v292: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 244 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:24 smithi171 ceph-mon[90031]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:10:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:24 smithi171 ceph-mon[90031]: osdmap e113: 8 total, 8 up, 8 in 2024-08-29T15:10:24.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:24.552 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:24.654 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:24.680 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:24.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:24.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:24.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:24.906 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:24.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:24.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:24.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:25.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:25.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:25.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:25.173 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:10:25.173 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:10:25.174 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:10:25.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:25.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:25.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:25.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:25.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:25.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:25.432 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:25.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:25.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:25.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:25.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.681 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:25.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:25.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:25.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:25.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:25.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:25.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:25.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:25.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:26.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:26.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:26.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:26.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:26.236 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:10:26.236 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:10:26.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:26.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:26.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:26 smithi027 ceph-mon[93921]: pgmap v294: 89 pgs: 1 creating+peering, 1 unknown, 87 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:26 smithi027 ceph-mon[98970]: pgmap v294: 89 pgs: 1 creating+peering, 1 unknown, 87 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:26.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:10:26.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:26.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:26.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:26 smithi171 ceph-mon[90031]: pgmap v294: 89 pgs: 1 creating+peering, 1 unknown, 87 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:26.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:26.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:26.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:26.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:26.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:26.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:26.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:26.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:26.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:26.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:26.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:26.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:10:27.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:27.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:27.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:27.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:27.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:27.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:27.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:27.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:27.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:27.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:27.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:27.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:27.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:27.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:27.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:27.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:27.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:27.924 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:27.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:27.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:27.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:27.946 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:28.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:28.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:28.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:28.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:28.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:28 smithi027 ceph-mon[98970]: pgmap v295: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:28.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:28 smithi027 ceph-mon[93921]: pgmap v295: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:28.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:28.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:28.319 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:28 smithi171 ceph-mon[90031]: pgmap v295: 89 pgs: 1 creating+peering, 88 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:28.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:28.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:28.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:28.444 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:28.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:28.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:28.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:28.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:28.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:28.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:28.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:28.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:28.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:28.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:28.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:29.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:29.067 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:29.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:29.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:29.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:29.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:29.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:10:29] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.51.0" 2024-08-29T15:10:29.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:29.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:29.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:29.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:29.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:29.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:29.567 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:29.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:29.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:29.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:29.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:29.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:29.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:29.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[98970]: pgmap v296: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:29.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:29.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:29.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/621940358' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:10:29.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:10:29.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[93921]: pgmap v296: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:29.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:29.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:29.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/621940358' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:10:29.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:29 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:10:29.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:29.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:29.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:29.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:29 smithi171 ceph-mon[90031]: pgmap v296: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/621940358' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:10:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:29 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:10:30.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:30.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:30.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:30.071 INFO:teuthology.orchestra.run.smithi027.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-08-29T15:10:30.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:30.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:30.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:30.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.318 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:30.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:30.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:30.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:30.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:30.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:30.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:30.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:30.687 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:30.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:30.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:30.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:30.912 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:30.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:30.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:31.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:31.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:31.072 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:10:31.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:31.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:31.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:31.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:10:31.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:31 smithi027 ceph-mon[98970]: osdmap e114: 8 total, 8 up, 8 in 2024-08-29T15:10:31.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4104466370' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:31.217 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:10:31.217 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:31 smithi027 ceph-mon[93921]: osdmap e114: 8 total, 8 up, 8 in 2024-08-29T15:10:31.217 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4104466370' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:31.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:31.306 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:31.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:31.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:10:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:31 smithi171 ceph-mon[90031]: osdmap e114: 8 total, 8 up, 8 in 2024-08-29T15:10:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4104466370' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:31.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:31.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:31.558 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:10:31.558 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:10:31.575 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:10:31.575 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:10:31.591 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:10:31.592 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:10:31.607 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:10:31.608 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:10:31.624 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:10:31.625 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:10:31.642 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:10:31.643 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:10:31.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:31.659 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:10:31.660 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:10:31.677 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:10:31.678 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:10:31.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:31.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:31.786 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:31.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:31.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:31.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:31.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:31.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:32.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:32.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:32.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:32.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:32.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:32.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:32.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:32.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:32.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[98970]: pgmap v298: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[98970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4104466370' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[98970]: osdmap e115: 8 total, 8 up, 8 in 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/798669192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[93921]: pgmap v298: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[93921]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:32.380 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4104466370' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:10:32.381 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[93921]: osdmap e115: 8 total, 8 up, 8 in 2024-08-29T15:10:32.381 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/798669192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:32.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:32.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:32.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:32 smithi171 ceph-mon[90031]: pgmap v298: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:32 smithi171 ceph-mon[90031]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:10:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:32 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4104466370' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:10:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:32 smithi171 ceph-mon[90031]: osdmap e115: 8 total, 8 up, 8 in 2024-08-29T15:10:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/798669192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:32.523 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:10:32.523 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:10:32.523 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-08-29T15:10:32.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-08-29T15:10:32.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:32.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:32.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:32.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:32.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:32.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:32.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:32.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:32.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:32.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:33.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:33.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:33.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:33.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:33.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:33.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:33.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:33.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:33.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:33.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:33.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:33.400 DEBUG:teuthology.orchestra.run.smithi027:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-08-29T15:10:33.403 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-29T15:10:33.403 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-29T15:10:33.447 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:33.455 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:hints = 1 2024-08-29T15:10:33.455 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-29T15:10:33.455 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object prefix: benchmark_data_smithi027_135490 2024-08-29T15:10:33.455 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:10:33.455 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 0 0 0 0 0 0 - 0 2024-08-29T15:10:33.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:33.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:33.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:33.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:33.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:33.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:33.754 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:33.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:33.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:33.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:33.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:33.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:33.986 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:33.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:34.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:34.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:34.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:34.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:34.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:34.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:34.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:34.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:34.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:34.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:34 smithi027 ceph-mon[93921]: pgmap v300: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:34.400 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:34 smithi027 ceph-mon[98970]: pgmap v300: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:34.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:34.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:34.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:34.455 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 1 16 4040 4024 251.486 251.5 0.00304083 0.00396426 2024-08-29T15:10:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:34 smithi171 ceph-mon[90031]: pgmap v300: 89 pgs: 89 active+clean; 4.6 MiB data, 248 MiB used, 715 GiB / 715 GiB avail 2024-08-29T15:10:34.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:34.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:34.598 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:34.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:34.699 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:34.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:34.736 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:34.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:34.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:10:34.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:35.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:35.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:35.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:35.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:35.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:10:35.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:35.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:35.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:35.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:35.456 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 2 15 7374 7359 229.918 208.438 0.00353528 0.00433946 2024-08-29T15:10:35.544 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:35.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:35.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:35.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:35.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:35.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:35.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:35.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:35.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:35.998 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:35.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:36.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:36.134 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:36.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:36.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:36.236 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:36.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:36.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:36 smithi027 ceph-mon[93921]: pgmap v301: 89 pgs: 89 active+clean; 94 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 179 op/s 2024-08-29T15:10:36.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:36 smithi027 ceph-mon[98970]: pgmap v301: 89 pgs: 89 active+clean; 94 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 179 op/s 2024-08-29T15:10:36.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:36.411 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:36.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:36.456 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 3 16 10777 10761 224.148 212.625 0.0039737 0.00445507 2024-08-29T15:10:36.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:36 smithi171 ceph-mon[90031]: pgmap v301: 89 pgs: 89 active+clean; 94 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 179 op/s 2024-08-29T15:10:36.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:36.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:36.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:36.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:36.684 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:36.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:36.786 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:36.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:36.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:36.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:36.912 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:10:36.913 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:10:36.913 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:10:36.913 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:10:36.913 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:10:36.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:36.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:36.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:36.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:37.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:37.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:37.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:37.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:37.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:37.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:37.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:37.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:37.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:37.456 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 4 16 13897 13881 216.857 195 0.00352962 0.00460634 2024-08-29T15:10:37.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:37.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:37.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:37.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:37.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:37.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:37.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:37.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:10:37.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:37.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:37.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:37.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:37.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:37.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:37.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:37.985 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:38.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:38.109 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:10:38.110 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:10:38.111 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:10:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:38.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:38.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:38.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:38.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:38.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:38.355 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:38.386 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:38.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:38.456 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 5 16 16879 16863 210.758 186.375 0.0044097 0.00473869 2024-08-29T15:10:38.488 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:38.489 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:38 smithi027 ceph-mon[98970]: pgmap v302: 89 pgs: 89 active+clean; 120 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s 2024-08-29T15:10:38.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:38 smithi027 ceph-mon[93921]: pgmap v302: 89 pgs: 89 active+clean; 120 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s 2024-08-29T15:10:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:38 smithi171 ceph-mon[90031]: pgmap v302: 89 pgs: 89 active+clean; 120 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s wr, 231 op/s 2024-08-29T15:10:38.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:38.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:38.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:38.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:38.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:38.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:38.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:38.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:38.794 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:38.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:38.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:38.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:38.930 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:38.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:38.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:38.965 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:10:38.966 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:10:38.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-08-29T15:10:39.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:39.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:39.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:39.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:39.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:39.213 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:10:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:10:39] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.51.0" 2024-08-29T15:10:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:39.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:39.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:39.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:39.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:39.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:39.456 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 6 16 19813 19797 206.191 183.375 0.0037882 0.00484459 2024-08-29T15:10:39.468 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:39.505 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:39.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:39.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:39.607 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:39.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:39.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:39.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:39.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:39.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:39.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:39.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:39.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:39.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:40.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:40.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:40.084 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:40.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:40.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:40.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:40.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:40 smithi027 ceph-mon[93921]: pgmap v303: 89 pgs: 89 active+clean; 475 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-29T15:10:40.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:40 smithi027 ceph-mon[98970]: pgmap v303: 89 pgs: 89 active+clean; 475 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-29T15:10:40.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:40.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:40.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:40.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:40.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:40.456 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 7 16 22155 22139 197.644 146.375 0.00170322 0.00503186 2024-08-29T15:10:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:40 smithi171 ceph-mon[90031]: pgmap v303: 89 pgs: 89 active+clean; 475 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-29T15:10:40.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:40.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:40.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:40.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:40.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:40.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:40.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:40.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:40.947 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:40.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:40.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:41.075 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:41.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:41.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:41.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:41.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:41.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:41.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:41.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:41.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:41.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:41.457 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 8 16 24085 24069 188.016 120.625 0.0012747 0.00528868 2024-08-29T15:10:41.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:41.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:41.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:41.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:41.632 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:41.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:41.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:41.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:41.765 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:41.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:41.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:41.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:41.904 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:41.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:41.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:42.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:42.046 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:42.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:42.148 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:42.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:42.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:42.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:42.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:42.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:42.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:42.412 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:42 smithi171 ceph-mon[90031]: pgmap v304: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-08-29T15:10:42.413 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:10:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=cleanup t=2024-08-29T15:10:42.201231352Z level=info msg="Completed cleanup jobs" duration=4.439666ms 2024-08-29T15:10:42.413 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:10:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana.update.checker t=2024-08-29T15:10:42.352339076Z level=info msg="Update check succeeded" duration=68.373996ms 2024-08-29T15:10:42.425 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:42.457 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 9 16 26397 26381 183.18 144.5 0.00428977 0.00545406 2024-08-29T15:10:42.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:42.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:42.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:42.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:42 smithi027 ceph-mon[98970]: pgmap v304: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-08-29T15:10:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:42 smithi027 ceph-mon[93921]: pgmap v304: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-08-29T15:10:42.598 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:42.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:42.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:42.700 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:42.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:42.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:42.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:42.767 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:10:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugins.update.checker t=2024-08-29T15:10:42.412492991Z level=info msg="Update check succeeded" duration=122.942497ms 2024-08-29T15:10:42.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:42.871 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:42.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:42.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:42.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:43.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:43.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:43.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.147 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:43.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:43.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:43.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:43.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:43.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:43.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:43.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:43.458 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 10 16 28859 28843 180.231 153.875 0.00342568 0.00554208 2024-08-29T15:10:43.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:43.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:43.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:43.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:43.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:43.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:43.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:43.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:43.971 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:43.977 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:43.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:43.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:44.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:44.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:44.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:44.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:44.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:44.249 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:44.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:44.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:44.350 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:44 smithi027 ceph-mon[98970]: pgmap v305: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-29T15:10:44.350 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:44.351 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:44.351 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:44 smithi027 ceph-mon[93921]: pgmap v305: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-29T15:10:44.351 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:44.351 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:44.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:44.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:44.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:44.458 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 11 16 31263 31247 177.504 150.25 0.00574928 0.00562824 2024-08-29T15:10:44.490 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:44 smithi171 ceph-mon[90031]: pgmap v305: 89 pgs: 89 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-08-29T15:10:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:44.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:44.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:44.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:44.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:44.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:44.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:44.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:44.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:44.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:44.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:44.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:44.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:45.026 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:45.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:45.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:45.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:45.217 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:45.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:45.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:45.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:45.281 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:45 smithi027 ceph-mon[93921]: pgmap v306: 89 pgs: 89 active+clean; 1.8 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-29T15:10:45.281 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:45 smithi027 ceph-mon[98970]: pgmap v306: 89 pgs: 89 active+clean; 1.8 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-29T15:10:45.352 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:45.388 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:45.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:45.458 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 12 16 33582 33566 174.789 144.938 0.00792078 0.00571592 2024-08-29T15:10:45.489 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:45 smithi171 ceph-mon[90031]: pgmap v306: 89 pgs: 89 active+clean; 1.8 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-29T15:10:45.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:45.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:45.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:45.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:45.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:45.654 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:45.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:45.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:45.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:45.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:45.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:45.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:45.931 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:45.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:46.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:46.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:46.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:46.136 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:10:46.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:46.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:46.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:46.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:46.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:46.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:46.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:46.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:46.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/435718489' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:10:46.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:46 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:10:46.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/435718489' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:10:46.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:46 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:10:46.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:46.458 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 13 16 35755 35739 171.791 135.812 0.00285932 0.00581577 2024-08-29T15:10:46.485 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:46.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/435718489' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:10:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:46 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:10:46.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:46.623 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:46.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:46.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:46.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:46.761 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:46.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:46.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:46.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:46.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:46.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:46.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:47.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:47.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:47.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:47.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:47.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:47.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:47.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:47.309 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:47.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:47.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:47.443 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:47.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:47.458 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 14 16 38067 38051 169.84 144.5 0.00830125 0.00588243 2024-08-29T15:10:47.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:47.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:10:47.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[93921]: osdmap e116: 8 total, 8 up, 8 in 2024-08-29T15:10:47.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[93921]: pgmap v308: 89 pgs: 89 active+clean; 1.8 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-08-29T15:10:47.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:47.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:10:47.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[98970]: osdmap e116: 8 total, 8 up, 8 in 2024-08-29T15:10:47.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:47 smithi027 ceph-mon[98970]: pgmap v308: 89 pgs: 89 active+clean; 1.8 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-08-29T15:10:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:47 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:10:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:47 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:10:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:47 smithi171 ceph-mon[90031]: osdmap e116: 8 total, 8 up, 8 in 2024-08-29T15:10:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:47 smithi171 ceph-mon[90031]: pgmap v308: 89 pgs: 89 active+clean; 1.8 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 173 MiB/s wr, 2.76k op/s 2024-08-29T15:10:47.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:47.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:47.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:47.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:47.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:47.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:47.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:47.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:47.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:47.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:47.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:48.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.003 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:48.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:48.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:48.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:48.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:48.282 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:48.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:48.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:48.458 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 15 16 40232 40216 167.538 135.312 0.00844882 0.0059637 2024-08-29T15:10:48.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:48.556 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:48.635 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:10:48.635 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:10:48.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:48.673 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:10:48.673 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:10:48.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:48.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:48.708 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:10:48.708 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:10:48.741 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:10:48.742 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:10:48.780 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:10:48.780 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:10:48.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:48.813 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:10:48.814 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:10:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:48.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:48.845 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:10:48.845 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:10:48.878 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:10:48.879 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:10:48.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:48.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:48.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:48.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:49.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:49.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:49.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:49.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:49.229 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:49.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:49.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:49.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:10:49.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:10:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:10:49] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-29T15:10:49.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:49.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:49.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:49.458 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 16 16 42323 42307 165.234 130.688 0.00255145 0.0060467 2024-08-29T15:10:49.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:49.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:49.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:49 smithi027 ceph-mon[93921]: pgmap v309: 89 pgs: 89 active+clean; 2.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-08-29T15:10:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/660645120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:49.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:49 smithi027 ceph-mon[98970]: pgmap v309: 89 pgs: 89 active+clean; 2.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-08-29T15:10:49.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:49 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/660645120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:49.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:49.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:49.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:49 smithi171 ceph-mon[90031]: pgmap v309: 89 pgs: 89 active+clean; 2.1 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 164 MiB/s wr, 2.62k op/s 2024-08-29T15:10:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/660645120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:49.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:49.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:49.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:49.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:49.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:49.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:50.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:50.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:50.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:50.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:50.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:50.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:50.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:50.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:50.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:50.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:50.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:50.459 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 17 16 44429 44413 163.257 131.625 0.00603551 0.00612011 2024-08-29T15:10:50.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:50.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:50.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:50.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:50.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:50.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:50.772 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:50.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:50.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:50.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:50.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:51.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:51.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:51.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:51.150 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:51.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:51.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:51.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:51.215 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:51.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:51.309 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:10:51.309 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:10:51.309 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:10:51.309 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:10:51.309 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:10:51.310 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:10:51.310 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:10:51.310 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:10:51.310 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:10:51.310 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:10:51.310 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:10:51.310 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:10:51.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:51.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:51.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:51.459 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 18 16 46267 46251 160.562 114.875 0.00953687 0.00622298 2024-08-29T15:10:51.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:51.475 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:51.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:51.513 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:10:51.513 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:10:51.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:51 smithi027 ceph-mon[98970]: pgmap v310: 89 pgs: 89 active+clean; 2.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-29T15:10:51.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:51 smithi027 ceph-mon[93921]: pgmap v310: 89 pgs: 89 active+clean; 2.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-29T15:10:51.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:51.611 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:51.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:51.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:51.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:51.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:51.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:51 smithi171 ceph-mon[90031]: pgmap v310: 89 pgs: 89 active+clean; 2.5 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-29T15:10:51.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:51.878 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:10:51.889 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:51.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:51.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:52.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:52.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:52.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:52.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:52.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:52.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:52.165 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:10:52.165 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:10:52.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-08-29T15:10:52.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:52.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:52.290 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:52.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:52.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:52.454 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:52.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:52.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:52.460 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 19 16 48186 48170 158.413 119.938 0.0124901 0.00630805 2024-08-29T15:10:52.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:52.598 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:52.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:52.700 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:52.737 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:52.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:52.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:52.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:52.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:52.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:52.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:52.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:53.014 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:53.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:53.116 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:53.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:53.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:53.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:53.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:53.303 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:53.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:53.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:53.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:53.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:53.460 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:10:53.460992+0000 min lat: 0.00105259 max lat: 0.150517 avg lat: 0.00637664 2024-08-29T15:10:53.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:10:53.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 20 16 50172 50156 156.698 124.125 0.00352623 0.00637664 2024-08-29T15:10:53.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:53 smithi027 ceph-mon[93921]: pgmap v311: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-08-29T15:10:53.529 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:53 smithi027 ceph-mon[98970]: pgmap v311: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-08-29T15:10:53.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:53.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:10:53.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:53.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:53.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:53.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:53.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:53.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:53 smithi171 ceph-mon[90031]: pgmap v311: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-08-29T15:10:53.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:53.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:53.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:53.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:53.986 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:54.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:54.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:54.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:54.105 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:54.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:54.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:54.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:54.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:54.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:54.296 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:54.322 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:54.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:54.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:54.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:54.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 21 16 51972 51956 154.593 112.5 0.00447821 0.00646259 2024-08-29T15:10:54.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:54.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:54.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:54.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:54.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:54.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:54.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:54.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:54.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:54.841 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:54.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:54.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:54.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:55.009 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:55.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:55.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:55.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:55.288 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:55.390 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:55.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:55.436 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:10:55.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 22 16 53727 53711 152.552 109.688 0.00861614 0.0065504 2024-08-29T15:10:55.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:55.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:55.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:55.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:55.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:55 smithi171 ceph-mon[90031]: pgmap v312: 89 pgs: 89 active+clean; 3.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-29T15:10:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2785432287' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:55 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:55.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:55.807 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:55 smithi027 ceph-mon[93921]: pgmap v312: 89 pgs: 89 active+clean; 3.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-29T15:10:55.807 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2785432287' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:55.807 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:55 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:55.807 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:55 smithi027 ceph-mon[98970]: pgmap v312: 89 pgs: 89 active+clean; 3.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-29T15:10:55.807 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2785432287' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:55.807 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:55 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:10:55.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:55.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:55.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:55.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:55.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:56.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:56.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:56.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:56.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:56.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:56.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:56.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:56.355 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:56.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:56.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:56.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 23 16 55609 55593 151.033 117.625 0.0114659 0.00661678 2024-08-29T15:10:56.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:56.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:56.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:10:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:10:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:10:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[98970]: osdmap e117: 8 total, 8 up, 8 in 2024-08-29T15:10:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:10:56.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:10:56.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:10:56.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:56 smithi027 ceph-mon[93921]: osdmap e117: 8 total, 8 up, 8 in 2024-08-29T15:10:56.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:56.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:56.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:56 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:10:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:56 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:10:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:56 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:10:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:56 smithi171 ceph-mon[90031]: osdmap e117: 8 total, 8 up, 8 in 2024-08-29T15:10:56.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:56.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:56.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:56.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:56.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:56.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:56.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:57.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:10:57.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:57.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:10:57.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:10:57.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:57.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:57.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:10:57.307 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:10:57.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:57.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:57.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:57.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:57.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 24 16 57337 57321 149.239 108 0.00599938 0.00669552 2024-08-29T15:10:57.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:57.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:57.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:57 smithi027 ceph-mon[98970]: pgmap v314: 89 pgs: 89 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-08-29T15:10:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:57 smithi027 ceph-mon[93921]: pgmap v314: 89 pgs: 89 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-08-29T15:10:57.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:57.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:57.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:57.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:57.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:57 smithi171 ceph-mon[90031]: pgmap v314: 89 pgs: 89 active+clean; 3.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-08-29T15:10:57.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:57.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:57.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:57.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:57.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:57.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:57.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:58.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:58.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:58.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:58.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:58.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:58.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:58.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:58.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:58.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:58.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:58.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:58.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 25 16 58965 58949 147.34 101.75 0.00578679 0.00678082 2024-08-29T15:10:58.481 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:58.511 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:58.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:58.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:58.613 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:58.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:58.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:58.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:58.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:10:58.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:58.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:10:58.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:10:58.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:58.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:10:59.024 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:10:59.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:59.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:59.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:10:59.282 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:10:59] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-29T15:10:59.282 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:10:59.282 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:10:59.282 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:10:59.282 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:10:59.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:10:59.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:10:59.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:59.426 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:59.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:59.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 26 16 60687 60671 145.812 107.625 0.00695892 0.00685319 2024-08-29T15:10:59.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:10:59.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:10:59.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:59.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:10:59.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:59.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:59.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:10:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:59 smithi171 ceph-mon[90031]: pgmap v315: 89 pgs: 89 active+clean; 3.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-08-29T15:10:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2212419624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:10:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:59.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[93921]: pgmap v315: 89 pgs: 89 active+clean; 3.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2212419624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[98970]: pgmap v315: 89 pgs: 89 active+clean; 3.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2212419624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:10:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:10:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:10:59.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:59.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:10:59.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:10:59.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:10:59.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:10:59.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:00.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:00.122 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:00.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:00.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:00.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:00.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:00.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:00.366 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:00.408 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:00.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:00.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:00.461 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 27 16 62383 62367 144.337 106 0.0176427 0.00692278 2024-08-29T15:11:00.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:00.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:00.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:00.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:00.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:00.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:00.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:00.808 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:00.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:00.905 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:11:00.905 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:11:00.905 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:11:00.905 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:11:00.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:00.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:00.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:00.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:01.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:01.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:01.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:01.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:01.222 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:01.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:01.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:01.229 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:01.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:01.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:01.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:01.462 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 28 16 63765 63749 142.267 86.375 0.00459148 0.00702317 2024-08-29T15:11:01.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:01.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:01.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:01.595 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:01.631 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:01.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:01.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:01.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:01 smithi027 ceph-mon[98970]: pgmap v316: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-08-29T15:11:01.702 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:01 smithi027 ceph-mon[93921]: pgmap v316: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-08-29T15:11:01.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:01 smithi171 ceph-mon[90031]: pgmap v316: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-08-29T15:11:01.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:01.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:01.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:01.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:01.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:01.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:01.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:11:02.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:02.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:02.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:02.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:02.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:02.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:02.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:02.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:02.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:02.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:02.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:02.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:02.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:02.462 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 29 15 65320 65305 140.715 97.25 0.011478 0.00710153 2024-08-29T15:11:02.471 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:11:02.492 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:11:02.493 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:11:02.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:02.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:02.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:02.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:02.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:02.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:02.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:02.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:02.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:02.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:02.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:03.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:03.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:03.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:03.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:03.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:03.389 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:03.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:03.462 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 30 15 66835 66820 139.178 94.6875 0.0253702 0.00717928 2024-08-29T15:11:03.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:03.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:03.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:03.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.725 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:03.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:03 smithi171 ceph-mon[90031]: pgmap v317: 89 pgs: 89 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-29T15:11:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:03 smithi027 ceph-mon[93921]: pgmap v317: 89 pgs: 89 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-29T15:11:03.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:03 smithi027 ceph-mon[98970]: pgmap v317: 89 pgs: 89 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-08-29T15:11:03.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:03.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:03.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:03.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:03.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:03.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:04.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:04.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:04.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:04.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:04.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:04.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:04.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:04.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:04.406 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:04.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:04.462 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 31 16 68340 68324 137.721 94 0.00783058 0.0072561 2024-08-29T15:11:04.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:04.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:04.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:04.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:04.670 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:04.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:04.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:04.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:04.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:04.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:04.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:04.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:04.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:04.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:05.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:05.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:05.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:05.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:05.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:05.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:05.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:05.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:05.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:05.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:05.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:05.462 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 32 16 69599 69583 135.875 78.6875 0.0153537 0.00735444 2024-08-29T15:11:05.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:05.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:05.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:05.597 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:05.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:05.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:05.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:05.744 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:05 smithi027 ceph-mon[98970]: pgmap v318: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-29T15:11:05.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:05 smithi027 ceph-mon[93921]: pgmap v318: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-29T15:11:05.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:05 smithi171 ceph-mon[90031]: pgmap v318: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-29T15:11:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:05.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:05.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:05.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:05.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:05.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:05.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:06.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:06.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:06.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:06.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:06.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:06.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:06.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:06.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:06.347 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:06.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:06.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:06.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 33 16 70720 70704 133.881 70.0625 0.00536253 0.00746344 2024-08-29T15:11:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:06.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:06.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:06.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:06.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:06.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:06.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:11:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:06.775 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:06.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:06.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:11:06.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:06.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:11:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:06.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:06.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:06.899 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:06.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:06.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:07.001 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:07.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.038 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:07.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:07.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:07.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:07.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:07.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:07.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:07.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:07.431 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:07.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 34 16 71948 71932 132.2 76.75 0.0218248 0.00755959 2024-08-29T15:11:07.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:07.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:07.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:07.670 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:07.709 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:07.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:07.811 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:07.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.858 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:07.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:07.959 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:07.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:07.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:08.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:08.142 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:08.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:08.244 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:07 smithi171 ceph-mon[90031]: pgmap v319: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:11:08.272 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:08.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:08.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:07 smithi027 ceph-mon[93921]: pgmap v319: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:11:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:07 smithi027 ceph-mon[98970]: pgmap v319: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:11:08.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:08.411 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:08.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:08.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:08.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 35 16 73274 73258 130.791 82.875 0.00503605 0.00764041 2024-08-29T15:11:08.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:08.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:08.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:08.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:08.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:08.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:08.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:08.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:08.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:08.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:08.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:08.959 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:08.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:08.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:09.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:09.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:09.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:09.199 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:09.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:09.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:09.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:11:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:11:09] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-29T15:11:09.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:09.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:09.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:09.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 36 16 74612 74596 129.481 83.625 0.00886428 0.00771886 2024-08-29T15:11:09.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:09.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:09.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:09.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:09.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:09.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:09.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:09.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:09.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:09.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:09.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:09.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:09.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:09.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:10.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:10.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:10.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:10.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:10.196 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:10.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:10.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:09 smithi171 ceph-mon[90031]: pgmap v320: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-08-29T15:11:10.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:10.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:09 smithi027 ceph-mon[93921]: pgmap v320: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-08-29T15:11:10.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:09 smithi027 ceph-mon[98970]: pgmap v320: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-08-29T15:11:10.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:10.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:10.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:10.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:10.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:10.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:10.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 37 16 75696 75680 127.812 67.75 0.00238664 0.00781025 2024-08-29T15:11:10.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:10.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:10.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:10.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:11:10.717 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:11:10.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-08-29T15:11:10.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:10.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:10.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:10.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:10.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:10.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:10.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:11.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:11.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:11.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:11.150 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:11.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:11.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:11.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:11.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:11.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:11.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:11.345 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:11.347 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:11.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:11.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:11.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:11.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:11.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 38 16 76917 76901 126.457 76.3125 0.0074627 0.00790335 2024-08-29T15:11:11.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:11.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:11.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:11.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:11.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:11.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:11.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:11.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:11.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:11.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:11.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:11.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:12.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:12.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:11 smithi027 ceph-mon[98970]: pgmap v321: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2024-08-29T15:11:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:12.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:12.058 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:11 smithi027 ceph-mon[93921]: pgmap v321: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2024-08-29T15:11:12.058 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:12.058 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:12.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:12.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:12.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:12.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:11 smithi171 ceph-mon[90031]: pgmap v321: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2024-08-29T15:11:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:12.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:12.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:12.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:12.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:12.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:12.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:12.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:12.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 39 16 78185 78169 125.246 79.25 0.0244749 0.00797882 2024-08-29T15:11:12.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:12.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:12.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:12.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:12.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:12.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:12.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:12.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:12.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:12.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:12.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:12.968 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:11:12.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:13.028 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:13.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:13.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:13.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:13.169 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:13.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/953216582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:11:13.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:13.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:13.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:13.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/953216582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:11:13.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/953216582' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:11:13.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:13.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:13.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:13.463 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:11:13.463922+0000 min lat: 0.00105259 max lat: 0.237678 avg lat: 0.00804221 2024-08-29T15:11:13.464 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:11:13.464 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 40 16 79564 79548 124.269 86.1875 0.0076427 0.00804221 2024-08-29T15:11:13.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:13.566 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:13.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:13.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:13.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:13.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:13.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:13.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:13.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:13.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:13.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:13.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:13.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:14.000 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:14.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:14.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:14.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:14.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:14.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:13 smithi171 ceph-mon[90031]: pgmap v322: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-29T15:11:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:13 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/953216582' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:11:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:13 smithi171 ceph-mon[90031]: osdmap e118: 8 total, 8 up, 8 in 2024-08-29T15:11:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:14.277 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[93921]: pgmap v322: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-29T15:11:14.277 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:14.277 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/953216582' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:11:14.277 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[93921]: osdmap e118: 8 total, 8 up, 8 in 2024-08-29T15:11:14.277 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:14.277 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:14.278 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[98970]: pgmap v322: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-29T15:11:14.278 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:14.278 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/953216582' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:11:14.278 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[98970]: osdmap e118: 8 total, 8 up, 8 in 2024-08-29T15:11:14.278 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:14.278 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:14.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:14.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:14.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:14.464 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 41 16 80973 80957 123.385 88.0625 0.0129768 0.00809937 2024-08-29T15:11:14.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:14.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:14.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:14.654 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:14.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:14.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:14.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:14.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:14.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:14.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:14.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:14.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:15.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:15.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:15.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:15.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:15.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:15.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:15.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:15.383 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:15.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:15.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:15.464 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 42 16 82265 82249 122.37 80.75 0.00975369 0.00816672 2024-08-29T15:11:15.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:15.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:15.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:15.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:15.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:15.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:15.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:15.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:15.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:15.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:15.793 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:15.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:15.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:15.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:15.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:15.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:16.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:16.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:11:16.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:16.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:16.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:16.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:16.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:16.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:16.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:15 smithi171 ceph-mon[90031]: pgmap v324: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:11:16.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:15 smithi027 ceph-mon[93921]: pgmap v324: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:11:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:15 smithi027 ceph-mon[98970]: pgmap v324: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:11:16.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:16.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:16.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:16.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:16.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:16.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:16.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:16.464 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 43 16 83448 83432 121.244 73.9375 0.00332005 0.00824163 2024-08-29T15:11:16.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:16.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:16.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:16.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:16.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:16.727 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:16.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:16.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:16.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:16.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:16.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:16.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.007 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:17.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:17.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:17.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:17.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:17.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:17.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:17.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:17.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:17.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:17.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:17.464 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 44 16 84795 84779 120.401 84.1875 0.00723311 0.00830039 2024-08-29T15:11:17.524 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:17.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:17.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:17.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:17.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:17.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:17.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:17.866 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:11:17.866 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:11:17.866 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:11:17.866 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:11:17.866 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:11:17.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:17.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:17.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:18.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:18.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:18.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:18.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:18.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:18.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:18.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:17 smithi171 ceph-mon[90031]: pgmap v325: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-08-29T15:11:18.280 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:17 smithi027 ceph-mon[93921]: pgmap v325: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-08-29T15:11:18.280 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:17 smithi027 ceph-mon[98970]: pgmap v325: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-08-29T15:11:18.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:18.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:18.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:18.464 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 45 16 86115 86099 119.559 82.5 0.00643977 0.0083589 2024-08-29T15:11:18.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:18.539 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:18.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:18.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:11:18.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:18.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:18.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:18.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:18.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:18.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:18.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:18.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:18.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:19.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:19.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:19.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:19.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:19.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:19.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:19.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:11:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:11:19] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-29T15:11:19.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:19.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:19.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:19.466 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 46 16 87422 87406 118.731 81.6875 0.00813304 0.00841757 2024-08-29T15:11:19.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:19.551 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:19.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:19.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:19.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:19.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:19.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:19.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:19.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:19.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:19.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:19.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:19.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:20.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:20.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:20.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:20.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:20.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:20.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:20.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:20.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:19 smithi171 ceph-mon[90031]: pgmap v326: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:11:20.310 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:19 smithi027 ceph-mon[93921]: pgmap v326: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:11:20.310 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:19 smithi027 ceph-mon[98970]: pgmap v326: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:11:20.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:20.368 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:20.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:20.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:20.466 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 47 16 88651 88635 117.839 76.8125 0.014436 0.0084811 2024-08-29T15:11:20.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:20.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:20.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:20.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:20.645 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:20.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:20.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:20.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:20.776 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:11:20.776 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:11:20.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:20.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:11:20.805 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:11:20.806 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:11:20.806 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:11:20.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:11:20.817 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:11:20.854 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:11:20.855 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:11:20.889 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:11:20.889 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:11:20.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:20.926 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:11:20.926 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:11:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:20.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:20.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:20.962 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:11:20.962 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:11:20.998 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:11:20.999 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:11:21.030 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:11:21.030 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:11:21.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:21.038 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:21.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:21.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:21.129 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2630194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:21.129 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2630194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:21.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:21.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:21.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:21.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2630194' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:21.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:21.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:21.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:21.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:21.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:21.466 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 48 16 89870 89854 116.971 76.1875 0.0172292 0.00854484 2024-08-29T15:11:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:11:21.477 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:11:21.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-08-29T15:11:21.500 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:21.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:21.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:21.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:21.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:21.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:21.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:21.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:21.756 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:21.812 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:21.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:21.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:21.950 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:21.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:22.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:22.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:22.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:22.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:22.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:22.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:22 smithi171 ceph-mon[90031]: pgmap v327: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:11:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:22 smithi027 ceph-mon[93921]: pgmap v327: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:11:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:22 smithi027 ceph-mon[98970]: pgmap v327: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:11:22.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:22.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:22.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:22.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:22.466 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 49 16 91100 91084 116.153 76.875 0.00321832 0.00859746 2024-08-29T15:11:22.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:22.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:22.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:22.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:22.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:22.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:22.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:22.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:22.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:22.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:22.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:22.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:22.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:22.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:23.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:23.054 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:23.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:23.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:23.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:23.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:23.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:23.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:23.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:23.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:23.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:23.466 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 50 16 92406 92390 115.462 81.625 0.00940062 0.00865621 2024-08-29T15:11:23.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:23.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:23.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:23.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:23.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:23.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:23.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:23.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:23.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:23.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:23.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:23.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:23.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:23.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:23.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:24.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:24.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:24.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:24.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:24.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:24.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:24 smithi171 ceph-mon[90031]: pgmap v328: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:11:24.280 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:24.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:24 smithi027 ceph-mon[93921]: pgmap v328: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:11:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:24 smithi027 ceph-mon[98970]: pgmap v328: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:11:24.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:24.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:24.467 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 51 16 93689 93673 114.77 80.1875 0.005678 0.00870833 2024-08-29T15:11:24.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:24.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:24.665 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:24.701 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:24.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:24.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:24.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:24.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:24.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:24.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:24.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:25.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:25.089 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:25.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:25.136 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:25.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:25.237 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:25.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:25.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:25.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:25.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:25.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:25.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:25.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:25.467 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 52 16 94852 94836 113.961 72.6875 0.0310577 0.00876927 2024-08-29T15:11:25.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:25.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:25.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:25.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:25.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:11:25.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:25.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:25.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:25.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:25.828 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:25.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:25.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:25.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:25.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:25.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:26.050 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:11:26.050 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:11:26.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:26.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:26.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:26.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:26.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:26.235 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:26.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:26.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:26 smithi027 ceph-mon[98970]: pgmap v329: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-29T15:11:26.246 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:26 smithi027 ceph-mon[93921]: pgmap v329: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-29T15:11:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:26 smithi171 ceph-mon[90031]: pgmap v329: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-29T15:11:26.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:26.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:26.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:26.467 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 53 16 95903 95887 113.05 65.6875 0.0160572 0.00884146 2024-08-29T15:11:26.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:26.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:26.505 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:26.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:26.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:26.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:26.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:26.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:26.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:26.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:26.893 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:26.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:26.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:27.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:27.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:27.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:27.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:27.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:27.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:27.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:27.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:27.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:27.370 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:27.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:27.467 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 54 16 96998 96982 112.224 68.4375 0.0171049 0.00890573 2024-08-29T15:11:27.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:27.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:27.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:27.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:27.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:27.660 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:27.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:27.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:11:27.762 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:27.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:27.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:27.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:27.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:27.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:27.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:27.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:28.028 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:28.033 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:11:28.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:28.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:28.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:28.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:28.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:28.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:28 smithi027 ceph-mon[93921]: pgmap v330: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-29T15:11:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2225473372' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:28 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:28 smithi027 ceph-mon[98970]: pgmap v330: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-29T15:11:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2225473372' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:28 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:28.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:28.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:28.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:28.467 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 55 16 98124 98108 111.463 70.375 0.00452224 0.00896652 2024-08-29T15:11:28.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:28.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:28.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:28 smithi171 ceph-mon[90031]: pgmap v330: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-29T15:11:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2225473372' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:28 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:28.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:28.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:28.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:28.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:28.716 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:28.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:28.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:28.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:28.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:28.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:28.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:29.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:29.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:29.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:29.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:29.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:29.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:29.188 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:29.188 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:11:29.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[98970]: osdmap e119: 8 total, 8 up, 8 in 2024-08-29T15:11:29.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:29.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:29.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:11:29.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[93921]: osdmap e119: 8 total, 8 up, 8 in 2024-08-29T15:11:29.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:29.190 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:11:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:11:29] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-29T15:11:29.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:29.325 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:29.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:29.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:29.403 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:29.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:29 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:29.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:29 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:11:29.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:29 smithi171 ceph-mon[90031]: osdmap e119: 8 total, 8 up, 8 in 2024-08-29T15:11:29.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:29.427 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:29.467 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 56 16 99291 99275 110.775 72.9375 0.00847581 0.00902278 2024-08-29T15:11:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:29.470 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:29.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:29.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:29.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:29.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:29.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:29.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:29.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:29.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:29.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:29.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:29.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:29.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:29.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:29.985 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:30.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:30 smithi027 ceph-mon[98970]: pgmap v332: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-29T15:11:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:30 smithi027 ceph-mon[93921]: pgmap v332: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-29T15:11:30.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:30.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:30.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:30.176 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:11:30.176 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:11:30.176 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:11:30.176 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:11:30.176 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:11:30.176 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:11:30.177 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:11:30.177 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:11:30.177 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:11:30.177 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:11:30.177 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:11:30.177 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:11:30.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:30 smithi171 ceph-mon[90031]: pgmap v332: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-29T15:11:30.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:30.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:30.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:30.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:30.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:30.468 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 57 16 100356 100340 109.999 66.5625 0.0168631 0.00908601 2024-08-29T15:11:30.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:30.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:30.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:30.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:30.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:30.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:30.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:30.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:30.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:31.080 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:31.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:31.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:11:31.125 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:11:31.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-08-29T15:11:31.225 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:31.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:31.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:31.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:31.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:31.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:31.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:31.467 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 58 16 101386 101370 109.212 64.375 0.00713093 0.00915138 2024-08-29T15:11:31.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:31.522 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:31.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:31.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:31.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:31.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:31.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:31.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:31.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:31.804 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:31.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:31.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:11:31.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:31.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:31.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:31.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:32.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:32.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:32 smithi027 ceph-mon[93921]: pgmap v333: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-29T15:11:32.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:32.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:32.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:32.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:32.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:32.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:32.310 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:32 smithi027 ceph-mon[98970]: pgmap v333: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-29T15:11:32.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:32.377 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:32.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:32.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:32.468 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 59 16 102540 102524 108.584 72.125 0.0135621 0.00920427 2024-08-29T15:11:32.482 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:32.512 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:32.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:32.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:32 smithi171 ceph-mon[90031]: pgmap v333: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-29T15:11:32.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:32.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:32.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:32.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:32.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:32.831 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:32.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:32.933 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:32.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:32.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:32.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:33.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:33.364 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:33.365 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:33.415 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:33.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:33.468 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:11:33.468069+0000 min lat: 0.00105259 max lat: 0.237678 avg lat: 0.00925638 2024-08-29T15:11:33.468 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:11:33.468 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 60 16 103688 103672 107.97 71.75 0.0102461 0.00925638 2024-08-29T15:11:33.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:33.556 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:33.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:33.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:33.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:33.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:33.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:33.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:33.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:33.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:33.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:33.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:33.950 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:33.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:33.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:34.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:34 smithi027 ceph-mon[93921]: pgmap v334: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-29T15:11:34.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:34 smithi027 ceph-mon[98970]: pgmap v334: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-29T15:11:34.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:34.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:34.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:34.195 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:34.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:34.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:34.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:34.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:34.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:34.468 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 61 16 104834 104818 107.374 71.625 0.00858288 0.00930796 2024-08-29T15:11:34.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:34 smithi171 ceph-mon[90031]: pgmap v334: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-29T15:11:34.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:34.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:34.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:34.670 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:34.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:34.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:34.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:34.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:34.868 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:34.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:34.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:34.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:35.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:35.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:35.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:35.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:35.226 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:35.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:35.328 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:35.365 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:35.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:35.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:35.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:35.468 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 62 16 106012 105996 106.83 73.625 0.0113871 0.0093558 2024-08-29T15:11:35.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:35.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:35.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:35.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:35.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:35.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:35.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:35.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:35.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:35.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:35.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:35.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:36.004 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:11:36.005 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:11:36.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:36.045 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:11:36.045 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:11:36.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:36.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:36.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:36.077 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:11:36.077 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:11:36.114 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:11:36.115 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:11:36.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:11:36.149 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:11:36.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:36.184 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:11:36.184 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:11:36.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:36.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:36.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:36.223 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:11:36.223 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:11:36.250 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:11:36.250 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:11:36.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:36 smithi027 ceph-mon[93921]: pgmap v335: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:11:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2400151591' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:36 smithi027 ceph-mon[98970]: pgmap v335: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:11:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2400151591' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:36.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:36.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:36.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:36.455 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:36.469 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 63 16 107105 107089 106.217 68.3125 0.00926585 0.00941005 2024-08-29T15:11:36.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:36.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:36 smithi171 ceph-mon[90031]: pgmap v335: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:11:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2400151591' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:36.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:36.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:36.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:36.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:36.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:36.766 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:36.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:36.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:36.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:36.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:36.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:37.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:37.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:37.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:37.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:37.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:37.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:37.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:37.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:37.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:37.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:37.469 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 64 16 108236 108220 105.662 70.6875 0.00508275 0.00945851 2024-08-29T15:11:37.470 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:37.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:37.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:37.572 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:37.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:37.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:37.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:37.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:37.735 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:37.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:37.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:37.838 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:37.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:37.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:37.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:38.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:38.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:38.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:38.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:38.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:38.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:38.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:38.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:38.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:38 smithi027 ceph-mon[93921]: pgmap v336: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-29T15:11:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:38 smithi027 ceph-mon[98970]: pgmap v336: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-29T15:11:38.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:38.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:38.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:38.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:38.469 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 65 16 109299 109283 105.058 66.4375 0.00577027 0.00951306 2024-08-29T15:11:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:38 smithi171 ceph-mon[90031]: pgmap v336: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-29T15:11:38.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:38.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:38.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:38.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:38.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:38.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:38.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:38.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:38.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:38.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:38.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:38.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:38.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:39.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:39.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:39.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:39.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:39.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:39.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:39.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:39.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:39.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:39.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:39.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:39.316 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:11:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:11:39] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-29T15:11:39.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:39.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:39.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:39.470 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 66 16 110335 110319 104.447 64.75 0.0204505 0.00956961 2024-08-29T15:11:39.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:39.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:39.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:39.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:39.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:39.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:39.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:39.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:39.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:39.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:39.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:40.020 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:40.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:40.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:40.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:40.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:40.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:40.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:40.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:40 smithi027 ceph-mon[98970]: pgmap v337: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:11:40.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:40 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:40.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:40 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:40.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:40 smithi027 ceph-mon[93921]: pgmap v337: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:11:40.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:40 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:40.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:40 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:40.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:40.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:40.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:40.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:40.471 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 67 16 111149 111133 103.646 50.875 0.0264758 0.00964122 2024-08-29T15:11:40.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:40.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:40 smithi171 ceph-mon[90031]: pgmap v337: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:11:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:40 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:40 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:40.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:40.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:40.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:40.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:40.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:40.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:40.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:40.915 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:40.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:41.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:41.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:41.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:41.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:41.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:41.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:41.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:41.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:41.323 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:41.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:41.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:11:41.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:11:41.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:41.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1889919741' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:11:41.408 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:11:41.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:11:41.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:41.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1889919741' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:11:41.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:41.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:41.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:41.470 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 68 16 112141 112125 103.034 62 0.0144392 0.00970056 2024-08-29T15:11:41.515 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:11:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:41 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:11:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:41 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:11:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:41 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:11:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1889919741' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:11:41.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:41.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:41.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:41.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:41.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:41.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:41.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:41.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:41.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:41.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:41.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:41.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:42.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:42.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:42.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:11:42.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:42.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:42.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:42.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[93921]: pgmap v338: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:11:42.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:42.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1889919741' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:11:42.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[93921]: osdmap e120: 8 total, 8 up, 8 in 2024-08-29T15:11:42.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[98970]: pgmap v338: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:11:42.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:42.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1889919741' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:11:42.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:42 smithi027 ceph-mon[98970]: osdmap e120: 8 total, 8 up, 8 in 2024-08-29T15:11:42.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:42.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:42.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:42.367 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:11:42.367 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:11:42.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.367 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:113004 5.a 5:58905a3c:::benchmark_data_smithi027_135490_object113003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-29T15:11:42.367 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:42.352950+0000", 2024-08-29T15:11:42.367 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012245867000000001, 2024-08-29T15:11:42.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012279635000000001, 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 113004 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.368 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.352950+0000", 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.352950+0000", 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.352952+0000", 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4699999999999999e-06 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.352980+0000", 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8289e-05 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.352996+0000", 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6393999999999999e-05 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353005+0000", 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2409999999999995e-06 2024-08-29T15:11:42.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353429+0000", 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042419099999999998 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353602+0000", 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000172941 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353758+0000", 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015647999999999999 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.358605+0000", 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048470179999999998 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361066+0000", 2024-08-29T15:11:42.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002460378 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:113005 5.b 5:d99ac589:::benchmark_data_smithi027_135490_object113004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:42.353025+0000", 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012171589, 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01229297, 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:11:42.374 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 113005 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353025+0000", 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353025+0000", 2024-08-29T15:11:42.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353025+0000", 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5499999999999997e-07 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353050+0000", 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4689999999999999e-05 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353058+0000", 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4780000000000008e-06 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353065+0000", 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1009999999999997e-06 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.354175+0000", 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011105220000000001 2024-08-29T15:11:42.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.354286+0000", 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011058800000000001 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.354408+0000", 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000122445 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.362787+0000", 2024-08-29T15:11:42.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083784730000000009 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.362799+0000", 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2673999999999999e-05 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:113008 5.b 5:dee2a903:::benchmark_data_smithi027_135490_object113007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:42.353081+0000", 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.01211506, 2024-08-29T15:11:42.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012308009999999999, 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 113008 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353081+0000", 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:11:42.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353081+0000", 2024-08-29T15:11:42.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353082+0000", 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.32e-07 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353105+0000", 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3482e-05 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353113+0000", 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6079999999999998e-06 2024-08-29T15:11:42.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:11:42.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.353118+0000", 2024-08-29T15:11:42.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6289999999999998e-06 2024-08-29T15:11:42.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.358074+0000", 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049562210000000002 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.358259+0000", 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184105 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.358453+0000", 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000194045 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.362843+0000", 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043905519999999998 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:11:42.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.364654+0000", 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001810889 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:113014 5.a 5:5659bf56:::benchmark_data_smithi027_135490_object113013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:42.360259+0000", 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.004937482, 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052014610000000001, 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:11:42.388 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 113014 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360259+0000", 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360259+0000", 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360259+0000", 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7999999999999995e-07 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360284+0000", 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4332e-05 2024-08-29T15:11:42.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360292+0000", 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1629999999999996e-06 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360298+0000", 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3339999999999998e-06 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361092+0000", 2024-08-29T15:11:42.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00079417799999999998 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361206+0000", 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011411799999999999 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361333+0000", 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012656700000000001 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:11:42.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.365260+0000", 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039266529999999996 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:113015 5.8 5:187c2de1:::benchmark_data_smithi027_135490_object113014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:42.360314+0000", 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.004881664, 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052074929999999997, 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:11:42.393 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 113015 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360314+0000", 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360314+0000", 2024-08-29T15:11:42.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360315+0000", 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3300000000000002e-07 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360339+0000", 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3657000000000001e-05 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:11:42.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360347+0000", 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1680000000000004e-06 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.360353+0000", 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1149999999999999e-06 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361652+0000", 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001299082 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361772+0000", 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011992199999999999 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361916+0000", 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144456 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.365352+0000", 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003435081 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.397 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:113003 5.a 5:51f7c95c:::benchmark_data_smithi027_135490_object113002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:42.344933+0000", 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.020263276, 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.02065212, 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:11:42.398 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 113003 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.344933+0000", 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.344933+0000", 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.344935+0000", 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1509999999999998e-06 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.344962+0000", 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6981000000000001e-05 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.344973+0000", 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1195e-05 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.344994+0000", 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0290999999999999e-05 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.345068+0000", 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4506999999999997e-05 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.345252+0000", 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184274 2024-08-29T15:11:42.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.345432+0000", 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018003600000000001 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.352097+0000", 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066647319999999996 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:42.361032+0000", 2024-08-29T15:11:42.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089349270000000005 2024-08-29T15:11:42.403 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.403 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:11:42.403 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.403 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:11:42.403 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:11:42.403 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 6 2024-08-29T15:11:42.403 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:11:42.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:42.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:42.472 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 69 16 113119 113103 102.424 61.125 0.00675786 0.00975805 2024-08-29T15:11:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:42 smithi171 ceph-mon[90031]: pgmap v338: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:11:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:42 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:11:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1889919741' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:11:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:42 smithi171 ceph-mon[90031]: osdmap e120: 8 total, 8 up, 8 in 2024-08-29T15:11:42.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:42.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:42.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:42.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:42.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:42.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:42.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:42.998 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:42.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:42.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:43.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:43.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:43.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:43.236 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:43.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:43.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:43.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:43.418 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:43.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:43.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:43.473 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 70 16 114197 114181 101.923 67.375 0.00604542 0.00980686 2024-08-29T15:11:43.519 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:43.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:43.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:43.574 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:11:43.575 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:11:43.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:43.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:43.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:43.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:43.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:43.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:43.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:43.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:43.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:43.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:43.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:44.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:44.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:44.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:44.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:44.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:44.249 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:44.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:44 smithi027 ceph-mon[98970]: pgmap v340: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:11:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:44 smithi027 ceph-mon[93921]: pgmap v340: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:11:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:44.351 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:44.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:44.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:44.473 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 71 16 115263 115247 101.426 66.625 0.0351992 0.0098539 2024-08-29T15:11:44.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:44 smithi171 ceph-mon[90031]: pgmap v340: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:11:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:11:44.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:44.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:44.627 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:44.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:11:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:44.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:44.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:44.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:44.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:44.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:44.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:44.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:44.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:44.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:44.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:45.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:45.100 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:45.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:45.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:45.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:45.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:45.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:45.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:11:45.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:45.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:45.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:45.473 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 72 16 116254 116238 100.877 61.9375 0.00922615 0.0099082 2024-08-29T15:11:45.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:45.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:45.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:45.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:45.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:45.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:45.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:45.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:45.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:45.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:45.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:45.856 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:11:45.856 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:11:45.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-08-29T15:11:45.902 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:45.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:46.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:46.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:46.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:46.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:46.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:46.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:46.278 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:46.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:46.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:46.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:46.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:46 smithi027 ceph-mon[98970]: pgmap v341: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-08-29T15:11:46.395 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:46 smithi027 ceph-mon[93921]: pgmap v341: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-08-29T15:11:46.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:46.456 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:46.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:46.473 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 73 16 117247 117231 100.346 62.0625 0.00900979 0.00996067 2024-08-29T15:11:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:46 smithi171 ceph-mon[90031]: pgmap v341: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2024-08-29T15:11:46.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:46.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:46.604 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:46.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:46.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:46.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:46.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:46.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:46.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:46.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:46.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:46.985 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:47.024 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:47.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:47.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:47.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:47.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:47.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:47.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:47.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:47.322 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:47.324 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:47.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:47.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:47.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:47.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 74 16 118293 118277 99.8714 65.375 0.0156243 0.0100084 2024-08-29T15:11:47.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:47.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:47.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:47.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:47.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:47.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:47.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:47.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:47.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:47.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:47.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:47.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:48.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:48.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:48.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:48.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:48.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:48.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:48.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:48.310 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:48 smithi027 ceph-mon[98970]: pgmap v342: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:11:48.310 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:48 smithi027 ceph-mon[93921]: pgmap v342: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:11:48.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:48.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:48.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:48.474 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 75 16 119334 119318 99.4073 65.0625 0.0153999 0.0100544 2024-08-29T15:11:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:48 smithi171 ceph-mon[90031]: pgmap v342: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:11:48.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:48.551 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:48.559 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:48.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:48.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:48.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:48.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:48.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.845 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:48.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:48.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:11:48.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:11:48.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:11:48.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:48.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:48.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:49.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:49.109 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:11:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:11:49] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-29T15:11:49.115 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:49.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:49.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:49.217 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:49.252 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:49.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:49.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:49.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:49.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:49.474 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 76 16 120297 120281 98.8913 60.1875 0.00630796 0.0101075 2024-08-29T15:11:49.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:49.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:49.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:49.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/139154170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/139154170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:49.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:49 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/139154170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:11:49.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:49.652 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:49.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:49.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:49.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:49.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:49.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:49.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:49.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:50.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:50.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:50.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:50.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:50.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:50.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:50.223 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:50.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:50.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:50.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:50.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:50.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:50.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:50.474 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 77 16 121304 121288 98.4239 62.9375 0.0146509 0.0101539 2024-08-29T15:11:50.493 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:50 smithi027 ceph-mon[98970]: pgmap v343: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-08-29T15:11:50.493 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:50 smithi027 ceph-mon[93921]: pgmap v343: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-08-29T15:11:50.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:50.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:50 smithi171 ceph-mon[90031]: pgmap v343: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-08-29T15:11:50.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:50.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:50.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:50.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:50.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:50.772 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:50.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:50.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:50.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:50.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:51.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:51.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:51.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:51.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:51.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:51.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:51.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:51.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:51.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:51.421 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:51.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:51.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:51.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 78 16 122349 122333 97.9994 65.3125 0.0184992 0.0101997 2024-08-29T15:11:51.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:51.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:51.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:51.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:51.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:51.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:51.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:51.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:51.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:51.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:51.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:51.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.009 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:52.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:52.110 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:52.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:52.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:52.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:52.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:52.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:52 smithi027 ceph-mon[98970]: pgmap v344: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:11:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:52 smithi027 ceph-mon[93921]: pgmap v344: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:11:52.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:52.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:52.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:52.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 79 16 123375 123359 97.5706 64.125 0.0114385 0.0102444 2024-08-29T15:11:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:52 smithi171 ceph-mon[90031]: pgmap v344: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:11:52.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:52.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:52.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:52.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:52.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:52.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:52.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:52.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:52.952 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:52.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:52.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:53.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:11:53.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:53.178 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:53.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:53.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:53.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:53.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:53.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:53.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:53.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:53.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:53.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:53.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:11:53.475108+0000 min lat: 0.00105259 max lat: 0.237678 avg lat: 0.0102923 2024-08-29T15:11:53.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:11:53.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 80 16 124353 124337 97.115 61.125 0.0265398 0.0102923 2024-08-29T15:11:53.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:53.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:53.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:53.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:53.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:53.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:53.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:53.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:53.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:53.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:53.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:53.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:54.005 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:54.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:54.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:54.130 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:11:54.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:54.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:54.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:54.299 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:54.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:54.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:54.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:54.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:54.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 81 16 125304 125288 96.6499 59.4375 0.0211642 0.0103413 2024-08-29T15:11:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:54 smithi171 ceph-mon[90031]: pgmap v345: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-08-29T15:11:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:54 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/16634134' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:54 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:54.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:54 smithi027 ceph-mon[98970]: pgmap v345: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-08-29T15:11:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:54 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/16634134' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:54 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:54 smithi027 ceph-mon[93921]: pgmap v345: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-08-29T15:11:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:54 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/16634134' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:54 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:11:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:54.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:54.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:54.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:54.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:54.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:54.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:54.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:54.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:54.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:54.927 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:11:54.927 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:11:54.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:55.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:55.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:55.111 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:55.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:55.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:55.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:55.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:55.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:55.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:55.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:11:55.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:11:55.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:11:55.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[98970]: osdmap e121: 8 total, 8 up, 8 in 2024-08-29T15:11:55.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:11:55.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:11:55.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:11:55.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:55 smithi027 ceph-mon[93921]: osdmap e121: 8 total, 8 up, 8 in 2024-08-29T15:11:55.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:55.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:55.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:55.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 82 16 126319 126303 96.2448 63.4375 0.0139771 0.0103855 2024-08-29T15:11:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:55 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:11:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:55 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:11:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:55 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:11:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:55 smithi171 ceph-mon[90031]: osdmap e121: 8 total, 8 up, 8 in 2024-08-29T15:11:55.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:55.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:55.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:55.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:55.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:55.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:55.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:55.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:11:55.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:55.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:55.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:55.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:56.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:56.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:56.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:56.151 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:56.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:56.204 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:56 smithi027 ceph-mon[98970]: pgmap v347: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:11:56.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:56 smithi027 ceph-mon[93921]: pgmap v347: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:11:56.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:56.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:56.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:56.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:56.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:56.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:56.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:56.476 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 83 16 127326 127310 95.8435 62.9375 0.00885153 0.0104289 2024-08-29T15:11:56.510 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:56 smithi171 ceph-mon[90031]: pgmap v347: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:11:56.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:56.594 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:11:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:11:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:56.596 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:56.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:56.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:56.736 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:56.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:56.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:56.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:56.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:56.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:56.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:56.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.016 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:57.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:57.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:57.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:57.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:57.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:57.285 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:57.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:57.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:57.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 84 16 128302 128286 95.4287 61 0.00825629 0.0104738 2024-08-29T15:11:57.492 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:11:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:57 smithi171 ceph-mon[90031]: pgmap v348: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:11:57.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:57 smithi027 ceph-mon[98970]: pgmap v348: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:11:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:57 smithi027 ceph-mon[93921]: pgmap v348: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:11:57.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:57.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:57.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:57.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:57.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:57.851 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:11:57.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:57.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:57.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:58.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:11:58.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:58.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:58.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:11:58.237 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:58.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:11:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:58.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:58.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:58.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:58.425 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:58.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:58.475 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 85 16 129279 129263 95.0244 61.0625 0.017361 0.0105183 2024-08-29T15:11:58.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:58.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:58.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:58.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:58.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:58.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:58.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:58.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:58.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:58.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:58.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:58.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:11:58.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:58.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:58.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:11:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:11:59.067 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:11:59.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-08-29T15:11:59.084 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:11:59] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-08-29T15:11:59.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:11:59.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:59.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:59.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:11:59.227 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:11:59.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:59.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:11:59.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:11:59.361 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:11:59.362 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:11:59.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:59.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:59.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:11:59.413 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:11:59.414 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:11:59.452 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:11:59.452 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:11:59.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 86 16 130178 130162 94.5694 56.1875 0.0246977 0.0105694 2024-08-29T15:11:59.495 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:11:59.495 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:11:59.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:11:59.534 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:11:59.534 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:11:59.537 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:59.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:59.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:11:59.572 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:11:59.572 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:11:59.608 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:11:59.609 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:11:59.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:59.643 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:11:59.643 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:11:59.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:59.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:59.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:59.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:11:59.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:11:59.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:59.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:11:59.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:11:59.952 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:11:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:11:59.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:59 smithi171 ceph-mon[90031]: pgmap v349: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:12:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:11:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2482901606' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:00.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[98970]: pgmap v349: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:12:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2482901606' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[93921]: pgmap v349: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:12:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:11:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2482901606' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:00.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:00.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:00.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:00.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:00.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:00.336 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:00.376 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:00.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:00.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:00.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 87 16 131046 131030 94.106 54.25 0.0102502 0.0106213 2024-08-29T15:12:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:00.515 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:00.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:00.616 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:00.651 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:00.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:00.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:00.790 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:00.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:00.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:00.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:00.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:00.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:01.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:01.054 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:01.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:01.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:01.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:01.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:01.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:01.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:01.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:01.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:01.349 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:01.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:01.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:01.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 88 16 131887 131871 93.6339 52.5625 0.0157963 0.0106748 2024-08-29T15:12:01.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:01.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:01.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:01.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:01.625 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:01.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:01.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:01.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:01.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:01.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:01.861 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:01.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:01.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:01.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:01 smithi171 ceph-mon[90031]: pgmap v350: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-08-29T15:12:02.032 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:02.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:01 smithi027 ceph-mon[98970]: pgmap v350: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-08-29T15:12:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:01 smithi027 ceph-mon[93921]: pgmap v350: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2024-08-29T15:12:02.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:02.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:02.175 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:02.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:02.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:02.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:02.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:02.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:02.376 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:02.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:02.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:02.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:02.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 89 15 132816 132801 93.2349 58.125 0.00677661 0.0107207 2024-08-29T15:12:02.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:02.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:02.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:02.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:02.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:02.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:02.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:02.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:02.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:02.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:02.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:02.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:03.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:03.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:03.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:03.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:03.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:03.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23186 5.6 5:623074f2:::benchmark_data_smithi027_135490_object23185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.949059+0000", 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.251792922000007, 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14490519299999999, 2024-08-29T15:12:03.205 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23186 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949059+0000", 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949059+0000", 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949060+0000", 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3e-07 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949082+0000", 2024-08-29T15:12:03.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1919000000000001e-05 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949092+0000", 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0761000000000001e-05 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949099+0000", 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9330000000000004e-06 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949455+0000", 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035609599999999997 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949687+0000", 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023169900000000001 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949829+0000", 2024-08-29T15:12:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000142006 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953284+0000", 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034552850000000002 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953844+0000", 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00055922800000000005 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092116+0000", 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.138272688 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093921+0000", 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018042819999999999 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093964+0000", 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3566000000000003e-05 2024-08-29T15:12:03.211 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23191 5.6 5:65ba2179:::benchmark_data_smithi027_135490_object23190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.951583+0000", 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.249269100000006, 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14251501499999999, 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.212 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23191 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951583+0000", 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951583+0000", 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951584+0000", 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.595e-06 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951607+0000", 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2668999999999999e-05 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951621+0000", 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3484e-05 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951628+0000", 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9109999999999993e-06 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951686+0000", 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7859999999999998e-05 2024-08-29T15:12:03.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951815+0000", 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012877 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951960+0000", 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144976 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953673+0000", 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001712748 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954262+0000", 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058924800000000001 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094015+0000", 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13975259500000001 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094062+0000", 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7509999999999997e-05 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094098+0000", 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5648999999999997e-05 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23194 5.8 5:1e14095a:::benchmark_data_smithi027_135490_object23193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.953618+0000", 2024-08-29T15:12:03.218 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.247234179000003, 2024-08-29T15:12:03.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14412351700000001, 2024-08-29T15:12:03.219 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.219 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.219 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.219 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.219 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23194 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953618+0000", 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953618+0000", 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953619+0000", 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139e-06 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953643+0000", 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4454e-05 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953654+0000", 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1015999999999999e-05 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953661+0000", 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8179999999999998e-06 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953927+0000", 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026623199999999998 2024-08-29T15:12:03.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092057+0000", 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13812923499999999 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092234+0000", 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000176925 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097469+0000", 2024-08-29T15:12:03.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052357339999999997 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097509+0000", 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9536000000000002e-05 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097622+0000", 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011326400000000001 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097701+0000", 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9154999999999997e-05 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097741+0000", 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0009000000000003e-05 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:23186 5.6 e115/111 ondisk, result = 0)", 2024-08-29T15:12:03.225 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.953984+0000", 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.246867637999998, 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13999176999999999, 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953984+0000", 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953984+0000", 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953985+0000", 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7500000000000001e-07 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953987+0000", 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6249999999999999e-06 2024-08-29T15:12:03.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953996+0000", 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3359999999999995e-06 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954001+0000", 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4539999999999998e-06 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954288+0000", 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028707100000000002 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954294+0000", 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3080000000000004e-06 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093976+0000", 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13968160099999999 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.229 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:23191 5.6 e115/111 ondisk, result = 0)", 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.954027+0000", 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.246824759000006, 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.140091682, 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954027+0000", 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954027+0000", 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954028+0000", 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5400000000000002e-07 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954030+0000", 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6929999999999999e-06 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954038+0000", 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5789999999999998e-06 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954043+0000", 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1580000000000004e-06 2024-08-29T15:12:03.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094001+0000", 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13995839700000001 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094009+0000", 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5140000000000004e-06 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094119+0000", 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109887 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23196 5.b 5:db57f893:::benchmark_data_smithi027_135490_object23195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.954654+0000", 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.246197664999997, 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14965039399999999, 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.234 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23196 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954654+0000", 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954654+0000", 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954655+0000", 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.071e-06 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954678+0000", 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2886000000000001e-05 2024-08-29T15:12:03.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092083+0000", 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13740485399999999 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092101+0000", 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7850999999999999e-05 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092171+0000", 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9696999999999994e-05 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092322+0000", 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000151547 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094055+0000", 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017333260000000001 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097546+0000", 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003490904 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.103537+0000", 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059910780000000004 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.104211+0000", 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00067352000000000004 2024-08-29T15:12:03.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.104264+0000", 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2648999999999997e-05 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.104305+0000", 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1010999999999997e-05 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:23209 5.4 e115/111 5:24099454:::benchmark_data_smithi027_135490_object23208:head v 115'1489, mlcod=115'1487)", 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.957922+0000", 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.242930118999993, 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13573950700000001, 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957922+0000", 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957922+0000", 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957923+0000", 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.274e-06 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957979+0000", 2024-08-29T15:12:03.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5788000000000002e-05 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092353+0000", 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134373779 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092360+0000", 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2559999999999996e-06 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092409+0000", 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9481000000000003e-05 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092430+0000", 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0945e-05 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093619+0000", 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001188417 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093661+0000", 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2567000000000001e-05 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68618 5.b 5:dedf1630:::benchmark_data_smithi027_135490_object68617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.636700+0000", 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.564152131, 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134410845, 2024-08-29T15:12:03.245 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68618 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636700+0000", 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636700+0000", 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636701+0000", 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0699999999999996e-07 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636727+0000", 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5918000000000002e-05 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636738+0000", 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.13e-05 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636745+0000", 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9779999999999999e-06 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.643278+0000", 2024-08-29T15:12:03.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065328010000000004 2024-08-29T15:12:03.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.643406+0000", 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012882799999999999 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.643546+0000", 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013906899999999999 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648290+0000", 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047440140000000004 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648333+0000", 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3448999999999997e-05 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.770935+0000", 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.122602275 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771025+0000", 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9437000000000004e-05 2024-08-29T15:12:03.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771111+0000", 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5869000000000001e-05 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68623 5.8 5:174e3264:::benchmark_data_smithi027_135490_object68622:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.640335+0000", 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.560517111000003, 2024-08-29T15:12:03.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.119377473, 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68623 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.640335+0000", 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.640335+0000", 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.640336+0000", 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.15e-06 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.640358+0000", 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2365999999999999e-05 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.640368+0000", 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8539999999999992e-06 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.640375+0000", 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7909999999999999e-06 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642649+0000", 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022740939999999999 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642787+0000", 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137622 2024-08-29T15:12:03.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642934+0000", 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014766900000000001 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651028+0000", 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080933740000000004 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651079+0000", 2024-08-29T15:12:03.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1688999999999999e-05 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759293+0000", 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10821370900000001 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759545+0000", 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025190800000000001 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759712+0000", 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016724700000000001 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68626 5.8 5:16d5d3c4:::benchmark_data_smithi027_135490_object68625:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.642681+0000", 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.558170603000001, 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.117318329, 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.260 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68626 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642681+0000", 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642681+0000", 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642682+0000", 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1500000000000004e-07 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642737+0000", 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5284000000000001e-05 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642747+0000", 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8940000000000005e-06 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642754+0000", 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0690000000000004e-06 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645899+0000", 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031445639999999999 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.646028+0000", 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129176 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.646151+0000", 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012342899999999999 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759723+0000", 2024-08-29T15:12:03.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.113572126 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759767+0000", 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3591000000000003e-05 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759853+0000", 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5884000000000006e-05 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759922+0000", 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9247999999999998e-05 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760000+0000", 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7448999999999997e-05 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68627 5.8 5:1ea3b4ea:::benchmark_data_smithi027_135490_object68626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.266 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.642766+0000", 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.558085673000001, 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11752507500000001, 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68627 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642766+0000", 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642766+0000", 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642767+0000", 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9299999999999998e-07 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642787+0000", 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0035e-05 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642794+0000", 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3320000000000003e-06 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.642800+0000", 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4569999999999996e-06 2024-08-29T15:12:03.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648146+0000", 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053466219999999997 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648270+0000", 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012386600000000001 2024-08-29T15:12:03.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648393+0000", 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012327 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759728+0000", 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.111334973 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759807+0000", 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8975999999999997e-05 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760045+0000", 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000237827 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760143+0000", 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7532000000000001e-05 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760291+0000", 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014859199999999999 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.273 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68630 5.8 5:1e368417:::benchmark_data_smithi027_135490_object68629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.644255+0000", 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.556596982000002, 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11627560300000001, 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68630 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.644255+0000", 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.644255+0000", 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.644256+0000", 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1260000000000001e-06 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.644288+0000", 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.188e-05 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.644300+0000", 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1620000000000001e-05 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.644306+0000", 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9870000000000002e-06 2024-08-29T15:12:03.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651130+0000", 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068231719999999997 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651225+0000", 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5517e-05 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:03.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651359+0000", 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013347500000000001 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760305+0000", 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10894612200000001 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760345+0000", 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0649000000000003e-05 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760391+0000", 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5510999999999998e-05 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760490+0000", 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9355e-05 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760530+0000", 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0188999999999998e-05 2024-08-29T15:12:03.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68632 5.a 5:591a752d:::benchmark_data_smithi027_135490_object68631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.645144+0000", 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.555708133000003, 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10937205699999999, 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68632 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645144+0000", 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645144+0000", 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645145+0000", 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7999999999999993e-07 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645166+0000", 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0920999999999998e-05 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645176+0000", 2024-08-29T15:12:03.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0750999999999999e-05 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645183+0000", 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.708e-06 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645797+0000", 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00061362700000000005 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.645944+0000", 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014723899999999999 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.646084+0000", 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013979199999999999 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649300+0000", 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032160219999999998 2024-08-29T15:12:03.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.754334+0000", 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.105033882 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.754418+0000", 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4264999999999994e-05 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.754472+0000", 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3761000000000002e-05 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.754516+0000", 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4109000000000001e-05 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.286 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68633 5.b 5:d2f8fe1c:::benchmark_data_smithi027_135490_object68632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.647450+0000", 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.553402146000003, 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.123813615, 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68633 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647450+0000", 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647450+0000", 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647451+0000", 2024-08-29T15:12:03.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.559e-06 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647472+0000", 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0582e-05 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647482+0000", 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9809999999999998e-06 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647488+0000", 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.426e-06 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648380+0000", 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000891601 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648487+0000", 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106988 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648608+0000", 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012088599999999999 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760184+0000", 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.111575813 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760225+0000", 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1853000000000001e-05 2024-08-29T15:12:03.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771118+0000", 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010892924999999999 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771196+0000", 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7387999999999998e-05 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771263+0000", 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7613000000000005e-05 2024-08-29T15:12:03.292 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.293 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.293 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:03.294 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68635 5.b 5:dcf0f631:::benchmark_data_smithi027_135490_object68634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.294 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.647501+0000", 2024-08-29T15:12:03.294 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.553350416000001, 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.124082939, 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68635 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647501+0000", 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647501+0000", 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647502+0000", 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.58e-07 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647519+0000", 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7550999999999998e-05 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647526+0000", 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6309999999999997e-06 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647531+0000", 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0370000000000002e-06 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651133+0000", 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003602035 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651265+0000", 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131846 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:03.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651407+0000", 2024-08-29T15:12:03.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014216399999999999 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760269+0000", 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.108861867 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760307+0000", 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8053999999999999e-05 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771272+0000", 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010965216 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771351+0000", 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8393000000000007e-05 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771584+0000", 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000233687 2024-08-29T15:12:03.301 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68638 5.b 5:d8fff075:::benchmark_data_smithi027_135490_object68637:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.647877+0000", 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.552974487, 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12386913400000001, 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68638 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647877+0000", 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647877+0000", 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647878+0000", 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0299999999999998e-07 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647904+0000", 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5627e-05 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647915+0000", 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.149e-05 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.647922+0000", 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9990000000000002e-06 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.757672+0000", 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.109749981 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.757791+0000", 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118281 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.757931+0000", 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014079300000000001 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760349+0000", 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024177589999999998 2024-08-29T15:12:03.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.765324+0000", 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049751769999999999 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771591+0000", 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062670820000000002 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771670+0000", 2024-08-29T15:12:03.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9111999999999997e-05 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771747+0000", 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6030000000000002e-05 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68639 5.a 5:5e8ee11c:::benchmark_data_smithi027_135490_object68638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.649260+0000", 2024-08-29T15:12:03.308 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.551591397000003, 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.110636464, 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68639 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649260+0000", 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649260+0000", 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649262+0000", 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.108e-06 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649282+0000", 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0669000000000001e-05 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649410+0000", 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012768400000000001 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649418+0000", 2024-08-29T15:12:03.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9610000000000005e-06 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649474+0000", 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5745999999999998e-05 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649621+0000", 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014779600000000001 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649748+0000", 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012633100000000001 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.754376+0000", 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.104628577 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.754563+0000", 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000186236 2024-08-29T15:12:03.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.757116+0000", 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025537340000000002 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.757275+0000", 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015859700000000001 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.759897+0000", 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026220250000000001 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68641 5.8 5:1475e4b5:::benchmark_data_smithi027_135490_object68640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.649437+0000", 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.551415132999999, 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.113951883, 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.315 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68641 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649437+0000", 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649437+0000", 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649437+0000", 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0999999999999998e-07 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649458+0000", 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1038000000000001e-05 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649467+0000", 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.003e-06 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.649473+0000", 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7289999999999997e-06 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.756818+0000", 2024-08-29T15:12:03.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10734484800000001 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.756986+0000", 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016761200000000001 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.757189+0000", 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020349699999999999 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760357+0000", 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031674619999999998 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760579+0000", 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022237600000000001 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.763226+0000", 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026472940000000001 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.763340+0000", 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011416099999999999 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.763389+0000", 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8253e-05 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:68627 5.8 e117/111 ondisk, result = 0)", 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.651397+0000", 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.549454822000001, 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.108908755, 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651397+0000", 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651397+0000", 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651397+0000", 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.36e-07 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651399+0000", 2024-08-29T15:12:03.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.119e-06 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651406+0000", 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1829999999999999e-06 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.651410+0000", 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.561e-06 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760034+0000", 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.108623381 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760040+0000", 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6549999999999998e-06 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.760306+0000", 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026561999999999999 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.325 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68896 5.b 5:d4a8571b:::benchmark_data_smithi027_135490_object68895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 58.298636029000001, 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15188533300000001, 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:03.326 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68896 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902217+0000", 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2399999999999996e-07 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902238+0000", 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1038000000000001e-05 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902505+0000", 2024-08-29T15:12:03.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026729299999999999 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902514+0000", 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5869999999999993e-06 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908889+0000", 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063751140000000003 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909004+0000", 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011566299999999999 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909129+0000", 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012426399999999999 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.911589+0000", 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024604990000000001 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912563+0000", 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000974156 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053999+0000", 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14143561299999999 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054055+0000", 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6283999999999998e-05 2024-08-29T15:12:03.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054101+0000", 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5898e-05 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:03.332 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:12:03.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:03.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:03.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:03.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:03.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total time run: 90.0097 2024-08-29T15:12:03.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total writes made: 133749 2024-08-29T15:12:03.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Write size: 65536 2024-08-29T15:12:03.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object size: 65536 2024-08-29T15:12:03.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Bandwidth (MB/sec): 92.8712 2024-08-29T15:12:03.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Bandwidth: 40.732 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max bandwidth (MB/sec): 251.5 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min bandwidth (MB/sec): 50.875 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average IOPS: 1485 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev IOPS: 651.712 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max IOPS: 4024 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min IOPS: 814 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average Latency(s): 0.0107633 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Latency(s): 0.00819367 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max latency(s): 0.237678 2024-08-29T15:12:03.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min latency(s): 0.00105259 2024-08-29T15:12:03.486 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Cleaning up (deleting benchmark objects) 2024-08-29T15:12:03.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:03.488 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:03.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:03.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:03.623 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:03.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:03.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:03.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:03.760 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:03.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:03.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:03.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:03.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:03.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:03 smithi171 ceph-mon[90031]: pgmap v351: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:12:04.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:03 smithi027 ceph-mon[98970]: pgmap v351: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:12:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:03 smithi027 ceph-mon[93921]: pgmap v351: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:12:04.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:04.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.168 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:04.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:04.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:04.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:04.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:12:04.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:04.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:04.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:04.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:04.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:04.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:04.717 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:04.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:04.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:04.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:04.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:04.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:04.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:05.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:05.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:05.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:05.230 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:05.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:05.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:05.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:05.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:05.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:05.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:05.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:05.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:05.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:05.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:05.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:05.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:05.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:05.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:05.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:05.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:05.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:05.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:05 smithi027 ceph-mon[98970]: pgmap v352: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 973 op/s 2024-08-29T15:12:05.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:05 smithi027 ceph-mon[93921]: pgmap v352: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 973 op/s 2024-08-29T15:12:05.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:12:05.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:05.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:05.950 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:05.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:05 smithi171 ceph-mon[90031]: pgmap v352: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 973 op/s 2024-08-29T15:12:06.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:06.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:06.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:06.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:06.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:06.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:06.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:06.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:06.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:06.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:06.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:06.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:06.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:06.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:06.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:06.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:06.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:06.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:06.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:06.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:06.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:06.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:06.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:06.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:06.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:06.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:07.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:07.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:07.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:07.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:07.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:07.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:07.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:07.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:07.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:07.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:07.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:07.487 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:07.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:07.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:07.614 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:07.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:07.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:07.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:07.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:07.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:07.855 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:07.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:07.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:07.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:07 smithi171 ceph-mon[90031]: pgmap v353: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 972 op/s 2024-08-29T15:12:08.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:07 smithi027 ceph-mon[98970]: pgmap v353: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 972 op/s 2024-08-29T15:12:08.021 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:07 smithi027 ceph-mon[93921]: pgmap v353: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 972 op/s 2024-08-29T15:12:08.024 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:08.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:08.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:08.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:08.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:08.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:08.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:08.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:08.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:08.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:08.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:08.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:08.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:08.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:08.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:08.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:08.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:08.737 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:08.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:08.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:08.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:08.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:08.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:08.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:09.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:09.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:09.052 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:12:09.052 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:12:09.052 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:12:09.052 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:12:09.053 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:12:09.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:12:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:12:09] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-08-29T15:12:09.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:09.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:09.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:09.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:09.293 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:09.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:09.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:09.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:09.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:09.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:09.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:09.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:09.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:09.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:09.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:09.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:09.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:09.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:09.722 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:09.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:09.731 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:12:09.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:09 smithi027 ceph-mon[98970]: pgmap v354: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 47 MiB/s wr, 814 op/s 2024-08-29T15:12:09.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/228509028' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:12:09.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:12:09.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:09 smithi027 ceph-mon[93921]: pgmap v354: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 47 MiB/s wr, 814 op/s 2024-08-29T15:12:09.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/228509028' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:12:09.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:12:09.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:09.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:09.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:09.871 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:12:09.871 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:12:09.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:10.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:09 smithi171 ceph-mon[90031]: pgmap v354: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 47 MiB/s wr, 814 op/s 2024-08-29T15:12:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/228509028' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:12:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:12:10.078 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:10.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:10.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:10.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:10.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:10.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.282 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:10.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:10.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:10.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:10.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:10.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:10.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:10.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:10.688 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:10.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:10.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:10 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:10.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:10 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:10.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:10 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:12:10.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:10 smithi027 ceph-mon[93921]: osdmap e122: 8 total, 8 up, 8 in 2024-08-29T15:12:10.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:10.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:10.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:10.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:10.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:11.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:10 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:11.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:10 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:12:11.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:10 smithi171 ceph-mon[90031]: osdmap e122: 8 total, 8 up, 8 in 2024-08-29T15:12:11.061 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:11.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:10 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:12:11.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:10 smithi027 ceph-mon[98970]: osdmap e122: 8 total, 8 up, 8 in 2024-08-29T15:12:11.095 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:11.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:11.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:11.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:11.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:11.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:11.336 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:11.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:11.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:11.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:11.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:11.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:11.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:11.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:11.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:11.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:11.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:11.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:11.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:11.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:11.859 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:11 smithi027 ceph-mon[98970]: pgmap v356: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 933 op/s 2024-08-29T15:12:11.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:11 smithi027 ceph-mon[93921]: pgmap v356: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 933 op/s 2024-08-29T15:12:11.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:11.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:11.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:12.020 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:12.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:12.048 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:12.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:12.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:12.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:12.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:11 smithi171 ceph-mon[90031]: pgmap v356: 89 pgs: 89 active+clean; 7.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 933 op/s 2024-08-29T15:12:12.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:12.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:12.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:12.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:12.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:12.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:12.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:12:12.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:12.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:12.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:12.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:12.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:12.730 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:12.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:12.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:12.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:12.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:12.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:12.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:12.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:13.016 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:13.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:13.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:13.168 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:13.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:13.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:13.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:13.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:13.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:13.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:13.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:13.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:13.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:13.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:13.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:13.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:13.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:13.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:13.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:13 smithi027 ceph-mon[98970]: pgmap v357: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 929 op/s 2024-08-29T15:12:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:13 smithi027 ceph-mon[93921]: pgmap v357: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 929 op/s 2024-08-29T15:12:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:14.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:14.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:14.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:14.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:14.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:14.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:14.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:13 smithi171 ceph-mon[90031]: pgmap v357: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 929 op/s 2024-08-29T15:12:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:14.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:14.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:14.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:14.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:14.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:14.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:14.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:14.557 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:12:14.557 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:12:14.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.557 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:144220 5.0 5:092645ea:::benchmark_data_smithi027_135490_object10468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:12:14.546641+0000", 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0088493879999999997, 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088712940000000001, 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 144220 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546641+0000", 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546641+0000", 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546643+0000", 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0779999999999998e-06 2024-08-29T15:12:14.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546649+0000", 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5430000000000001e-06 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546666+0000", 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6932999999999999e-05 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546675+0000", 2024-08-29T15:12:14.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8999999999999995e-06 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546743+0000", 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8631000000000001e-05 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.546890+0000", 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014681500000000001 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.547012+0000", 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012231699999999999 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.551431+0000", 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044187619999999997 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:14.551441+0000", 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3889999999999994e-06 2024-08-29T15:12:14.562 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:14.563 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:12:14.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:14.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:12:14.563 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:12:14.563 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2024-08-29T15:12:14.563 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:12:14.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:14.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:14.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:14.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:14.769 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:14.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:14.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:14.873 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:12:14.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 0 1 2024-08-29T15:12:14.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:14.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:14.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:15.041 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:15.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:15.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:15.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:15.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:15.223 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:15.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:15.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:12:15.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:15.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:15.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:15.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:15.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:15.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:15.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:15.660 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:15.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:15.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:15.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:15.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:15.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:15.846 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:15 smithi027 ceph-mon[93921]: pgmap v358: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 937 op/s 2024-08-29T15:12:15.846 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:15 smithi027 ceph-mon[98970]: pgmap v358: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 937 op/s 2024-08-29T15:12:15.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:15.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:15.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:15.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:15.972 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:16.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:16.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:16.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:16.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:16.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:16.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:16.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:16.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:15 smithi171 ceph-mon[90031]: pgmap v358: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 937 op/s 2024-08-29T15:12:16.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:16.349 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:16.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:16.450 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:16.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:16.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:16.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:12:16.504 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:12:16.513 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:12:16.514 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:12:16.520 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:16.520 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:16.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:12:16.550 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:12:16.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:16.586 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:12:16.587 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:12:16.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:16.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:16.629 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:12:16.629 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:12:16.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:16.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:16.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:16.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:16.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:16.889 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:16.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:16.987 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:16 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1366878006' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:16.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:16 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1366878006' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:16.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:17.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:17.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:17.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:17.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:17.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:17.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:16 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1366878006' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:17.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:17.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:17.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:17.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:17.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:17.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:17.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:17.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:17.703 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:17.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:17.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:17.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:17.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:17.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:17.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:17.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:17 smithi027 ceph-mon[93921]: pgmap v359: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 672 KiB/s wr, 943 op/s 2024-08-29T15:12:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:17 smithi027 ceph-mon[98970]: pgmap v359: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 672 KiB/s wr, 943 op/s 2024-08-29T15:12:18.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:18.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:18.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:18.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:18.220 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:18.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:18.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:18.266 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:17 smithi171 ceph-mon[90031]: pgmap v359: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 672 KiB/s wr, 943 op/s 2024-08-29T15:12:18.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:18.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:18.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:18.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:18.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:18.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:18.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:18.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:18.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:18.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:18.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:18.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:18.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:18.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:18.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:18.947 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:18.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:18.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:19.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:19.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:19.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:19.138 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:12:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:12:19] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-29T15:12:19.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:19.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:19.230 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:19.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:19.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:19.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:19.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:19.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:19.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:19.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:19.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:19.614 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:19.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:19.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:19.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:19.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:19.794 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:19.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:19.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:19.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:19.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:19.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:20.021 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:19 smithi027 ceph-mon[93921]: pgmap v360: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 51 KiB/s wr, 942 op/s 2024-08-29T15:12:20.024 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:19 smithi027 ceph-mon[98970]: pgmap v360: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 51 KiB/s wr, 942 op/s 2024-08-29T15:12:20.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:20.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:20.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:20.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:20.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:20.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:20.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:19 smithi171 ceph-mon[90031]: pgmap v360: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 51 KiB/s wr, 942 op/s 2024-08-29T15:12:20.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:20.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:20.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:20.468 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:20.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:20.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:20.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:20.575 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:12:20.575 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:12:20.601 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:20.630 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:20.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:20.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:20.760 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:20.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:20.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:20.862 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:20.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:20.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:21.001 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:21.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:21.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:21.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:21.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:21.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:21.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:21.316 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:21.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:21.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:21.450 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:21.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:21.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:21.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:21.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:21.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:21.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:21.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:21.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:21.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:21.821 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.0 to 1 (10000) 2024-08-29T15:12:21.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:21.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:21.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:21.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:21.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:12:21.971 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:22.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:22.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:22.126 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:22.228 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:22.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:21 smithi171 ceph-mon[90031]: pgmap v361: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2024-08-29T15:12:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3790855587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:12:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:21 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:12:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:21 smithi027 ceph-mon[98970]: pgmap v361: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2024-08-29T15:12:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3790855587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:12:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:21 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:12:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:21 smithi027 ceph-mon[93921]: pgmap v361: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2024-08-29T15:12:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3790855587' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:12:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:21 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:12:22.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:22.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:22.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:22.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:22.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:22.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:22.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:22.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:22.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:22.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:22.950 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:22.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:22.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:23.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:23.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:12:23.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:22 smithi027 ceph-mon[98970]: osdmap e123: 8 total, 8 up, 8 in 2024-08-29T15:12:23.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:12:23.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:22 smithi027 ceph-mon[93921]: osdmap e123: 8 total, 8 up, 8 in 2024-08-29T15:12:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:23.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:23.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:23.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:23.230 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:23.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:12:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:22 smithi171 ceph-mon[90031]: osdmap e123: 8 total, 8 up, 8 in 2024-08-29T15:12:23.332 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:23.375 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:23.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:23.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:23.516 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:23.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:23.617 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:23.652 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:23.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:23.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:23.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:23.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:23.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:23.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:23.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:23.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:23.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:24.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:24.059 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:24.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:24.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:23 smithi027 ceph-mon[98970]: pgmap v363: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2024-08-29T15:12:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:23 smithi027 ceph-mon[93921]: pgmap v363: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2024-08-29T15:12:24.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:24.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:24.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:24.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 1 1 2024-08-29T15:12:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:23 smithi171 ceph-mon[90031]: pgmap v363: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2024-08-29T15:12:24.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:24.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:24.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:24.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:24.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:24.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:24.519 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:24.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:24.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:24.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:24.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:24.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:24.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:24.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:24.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:24.912 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:24.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:24.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:25.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:25.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:25.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:25.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:25.105 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:12:25.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:25.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:25.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:25.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:25.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:25.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:25.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:25.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:25.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:25.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:25.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:25.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:25.598 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:25.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:25.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:25.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:25.736 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:25.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:25.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:25.854 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:12:25.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:25.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:25.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:26.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:26.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:26.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:25 smithi027 ceph-mon[98970]: pgmap v364: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-08-29T15:12:26.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1227400638' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:26.018 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:25 smithi027 ceph-mon[93921]: pgmap v364: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-08-29T15:12:26.018 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1227400638' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:26.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:26.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:26.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:26.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:25 smithi171 ceph-mon[90031]: pgmap v364: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-08-29T15:12:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1227400638' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:26.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:26.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:26.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:26.298 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:26.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:26.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:26.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:26.536 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:26.572 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:26.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:26.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:26.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:26.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:26.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:26.742 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:26.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:26.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:26.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:26.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:27.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:27.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:26 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:27.031 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1227400638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:12:27.031 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:26 smithi027 ceph-mon[93921]: osdmap e124: 8 total, 8 up, 8 in 2024-08-29T15:12:27.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:26 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:27.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1227400638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:12:27.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:26 smithi027 ceph-mon[98970]: osdmap e124: 8 total, 8 up, 8 in 2024-08-29T15:12:27.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:27.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:27.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:26 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1227400638' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:12:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:26 smithi171 ceph-mon[90031]: osdmap e124: 8 total, 8 up, 8 in 2024-08-29T15:12:27.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:27.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:27.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.419 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:27.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:27.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:27.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:27.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:27.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:27.691 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:27.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:27.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:27.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:27.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:27.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:28.078 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:28.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:28.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:28.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:28.257 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:28.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:27 smithi171 ceph-mon[90031]: pgmap v366: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-29T15:12:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:27 smithi027 ceph-mon[98970]: pgmap v366: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-29T15:12:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:27 smithi027 ceph-mon[93921]: pgmap v366: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-29T15:12:28.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:28.394 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:28.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:28.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:28.523 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:28.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:28.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:28.642 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:12:28.643 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:12:28.664 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:28.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:28.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:12:28.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:28.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:28.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:28.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:28.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:28.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:29.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:29.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:12:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:12:29] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-29T15:12:29.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:29.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:29.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:29.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:29.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:29.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:29.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:29.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:29.384 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:29.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:29.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:29.515 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:29.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:29.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:29.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:29.647 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:29.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:29.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:29.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:29.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:29.888 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:29.931 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:29.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:30.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:30.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:30.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:29 smithi027 ceph-mon[98970]: pgmap v367: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-08-29T15:12:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:29 smithi027 ceph-mon[93921]: pgmap v367: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-08-29T15:12:30.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:30.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:29 smithi171 ceph-mon[90031]: pgmap v367: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-08-29T15:12:30.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:30.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:30.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:30.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:30.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:30.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:30.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:30.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:30.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:30.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:30.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:30.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:30.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:30.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:30.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:30.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:30.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:30.872 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:30.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:30.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:31.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:31.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:31.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:31.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:31.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:31.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:12:31.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:31.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:31.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:31.378 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:31.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:31.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:31.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:31.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:31.561 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:12:31.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:31.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:31.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:31.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:31.726 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:31.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:31.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:31.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:31.876 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:31.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:31.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:31.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:31 smithi027 ceph-mon[93921]: pgmap v368: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-29T15:12:31.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:31 smithi027 ceph-mon[98970]: pgmap v368: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-29T15:12:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:32.009 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:32.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:31 smithi171 ceph-mon[90031]: pgmap v368: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-29T15:12:32.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:32.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:32.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:32.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:32.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:32.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:32.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:32.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:32.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:32.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:32.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:32.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:32.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:32.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:32.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:32.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:32.693 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:32.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:32.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:32.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:32.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:32.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:32.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:32.900 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.1 to 1 (10000) 2024-08-29T15:12:32.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:32.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3888360591' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:12:32.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3888360591' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:12:33.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:33.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:33.126 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3888360591' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:12:33.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:33.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:33.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:33.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:33.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:33.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:33.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:33.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:33.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:33.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:33.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:33.588 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:12:33.588 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:12:33.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:33.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:33.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:33.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:33.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 2 1 2024-08-29T15:12:33.756 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:33.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:33.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:33.902 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:33.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:33.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:34.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:33 smithi027 ceph-mon[98970]: pgmap v369: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-29T15:12:34.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3888360591' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:12:34.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:33 smithi027 ceph-mon[98970]: osdmap e125: 8 total, 8 up, 8 in 2024-08-29T15:12:34.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:33 smithi027 ceph-mon[93921]: pgmap v369: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-29T15:12:34.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3888360591' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:12:34.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:33 smithi027 ceph-mon[93921]: osdmap e125: 8 total, 8 up, 8 in 2024-08-29T15:12:34.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:34.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:34.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:34.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:34.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:34.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:34.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:33 smithi171 ceph-mon[90031]: pgmap v369: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-29T15:12:34.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3888360591' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:12:34.244 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:33 smithi171 ceph-mon[90031]: osdmap e125: 8 total, 8 up, 8 in 2024-08-29T15:12:34.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:12:34.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:34.333 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:12:34.333 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:12:34.355 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:12:34.355 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:12:34.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:34.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:34.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:34.385 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:12:34.385 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:12:34.415 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:12:34.415 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:12:34.452 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:12:34.453 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:12:34.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:34.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:34.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:34.514 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:12:34.514 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:12:34.545 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:12:34.545 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:12:34.585 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:12:34.585 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:12:34.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:34.637 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:34.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:34.674 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:12:34.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:34.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:34.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:34.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:34.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:34.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:34.985 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1676026375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:35.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:35.051 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:35.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:35.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:35.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:35.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:35.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:35.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:35.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1676026375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:35.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1676026375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:35.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:35.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:35.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:35.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:35.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:35.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:35.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:35.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:35.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:35.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:35.704 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:35.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:35.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:35.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:35.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:35.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:35.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:36.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:36.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:36.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:35 smithi027 ceph-mon[98970]: pgmap v371: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-08-29T15:12:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:35 smithi027 ceph-mon[93921]: pgmap v371: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-08-29T15:12:36.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:36.129 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:35 smithi171 ceph-mon[90031]: pgmap v371: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-08-29T15:12:36.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:36.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:36.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:36.287 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:36.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:36.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:36.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:36.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:36.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:36.684 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:12:36.684 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:12:36.684 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:12:36.684 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:12:36.684 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:12:36.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:36.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:36.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:36.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:36.959 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:36.989 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:36.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:36.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:37.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:37.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:37.121 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:37.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:37.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:37.225 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:37.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:37.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:37.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:37.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:37.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:37.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:37.401 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:12:37.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:37.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:37.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:37.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:37.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:37.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:37.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:37.778 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:12:37.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:37.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:37.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:37.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:37.927 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:12:37.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:37.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:38.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:38.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:38.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:38.179 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:38.211 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:37 smithi171 ceph-mon[90031]: pgmap v372: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-08-29T15:12:38.212 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:37 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4287051597' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:12:38.212 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:37 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:12:38.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:38.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:38.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:37 smithi027 ceph-mon[93921]: pgmap v372: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-08-29T15:12:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:37 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4287051597' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:12:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:37 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:12:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:37 smithi027 ceph-mon[98970]: pgmap v372: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-08-29T15:12:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:37 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4287051597' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:12:38.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:37 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:12:38.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:38.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:38.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:38.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:38.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:38.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:38.487 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:38.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:38.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:12:38.589 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:38.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:38.630 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:38.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:38.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:38.769 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:38.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:38.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:38.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:38.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:39.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:39.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:39.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:39.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:39.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:39.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:39.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:39.219 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:38 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:38 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:12:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:38 smithi171 ceph-mon[90031]: osdmap e126: 8 total, 8 up, 8 in 2024-08-29T15:12:39.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:39.313 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:12:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:12:39] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-29T15:12:39.313 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:38 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:39.313 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:38 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:12:39.313 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:38 smithi027 ceph-mon[98970]: osdmap e126: 8 total, 8 up, 8 in 2024-08-29T15:12:39.313 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:38 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:12:39.314 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:38 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:12:39.314 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:38 smithi027 ceph-mon[93921]: osdmap e126: 8 total, 8 up, 8 in 2024-08-29T15:12:39.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:39.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:39.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:39.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:39.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:39.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:39.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:39.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:39.611 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:39.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:39.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:39.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:39.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:23215 5.e 5:7c274c1d:::benchmark_data_smithi027_135490_object23214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 118.813444971, 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13346666099999999, 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.777 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 23215 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.778 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960452+0000", 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960487+0000", 2024-08-29T15:12:39.779 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4594000000000001e-05 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960497+0000", 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0348e-05 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960504+0000", 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6660000000000002e-06 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.780 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960556+0000", 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2392999999999998e-05 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960676+0000", 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119975 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.781 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960812+0000", 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013557900000000001 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.961708+0000", 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00089608400000000001 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.962192+0000", 2024-08-29T15:12:39.782 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00048372900000000002 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093816+0000", 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13162422100000001 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093876+0000", 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9784000000000002e-05 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.783 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093920+0000", 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4129999999999999e-05 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68906 5.e 5:73dcaf78:::benchmark_data_smithi027_135490_object68905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 94.865419430000003, 2024-08-29T15:12:39.784 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14042518300000001, 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68906 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.785 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908477+0000", 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.786 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908551+0000", 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3900999999999996e-05 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908563+0000", 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1518999999999999e-05 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908570+0000", 2024-08-29T15:12:39.787 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0559999999999999e-06 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908679+0000", 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010953700000000001 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908808+0000", 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012906199999999999 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:39.788 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908966+0000", 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000158018 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913092+0000", 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041260869999999996 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.916369+0000", 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032766510000000002 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.789 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048843+0000", 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13247402 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048878+0000", 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5175000000000001e-05 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.790 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048904+0000", 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5653e-05 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68910 5.e 5:7bc74f0d:::benchmark_data_smithi027_135490_object68909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 94.861172504999999, 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13951414700000001, 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.791 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.792 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.792 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68910 2024-08-29T15:12:39.792 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.792 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912724+0000", 2024-08-29T15:12:39.793 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912763+0000", 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9082999999999998e-05 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912777+0000", 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4174e-05 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912784+0000", 2024-08-29T15:12:39.794 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0419999999999997e-06 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913119+0000", 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00033500199999999999 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913247+0000", 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012835799999999999 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.795 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913411+0000", 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016349600000000001 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.916432+0000", 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030208549999999998 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.917924+0000", 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014924789999999999 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.796 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052043+0000", 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13411882899999999 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052180+0000", 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013707500000000001 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052240+0000", 2024-08-29T15:12:39.797 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9765999999999997e-05 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71204 5.e 5:72845dde:::benchmark_data_smithi027_135490_object71203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 92.937243506000002, 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10090418, 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.798 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71204 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.799 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836653+0000", 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.800 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836686+0000", 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3639000000000001e-05 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836700+0000", 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3372e-05 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836715+0000", 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5500000000000001e-05 2024-08-29T15:12:39.801 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836792+0000", 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7065999999999997e-05 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836904+0000", 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111976 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.837024+0000", 2024-08-29T15:12:39.802 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119389 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891058+0000", 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.054034182 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891105+0000", 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6613999999999998e-05 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.803 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937443+0000", 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046338468000000001 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937523+0000", 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0029000000000002e-05 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937559+0000", 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5862999999999999e-05 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.804 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71206 5.e 5:74b85a3d:::benchmark_data_smithi027_135490_object71205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 92.935345526999996, 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.099155491999999998, 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.805 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71206 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.806 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838551+0000", 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838623+0000", 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2087000000000003e-05 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838635+0000", 2024-08-29T15:12:39.807 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.164e-05 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838642+0000", 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7109999999999998e-06 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839510+0000", 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00086809300000000001 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.808 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839722+0000", 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021236700000000001 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839853+0000", 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013045999999999999 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891096+0000", 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051243155999999998 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.809 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891152+0000", 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5998000000000002e-05 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937606+0000", 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046454595000000001 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937672+0000", 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5773999999999997e-05 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.810 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937708+0000", 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5984e-05 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71207 5.e 5:712e06d6:::benchmark_data_smithi027_135490_object71206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 92.934323302999999, 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.101516334, 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.811 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71207 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:12:39.812 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839574+0000", 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839633+0000", 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8607999999999999e-05 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839644+0000", 2024-08-29T15:12:39.813 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.187e-05 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839652+0000", 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1910000000000003e-06 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844029+0000", 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004377148 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.814 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844134+0000", 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010534100000000001 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844266+0000", 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000132223 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891145+0000", 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046878523999999998 2024-08-29T15:12:39.815 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891156+0000", 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1148000000000001e-05 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.940954+0000", 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049798430999999997 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.816 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.941041+0000", 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7043e-05 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.941091+0000", 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9765999999999998e-05 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71214 5.1 5:82269f1e:::benchmark_data_smithi027_135490_object71213:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:39.817 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.843154+0000", 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 92.930744383999993, 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.091169491000000005, 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71214 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.818 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843154+0000", 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843154+0000", 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843152+0000", 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.819 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843189+0000", 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6505999999999999e-05 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843202+0000", 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2629999999999999e-05 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843209+0000", 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9630000000000001e-06 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.820 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844577+0000", 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001368729 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844683+0000", 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010556900000000001 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844818+0000", 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013473900000000001 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.821 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.893196+0000", 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.048378681999999999 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.893415+0000", 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000218371 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934240+0000", 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040825314000000001 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.822 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934288+0000", 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8477999999999999e-05 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934323+0000", 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4941000000000003e-05 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.823 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82159 5.1 5:8a273d61:::benchmark_data_smithi027_135490_object82158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.458161767000007, 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098041845000000002, 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82159 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.824 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315735+0000", 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.825 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315774+0000", 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8905000000000001e-05 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315787+0000", 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.288e-05 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.826 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315794+0000", 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9639999999999997e-06 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322730+0000", 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0069358759999999997 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322849+0000", 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119078 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:12:39.827 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323562+0000", 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00071293799999999998 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329649+0000", 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060869790000000002 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329680+0000", 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1455999999999998e-05 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413675+0000", 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083995138999999996 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413734+0000", 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9247999999999999e-05 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413778+0000", 2024-08-29T15:12:39.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.384e-05 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82163 5.1 5:8a4e1f99:::benchmark_data_smithi027_135490_object82162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.457223945999999, 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100377148, 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82163 2024-08-29T15:12:39.830 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.831 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316673+0000", 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316702+0000", 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8294e-05 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316719+0000", 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7774e-05 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316726+0000", 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9430000000000003e-06 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326240+0000", 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0095137480000000007 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326418+0000", 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017811300000000001 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:12:39.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326530+0000", 2024-08-29T15:12:39.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111513 2024-08-29T15:12:39.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335223+0000", 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086934460000000005 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413816+0000", 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078592598999999999 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416754+0000", 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029382990000000001 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416879+0000", 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012490299999999999 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417051+0000", 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000172492 2024-08-29T15:12:39.836 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82171 5.9 e118/111 5:952386eb:::benchmark_data_smithi027_135490_object82170:head v 118'5035, mlcod=118'5031)", 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.450477371999995, 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093309638, 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323420+0000", 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:39.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323447+0000", 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.656e-05 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323458+0000", 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1242e-05 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323464+0000", 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5130000000000002e-06 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323833+0000", 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00036861199999999999 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323850+0000", 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6948000000000001e-05 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416578+0000", 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.092727863999999993 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416730+0000", 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015279099999999999 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.841 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82176 5.9 e118/111 5:962339e0:::benchmark_data_smithi027_135490_object82175:head v 118'5036, mlcod=118'5033)", 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.326875+0000", 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.447022785000001, 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090324798999999997, 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326875+0000", 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326875+0000", 2024-08-29T15:12:39.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326875+0000", 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326895+0000", 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0259999999999999e-05 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326904+0000", 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5590000000000006e-06 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326909+0000", 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4330000000000003e-06 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329354+0000", 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0024452879999999999 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329372+0000", 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7753000000000001e-05 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417067+0000", 2024-08-29T15:12:39.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.087694921999999995 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417200+0000", 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013321399999999999 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:82159 5.1 e118/111 ondisk, result = 0)", 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.327042+0000", 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.446856580000002, 2024-08-29T15:12:39.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090021891000000007, 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.327042+0000", 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.327042+0000", 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.327041+0000", 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.327046+0000", 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6920000000000001e-06 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.327056+0000", 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0278000000000001e-05 2024-08-29T15:12:39.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.327062+0000", 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1290000000000001e-06 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329669+0000", 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002607304 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.849 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329675+0000", 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0900000000000001e-06 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417064+0000", 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.087388139000000004 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82178 5.9 e118/111 5:90281a6a:::benchmark_data_smithi027_135490_object82177:head v 118'5037, mlcod=118'5033)", 2024-08-29T15:12:39.850 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.329570+0000", 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.444328010000007, 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.087683966000000002, 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329570+0000", 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329570+0000", 2024-08-29T15:12:39.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329569+0000", 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329593+0000", 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3779999999999999e-05 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329603+0000", 2024-08-29T15:12:39.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3959999999999995e-06 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329608+0000", 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5799999999999999e-06 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.331910+0000", 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0023019 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.331934+0000", 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4230999999999999e-05 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417202+0000", 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085268174000000002 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417254+0000", 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1789999999999997e-05 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.854 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82186 5.e 5:75246baf:::benchmark_data_smithi027_135490_object82185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.333041+0000", 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.440857261999994, 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.081148315999999998, 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82186 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.855 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333041+0000", 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333041+0000", 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333039+0000", 2024-08-29T15:12:39.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333078+0000", 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8383999999999997e-05 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333091+0000", 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3164e-05 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333098+0000", 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1609999999999997e-06 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333148+0000", 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0303999999999998e-05 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333268+0000", 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119716 2024-08-29T15:12:39.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333388+0000", 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012021800000000001 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336245+0000", 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028572739999999999 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413761+0000", 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.077515641999999996 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.414111+0000", 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00034963799999999999 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.414157+0000", 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6038000000000002e-05 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.414189+0000", 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2579000000000001e-05 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82180 5.9 e118/111 5:906f714e:::benchmark_data_smithi027_135490_object82179:head v 118'5038, mlcod=118'5033)", 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.333494+0000", 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.440403773, 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083809607999999994, 2024-08-29T15:12:39.861 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333494+0000", 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333494+0000", 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333494+0000", 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333516+0000", 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2249e-05 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333525+0000", 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7860000000000002e-06 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.333530+0000", 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5979999999999996e-06 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335247+0000", 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001716897 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335264+0000", 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6748e-05 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417257+0000", 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.081992458000000004 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417304+0000", 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7561000000000001e-05 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82187 5.c e118/111 5:36a96e4f:::benchmark_data_smithi027_135490_object82186:head v 118'5135, mlcod=118'5131)", 2024-08-29T15:12:39.865 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.334228+0000", 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.439670566999993, 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080018819000000005, 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334228+0000", 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334228+0000", 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334227+0000", 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334286+0000", 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8569000000000001e-05 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334295+0000", 2024-08-29T15:12:39.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3400000000000004e-06 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334300+0000", 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5679999999999999e-06 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334345+0000", 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4240000000000003e-05 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334361+0000", 2024-08-29T15:12:39.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6796999999999999e-05 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.414186+0000", 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.079824488999999998 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.414246+0000", 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0529999999999998e-05 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:82159 5.1 e118/111 ondisk, result = 0)", 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.334932+0000", 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.438966626999999, 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078853037000000001, 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334932+0000", 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334932+0000", 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334931+0000", 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334936+0000", 2024-08-29T15:12:39.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1680000000000003e-06 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334947+0000", 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0769e-05 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334953+0000", 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0050000000000001e-06 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413658+0000", 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078705213999999996 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413669+0000", 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0535e-05 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413785+0000", 2024-08-29T15:12:39.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011604600000000001 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:82163 5.1 e118/111 ondisk, result = 0)", 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.334959+0000", 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.438939468000001, 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078859114999999994, 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.874 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334959+0000", 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334959+0000", 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334959+0000", 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334960+0000", 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8649999999999999e-06 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334965+0000", 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8590000000000004e-06 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334969+0000", 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.337e-06 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413804+0000", 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078834937999999993 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413810+0000", 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7469999999999999e-06 2024-08-29T15:12:39.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413818+0000", 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6199999999999999e-06 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82191 5.5 e118/111 5:a9c233ba:::benchmark_data_smithi027_135490_object82190:head v 118'5112, mlcod=118'5110)", 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.336074+0000", 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.437824445999993, 2024-08-29T15:12:39.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080970154000000003, 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336074+0000", 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336074+0000", 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336073+0000", 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336096+0000", 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3141000000000001e-05 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336102+0000", 2024-08-29T15:12:39.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4720000000000003e-06 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336105+0000", 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8600000000000001e-06 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336144+0000", 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9208999999999997e-05 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336163+0000", 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9038e-05 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416966+0000", 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080802970000000002 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417044+0000", 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7927000000000001e-05 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82193 5.9 e118/111 5:9843b4f0:::benchmark_data_smithi027_135490_object82192:head v 118'5039, mlcod=118'5033)", 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.338558+0000", 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 84.435340604999993, 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083080518000000006, 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338558+0000", 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338558+0000", 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338557+0000", 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338582+0000", 2024-08-29T15:12:39.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5182e-05 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413537+0000", 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.074954813999999995 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413551+0000", 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4609e-05 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413836+0000", 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00028438500000000001 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413856+0000", 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9681e-05 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.421381+0000", 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075259389999999997 2024-08-29T15:12:39.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.421638+0000", 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00025668700000000002 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:39.887 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:12:39.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:39.920 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:39.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:40.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:40.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:40.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:40.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:40.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:40.188 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:39 smithi027 ceph-mon[98970]: pgmap v374: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-08-29T15:12:40.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:39 smithi027 ceph-mon[93921]: pgmap v374: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-08-29T15:12:40.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:40.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:39 smithi171 ceph-mon[90031]: pgmap v374: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2024-08-29T15:12:40.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:40.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:40.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:40.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:40.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:40.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:40.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:40.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:40.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:40.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:40.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:40.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:40.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:12:40.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:40.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:40.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:40.964 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.2 to 1 (10000) 2024-08-29T15:12:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1143355148' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:12:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:40 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:12:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:40 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:12:41.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1143355148' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:12:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:40 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:12:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:40 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:12:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1143355148' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:12:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:40 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:12:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:40 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:12:41.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:41.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:41.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:41.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:12:41.208 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:12:41.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:41.276 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:41.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:41.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:41.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:41.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:41.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:41.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:41.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:41.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:41.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:41.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:41.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:41.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:41.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:41.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:41.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:41.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:41.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:41.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:42.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:42.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:42.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:42.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:42.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:42.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:42.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:42.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:41 smithi171 ceph-mon[90031]: pgmap v375: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-29T15:12:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:41 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:12:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:41 smithi171 ceph-mon[90031]: osdmap e127: 8 total, 8 up, 8 in 2024-08-29T15:12:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:41 smithi027 ceph-mon[98970]: pgmap v375: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-29T15:12:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:41 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:12:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:41 smithi027 ceph-mon[98970]: osdmap e127: 8 total, 8 up, 8 in 2024-08-29T15:12:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:41 smithi027 ceph-mon[93921]: pgmap v375: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-29T15:12:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:41 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:12:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:41 smithi027 ceph-mon[93921]: osdmap e127: 8 total, 8 up, 8 in 2024-08-29T15:12:42.327 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:42.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:42.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:42.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:42.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:42.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:42.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:42.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:42.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:42.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:42.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:42.732 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:42.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 3 1 2024-08-29T15:12:42.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:42.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:42.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:42.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:42.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:43.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:43.053 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:43.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:43.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:43.180 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:12:43.180 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:12:43.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.180 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169323 5.4 e127/111 5:283b067e:::benchmark_data_smithi027_135490_object35571:head v 127'10712, mlcod=127'10710)", 2024-08-29T15:12:43.180 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.143830+0000", 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.034779471999999999, 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034815034000000002, 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.143830+0000", 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.143830+0000", 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.143829+0000", 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.182 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.143837+0000", 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2279999999999998e-06 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.143859+0000", 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1818999999999999e-05 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.143866+0000", 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6869999999999994e-06 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.144044+0000", 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017749899999999999 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.144103+0000", 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9005000000000002e-05 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:169332 5.f 5:f94ad16e:::benchmark_data_smithi027_135490_object35580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.145284+0000", 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.033325536000000003, 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033430297999999997, 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 169332 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.145284+0000", 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.145284+0000", 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.145283+0000", 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.145289+0000", 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.516e-06 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.145299+0000", 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7229999999999999e-06 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.145305+0000", 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9020000000000004e-06 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.177727+0000", 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032421867 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.188 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169328 5.4 e127/111 5:2c9d6459:::benchmark_data_smithi027_135490_object35576:head v 127'10713, mlcod=127'10710)", 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.147874+0000", 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.030735795999999999, 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030899157, 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.189 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.147874+0000", 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.147874+0000", 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.147873+0000", 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.147881+0000", 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7589999999999997e-06 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.147897+0000", 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6821999999999999e-05 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.147904+0000", 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6980000000000001e-06 2024-08-29T15:12:43.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.151233+0000", 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033290870000000001 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.151254+0000", 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.071e-05 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169334 5.4 e127/111 5:2cde86f3:::benchmark_data_smithi027_135490_object35582:head v 127'10714, mlcod=127'10710)", 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.152236+0000", 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.026373840999999999, 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026590227000000001, 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.152236+0000", 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.152236+0000", 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.152235+0000", 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.152250+0000", 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4973e-05 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.152265+0000", 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4688999999999999e-05 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.152271+0000", 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1500000000000004e-06 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.155820+0000", 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003549249 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.156116+0000", 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000295949 2024-08-29T15:12:43.196 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169338 5.6 e127/111 5:67379d2a:::benchmark_data_smithi027_135490_object35586:head v 127'10666, mlcod=127'10664)", 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.158754+0000", 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.019855773, 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020125632000000001, 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.197 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.158754+0000", 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.158754+0000", 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.158753+0000", 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.158760+0000", 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8630000000000008e-06 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.158777+0000", 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6439999999999998e-05 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.158784+0000", 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6959999999999999e-06 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.158961+0000", 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000177598 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.160692+0000", 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017303889999999999 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169340 5.6 e127/111 5:680619a8:::benchmark_data_smithi027_135490_object35588:head v 127'10667, mlcod=127'10664)", 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.163862+0000", 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.014747209000000001, 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015068938, 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.163862+0000", 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.163862+0000", 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.163862+0000", 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.163869+0000", 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2060000000000001e-06 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.163880+0000", 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0943e-05 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.163886+0000", 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1929999999999998e-06 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.171722+0000", 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078364960000000001 2024-08-29T15:12:43.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.171742+0000", 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0105000000000001e-05 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169341 5.4 e127/111 5:29c93edd:::benchmark_data_smithi027_135490_object35589:head v 127'10715, mlcod=127'10710)", 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.164194+0000", 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.014415641999999999, 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014789170000000001, 2024-08-29T15:12:43.204 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.164194+0000", 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.164194+0000", 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.164193+0000", 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.164199+0000", 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3090000000000001e-06 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.164210+0000", 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0559e-05 2024-08-29T15:12:43.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.164217+0000", 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5579999999999997e-06 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.166505+0000", 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022882100000000002 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.166527+0000", 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2603999999999999e-05 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169343 5.4 e127/111 5:252120cc:::benchmark_data_smithi027_135490_object35591:head v 127'10716, mlcod=127'10710)", 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.170918+0000", 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0076912489999999998, 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081165149999999995, 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.208 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170918+0000", 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170918+0000", 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170917+0000", 2024-08-29T15:12:43.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170925+0000", 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0150000000000003e-06 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170958+0000", 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2682000000000003e-05 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.210 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170964+0000", 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3400000000000003e-06 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176039+0000", 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050751709999999998 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176059+0000", 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.942e-05 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.211 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:169349 5.d e127/111 5:b74449da:::benchmark_data_smithi027_135490_object35597:head v 127'10503, mlcod=127'10501)", 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.175743+0000", 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0028666339999999998, 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033450889999999999, 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175743+0000", 2024-08-29T15:12:43.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175743+0000", 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175742+0000", 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175750+0000", 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5750000000000001e-06 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175763+0000", 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2809999999999999e-05 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175771+0000", 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6680000000000001e-06 2024-08-29T15:12:43.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175828+0000", 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6938999999999998e-05 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.175846+0000", 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7839999999999999e-05 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:169322 5.f e127/111)", 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.176608+0000", 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0020010269999999998, 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025272480000000002, 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176608+0000", 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176608+0000", 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176608+0000", 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176613+0000", 2024-08-29T15:12:43.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3639999999999998e-06 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176625+0000", 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.146e-05 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.176631+0000", 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1909999999999997e-06 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:169322 5.f 5:f0205e73:::benchmark_data_smithi027_135490_object35570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:43.138448+0000", 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.040161082000000001, 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040733457000000001, 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 169322 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:43.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.138448+0000", 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.138448+0000", 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.138448+0000", 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:43.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.138452+0000", 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7160000000000002e-06 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.138497+0000", 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4672999999999997e-05 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.138506+0000", 2024-08-29T15:12:43.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2050000000000008e-06 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170448+0000", 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031941352999999999 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.170725+0000", 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00027768600000000002 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:43.171103+0000", 2024-08-29T15:12:43.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00037751800000000003 2024-08-29T15:12:43.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.223 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:43.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:43.223 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:12:43.223 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 11 2024-08-29T15:12:43.223 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:12:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:43.224 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:43.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:43.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:43.329 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:12:43.330 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:12:43.361 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:12:43.362 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:12:43.376 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:43.394 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:43.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:43.404 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:12:43.404 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:12:43.441 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:12:43.441 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:12:43.454 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:43.483 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:12:43.483 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:12:43.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:43.521 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:12:43.522 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:12:43.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:43.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:43.549 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:12:43.549 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:12:43.577 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:12:43.577 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:12:43.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:43.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:43.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:43.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:43.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:43.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:43.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:43.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:43.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:43.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:44.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:44.100 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:44.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:44.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:44.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:44.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:44.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:43 smithi171 ceph-mon[90031]: pgmap v377: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-08-29T15:12:44.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1469444517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:44.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:43 smithi027 ceph-mon[98970]: pgmap v377: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-08-29T15:12:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1469444517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:43 smithi027 ceph-mon[93921]: pgmap v377: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-08-29T15:12:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1469444517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:44.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:44.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:44.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:44.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:44.483 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:44.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:44.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:44.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:44.647 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:44.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:44.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:44.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:44.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:44.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:44.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:44.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:44.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:45.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:45.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:45.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:45.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:45.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:45.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:45.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:45.329 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:12:45.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:45.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:45.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:45.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:45.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:45.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:45.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:45.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:45.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:45.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:45.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:45.796 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:45.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:45.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:45.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:45.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:46.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:46.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:46.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:46.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:46.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:46.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:46.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:46.237 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:45 smithi027 ceph-mon[93921]: pgmap v378: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-29T15:12:46.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:45 smithi027 ceph-mon[98970]: pgmap v378: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-29T15:12:46.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:45 smithi171 ceph-mon[90031]: pgmap v378: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-29T15:12:46.270 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:12:46.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:46.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:46.389 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:46.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:46.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:46.526 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:46.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:46.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:46.627 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:46.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:46.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:46.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:46.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:46.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:46.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:46.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:46.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:46.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:47.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:47.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:47.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:47.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:47.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:47.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:47.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:47.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:47.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:47.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:47.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:47.500 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:47.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:47.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:47.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:47.637 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:47.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:47.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:47.739 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:47.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:47.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:47.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:47.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:47.926 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:47.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:48.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:48.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:48.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:48.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:48.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:48.214 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:48.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:48.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:47 smithi171 ceph-mon[90031]: pgmap v379: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2024-08-29T15:12:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:47 smithi027 ceph-mon[98970]: pgmap v379: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2024-08-29T15:12:48.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:47 smithi027 ceph-mon[93921]: pgmap v379: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2024-08-29T15:12:48.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:12:48.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:48.349 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:48.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:48.451 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:48.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:48.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:48.614 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:48.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:48.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:48.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:48.799 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:48.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:48.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:48.901 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:48.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:48.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:49.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:49.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:49.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:49.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:49.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:49.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:12:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:12:49] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-29T15:12:49.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:49.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:49.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:49.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:49.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:49.495 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:49.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:49.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:49.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:49.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:49.663 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:12:49.663 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:12:49.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:49.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:49.776 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:49.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:49.877 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:49.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:49.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:49.966 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:50.019 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:12:50.022 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.3 to 1 (10000) 2024-08-29T15:12:50.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:50.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:50.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:50.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:50.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:50.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:50.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:50.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:49 smithi171 ceph-mon[90031]: pgmap v380: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2024-08-29T15:12:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3087254528' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:49 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4053804198' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:12:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:49 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:12:50.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:49 smithi027 ceph-mon[93921]: pgmap v380: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2024-08-29T15:12:50.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3087254528' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:50.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:49 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:50.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4053804198' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:12:50.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:49 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:12:50.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: pgmap v380: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2024-08-29T15:12:50.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3087254528' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:50.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:12:50.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4053804198' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:12:50.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:12:50.329 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:50.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:50.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:50.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:50.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:50.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:50.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:50.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:50.621 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:50.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:50.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:50.722 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:50.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:50.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:50.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:50.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:50.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:50.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:50.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:50.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:51.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 4 1 2024-08-29T15:12:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:51.047 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:51.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:51.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:12:51.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:51.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:51.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:51.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:12:51.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:12:51.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:12:51.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[93921]: osdmap e128: 8 total, 8 up, 8 in 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:50 smithi027 ceph-mon[98970]: osdmap e128: 8 total, 8 up, 8 in 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:50 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:12:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:50 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:12:51.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:50 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:12:51.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:50 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:12:51.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:50 smithi171 ceph-mon[90031]: osdmap e128: 8 total, 8 up, 8 in 2024-08-29T15:12:51.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:51.349 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:51.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:51.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:51.451 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:51.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:51.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:51.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:51.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:51.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:51.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:51.761 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:51.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:51.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:51.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:51.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:51.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:51.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:52.033 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:52.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:52.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:52.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:52.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:52.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:52.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:52.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:52.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:52 smithi027 ceph-mon[98970]: pgmap v382: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-08-29T15:12:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:52 smithi027 ceph-mon[93921]: pgmap v382: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-08-29T15:12:52.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:52.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:52.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:52.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:52 smithi171 ceph-mon[90031]: pgmap v382: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-08-29T15:12:52.544 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:52.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:52.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:52.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:52.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:52.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:52.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:52.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:52.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:52.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:53.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:53.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:53.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:53.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:12:53.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:53.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:53.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:12:53.249 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:53.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:53.316 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:53.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:53.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:53.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:53.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:53.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:53.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:53.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:53.577 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:12:53.578 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:12:53.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:53.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:53.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:53.626 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:12:53.626 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:12:53.627 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:12:53.659 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:12:53.660 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:12:53.688 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:12:53.688 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:12:53.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:53.725 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:12:53.725 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:12:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:53.761 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:53.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:53.762 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:12:53.762 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:12:53.796 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:12:53.796 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:12:53.837 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:12:53.838 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:12:53.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:53.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:53.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:53.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:54.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:54.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:54.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:54.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:54.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:54.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:54.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:54.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:54.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:54.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:54 smithi027 ceph-mon[98970]: pgmap v383: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-29T15:12:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:54 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2283153415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:54 smithi027 ceph-mon[93921]: pgmap v383: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-29T15:12:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:54 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2283153415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:54.353 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:54 smithi171 ceph-mon[90031]: pgmap v383: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-29T15:12:54.353 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:54 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2283153415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:12:54.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:54.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:54.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:54.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:54.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:54.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:54.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:54.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:54.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:54.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:54.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:54.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:54.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:54.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:54.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:12:54.994 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:55.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:55.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:55.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:55.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:55.171 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:55.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:55.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:55.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:55.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:55.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:55.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:55.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:55.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:55.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:55.614 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:55.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:55.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:55.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:55.745 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:55.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:55.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:55.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:55.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:55.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:55.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:55.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:56.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:56.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:56.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:56.120 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:56.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:56.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:56.214 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:56.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:56.287 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:56 smithi027 ceph-mon[93921]: pgmap v384: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-29T15:12:56.287 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:56 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:56.287 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:56 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:56.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:56 smithi027 ceph-mon[98970]: pgmap v384: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-29T15:12:56.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:56 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:56.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:56 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:56.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:56.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:56.307 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:56 smithi171 ceph-mon[90031]: pgmap v384: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-29T15:12:56.307 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:56 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:56.307 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:56 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:56.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:56.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:56.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:56.471 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:12:56.471 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:24059 5.3 e115/111 5:c5d79acb:::benchmark_data_smithi027_135490_object24058:head v 115'1506, mlcod=115'1501)", 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:41.414951+0000", 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 135.05340293399999, 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082584268000000002, 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414951+0000", 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414951+0000", 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414950+0000", 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414981+0000", 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0258999999999999e-05 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414999+0000", 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8227999999999999e-05 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415005+0000", 2024-08-29T15:12:56.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.071e-06 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415784+0000", 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00077869500000000004 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415809+0000", 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5570000000000001e-05 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.497499+0000", 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.081689308000000002 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.497535+0000", 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6752999999999999e-05 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.476 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:24066 5.f 5:f6bd3485:::benchmark_data_smithi027_135490_object24065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:41.415037+0000", 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 135.05331667499999, 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082574817999999994, 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 24066 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415037+0000", 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415037+0000", 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415036+0000", 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415071+0000", 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4523999999999997e-05 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415085+0000", 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4803e-05 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415093+0000", 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3350000000000001e-06 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415769+0000", 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00067616099999999999 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416021+0000", 2024-08-29T15:12:56.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00025209599999999999 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416309+0000", 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00028821400000000002 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496079+0000", 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.079769481000000003 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496495+0000", 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00041647800000000002 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.497534+0000", 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001038413 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.497573+0000", 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9542000000000002e-05 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.497612+0000", 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9026999999999998e-05 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:24071 5.f 5:fe74ce3f:::benchmark_data_smithi027_135490_object24070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:41.415846+0000", 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 135.05250750299999, 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.084225789999999995, 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.483 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 24071 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415846+0000", 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415846+0000", 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415846+0000", 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415879+0000", 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3247000000000002e-05 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415887+0000", 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7129999999999994e-06 2024-08-29T15:12:56.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415896+0000", 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5739999999999996e-06 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.494830+0000", 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078934262000000005 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.494968+0000", 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013754500000000001 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.495124+0000", 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015630799999999999 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.497625+0000", 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025010140000000002 2024-08-29T15:12:56.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.499343+0000", 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017173430000000001 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.499920+0000", 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00057796900000000001 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.499968+0000", 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7262000000000003e-05 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.500072+0000", 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010448800000000001 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:24072 5.7 5:e94e0127:::benchmark_data_smithi027_135490_object24071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:41.416148+0000", 2024-08-29T15:12:56.490 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 135.052205853, 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.084164327999999997, 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 24072 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416148+0000", 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416148+0000", 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416147+0000", 2024-08-29T15:12:56.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.493 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416200+0000", 2024-08-29T15:12:56.494 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2432000000000002e-05 2024-08-29T15:12:56.494 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.494 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.494 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.494 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416210+0000", 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0587999999999999e-05 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416373+0000", 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000163208 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496102+0000", 2024-08-29T15:12:56.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.079728946999999994 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496172+0000", 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9250000000000003e-05 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496246+0000", 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4067999999999996e-05 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.496 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.497630+0000", 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001384487 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.498853+0000", 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012230450000000001 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.500233+0000", 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001380161 2024-08-29T15:12:56.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.500281+0000", 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7951000000000002e-05 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.500312+0000", 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1134999999999999e-05 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:24073 5.f 5:f603afd6:::benchmark_data_smithi027_135490_object24072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:41.416384+0000", 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 135.05197013899999, 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083811970999999999, 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 24073 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416384+0000", 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416384+0000", 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416383+0000", 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416547+0000", 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016364799999999999 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416557+0000", 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.024e-05 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416564+0000", 2024-08-29T15:12:56.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7209999999999997e-06 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496519+0000", 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.079955732000000002 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496634+0000", 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011453100000000001 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:12:56.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.496771+0000", 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013663799999999999 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.498881+0000", 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021108379999999999 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.499409+0000", 2024-08-29T15:12:56.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00052800000000000004 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.500109+0000", 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00069946799999999996 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.500149+0000", 2024-08-29T15:12:56.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9727000000000001e-05 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.500196+0000", 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7139999999999999e-05 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68909 5.7 5:eeb8103f:::benchmark_data_smithi027_135490_object68908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.911879+0000", 2024-08-29T15:12:56.505 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 111.556474534, 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14841467699999999, 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68909 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.911879+0000", 2024-08-29T15:12:56.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.911879+0000", 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.911878+0000", 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:56.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.911908+0000", 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0292000000000001e-05 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.911919+0000", 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0757000000000001e-05 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.911926+0000", 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7959999999999998e-06 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912041+0000", 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000115355 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912162+0000", 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000121233 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.510 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:56.510 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912284+0000", 2024-08-29T15:12:56.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012142300000000001 2024-08-29T15:12:56.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.917655+0000", 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0053715389999999998 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.917824+0000", 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016892099999999999 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.057905+0000", 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14008028 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.057999+0000", 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4812e-05 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.060294+0000", 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022947919999999999 2024-08-29T15:12:56.512 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.513 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.513 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.513 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.513 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68913 5.7 5:e04f4a13:::benchmark_data_smithi027_135490_object68912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.913946+0000", 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 111.554407879, 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14653488200000001, 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68913 2024-08-29T15:12:56.514 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913946+0000", 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913946+0000", 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.515 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913945+0000", 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.914017+0000", 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2261000000000005e-05 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.914031+0000", 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3838e-05 2024-08-29T15:12:56.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.914039+0000", 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8499999999999994e-06 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.914922+0000", 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00088315699999999997 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.915027+0000", 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000104805 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.915143+0000", 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000116824 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.917829+0000", 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026851620000000001 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.918156+0000", 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00032712300000000002 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.060342+0000", 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14218592799999999 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.060439+0000", 2024-08-29T15:12:56.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7615000000000006e-05 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.060481+0000", 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1661e-05 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68924 5.7 5:e4de7c1d:::benchmark_data_smithi027_135490_object68923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.924297+0000", 2024-08-29T15:12:56.520 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 111.54405658500001, 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.136376998, 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68924 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.521 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924297+0000", 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924297+0000", 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924296+0000", 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.522 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924414+0000", 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011803600000000001 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924426+0000", 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1969e-05 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924433+0000", 2024-08-29T15:12:56.523 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6549999999999998e-06 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924476+0000", 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3263999999999998e-05 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924602+0000", 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012615400000000001 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.524 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.924750+0000", 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014737600000000001 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.926827+0000", 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002077658 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.926898+0000", 2024-08-29T15:12:56.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0130999999999997e-05 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.060562+0000", 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13366481599999999 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.060636+0000", 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3949000000000006e-05 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.060674+0000", 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8059999999999998e-05 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71205 5.f 5:f376f519:::benchmark_data_smithi027_135490_object71204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.839169+0000", 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.62918513, 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.092835184000000001, 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71205 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.528 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839169+0000", 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839169+0000", 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839167+0000", 2024-08-29T15:12:56.529 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839200+0000", 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3102999999999998e-05 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839212+0000", 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1956e-05 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.530 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839220+0000", 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.221e-06 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839346+0000", 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012594900000000001 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839472+0000", 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000126735 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839614+0000", 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014209699999999999 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928971+0000", 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.089356202999999995 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928978+0000", 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3390000000000004e-06 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931928+0000", 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029498829999999999 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931967+0000", 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9681000000000002e-05 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.932004+0000", 2024-08-29T15:12:56.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6539999999999999e-05 2024-08-29T15:12:56.534 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.534 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.534 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.534 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.534 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71204 5.e e117/111 5:72845dde:::benchmark_data_smithi027_135490_object71203:head v 117'4521, mlcod=117'4519)", 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.839491+0000", 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.628863076, 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095320315000000003, 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839491+0000", 2024-08-29T15:12:56.536 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839491+0000", 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839490+0000", 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.537 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839515+0000", 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4700000000000001e-05 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839523+0000", 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2279999999999998e-06 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839528+0000", 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8400000000000002e-06 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839567+0000", 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8568000000000001e-05 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839586+0000", 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8810000000000001e-05 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934616+0000", 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095030451000000002 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934811+0000", 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019522699999999999 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.540 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71211 5.7 5:e5b26fc3:::benchmark_data_smithi027_135490_object71210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.840080+0000", 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.628273607, 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.091804782000000001, 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71211 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.541 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840080+0000", 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840080+0000", 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840079+0000", 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840240+0000", 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016109300000000001 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840252+0000", 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1586000000000001e-05 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840259+0000", 2024-08-29T15:12:56.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5690000000000001e-06 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842304+0000", 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020454459999999998 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842421+0000", 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011646000000000001 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:12:56.544 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842545+0000", 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000124616 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928886+0000", 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.08634095 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.545 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928983+0000", 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6537000000000001e-05 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931810+0000", 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028268640000000001 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931850+0000", 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0930999999999998e-05 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931885+0000", 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4690999999999997e-05 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71209 5.4 e117/111 5:275284da:::benchmark_data_smithi027_135490_object71208:head v 117'4506, mlcod=117'4504)", 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.841295+0000", 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.62705861000001, 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.087536235000000004, 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.547 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841295+0000", 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841295+0000", 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.548 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841294+0000", 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841317+0000", 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2705999999999999e-05 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841328+0000", 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0961000000000001e-05 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841334+0000", 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1489999999999999e-06 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841401+0000", 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6908000000000004e-05 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841419+0000", 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8323999999999999e-05 2024-08-29T15:12:56.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846883+0000", 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005463435 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928832+0000", 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.081948682999999994 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.551 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71206 5.e e117/111 5:74b85a3d:::benchmark_data_smithi027_135490_object71205:head v 117'4522, mlcod=117'4519)", 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.841826+0000", 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.626528154, 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093015181000000002, 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841826+0000", 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.552 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841826+0000", 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841825+0000", 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841850+0000", 2024-08-29T15:12:56.553 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4491999999999999e-05 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841859+0000", 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5959999999999993e-06 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841865+0000", 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7769999999999999e-06 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842186+0000", 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00032089999999999999 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842202+0000", 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6160000000000001e-05 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934815+0000", 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.092613443000000004 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934841+0000", 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5599999999999999e-05 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71212 5.3 e117/111 5:c460b82e:::benchmark_data_smithi027_135490_object71211:head v 117'4416, mlcod=117'4413)", 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.841885+0000", 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.626468512, 2024-08-29T15:12:56.556 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.089167832000000002, 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841885+0000", 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841885+0000", 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.557 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841885+0000", 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841912+0000", 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6633999999999999e-05 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841918+0000", 2024-08-29T15:12:56.558 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4960000000000001e-06 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841923+0000", 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4939999999999997e-06 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841985+0000", 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2444e-05 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842003+0000", 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7387000000000001e-05 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.930933+0000", 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088930082999999993 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.560 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931053+0000", 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000120642 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71216 5.f 5:f83f6a17:::benchmark_data_smithi027_135490_object71215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.842712+0000", 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.625642289, 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.091437989999999997, 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.561 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71216 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842712+0000", 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.562 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842712+0000", 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842710+0000", 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842754+0000", 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3577e-05 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842766+0000", 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2149e-05 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842781+0000", 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5009e-05 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844475+0000", 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016937390000000001 2024-08-29T15:12:56.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844597+0000", 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012238200000000001 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844740+0000", 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014299700000000001 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928987+0000", 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.084246891000000004 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931763+0000", 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027756909999999998 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.933982+0000", 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022192560000000002 2024-08-29T15:12:56.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934110+0000", 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012781700000000001 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934150+0000", 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9879999999999998e-05 2024-08-29T15:12:56.567 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71217 5.f 5:fdf09791:::benchmark_data_smithi027_135490_object71216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.843569+0000", 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.62478466100001, 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100780693, 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:12:56.568 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71217 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843569+0000", 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843569+0000", 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843569+0000", 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843597+0000", 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8935999999999999e-05 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843608+0000", 2024-08-29T15:12:56.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0129000000000001e-05 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843614+0000", 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2759999999999997e-06 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.929010+0000", 2024-08-29T15:12:56.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085396142999999994 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.929136+0000", 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012580600000000001 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.929290+0000", 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000153969 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:12:56.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.932662+0000", 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033720320000000001 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934205+0000", 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015433059999999999 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.944205+0000", 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010000171 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.944312+0000", 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106513 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.944350+0000", 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8179000000000002e-05 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71218 5.6 e117/111 5:6d390296:::benchmark_data_smithi027_135490_object71217:head v 117'4472, mlcod=117'4470)", 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.844183+0000", 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.62417062, 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090718766000000006, 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844183+0000", 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844183+0000", 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844182+0000", 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844227+0000", 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4821000000000003e-05 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844238+0000", 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0424999999999999e-05 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844244+0000", 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4069999999999998e-06 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844298+0000", 2024-08-29T15:12:56.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4194999999999999e-05 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844317+0000", 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.914e-05 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934843+0000", 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090525987000000002 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934902+0000", 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8831e-05 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:71211 5.7 e117/111 ondisk, result = 0)", 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.845206+0000", 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.62314779899999, 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083683250000000001, 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.579 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845206+0000", 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845206+0000", 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845205+0000", 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845220+0000", 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5002e-05 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.581 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845230+0000", 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7180000000000008e-06 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845245+0000", 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4894e-05 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928870+0000", 2024-08-29T15:12:56.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083624646999999996 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928879+0000", 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1109999999999997e-06 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928889+0000", 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0557000000000001e-05 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.583 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71219 5.d e117/111 5:be85b71d:::benchmark_data_smithi027_135490_object71218:head v 117'4505, mlcod=117'4503)", 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.845272+0000", 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.62308185800001, 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085848805, 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845272+0000", 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845272+0000", 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845272+0000", 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845309+0000", 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7160999999999999e-05 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845316+0000", 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5109999999999997e-06 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845321+0000", 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7790000000000003e-06 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845370+0000", 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8980000000000002e-05 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.845387+0000", 2024-08-29T15:12:56.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6940999999999999e-05 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931016+0000", 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085628409000000003 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931121+0000", 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010539600000000001 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.588 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71207 5.e e117/111 5:712e06d6:::benchmark_data_smithi027_135490_object71206:head v 117'4523, mlcod=117'4519)", 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.846022+0000", 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 109.622331962, 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093346332000000004, 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:12:56.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846022+0000", 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846022+0000", 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846021+0000", 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846046+0000", 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4960000000000002e-05 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928767+0000", 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082720882999999995 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928779+0000", 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1605e-05 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931199+0000", 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0024203950000000001 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931255+0000", 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6369000000000002e-05 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:12:56.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.939333+0000", 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0080775540000000007 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.939368+0000", 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5373000000000002e-05 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:12:56.593 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:12:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:56.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:56.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:56.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:56.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:56.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:56.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:56.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:56.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:56.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:56.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:57.009 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:57.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:57.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:57.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:57.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:57.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:57.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:12:57.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:57.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:57.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:57.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:57.476 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:57.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:57.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:57.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:57.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:57.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:57.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:57.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:57.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:57.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:12:57.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:57.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:57.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:57.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:58.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:58.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:58.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:58.046 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.4 to 1 (10000) 2024-08-29T15:12:58.131 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:12:58.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:12:58.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:58.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:12:58.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:58 smithi027 ceph-mon[98970]: pgmap v385: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-29T15:12:58.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4242603643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:12:58.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:58 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:12:58.213 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:58 smithi027 ceph-mon[93921]: pgmap v385: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-29T15:12:58.213 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4242603643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:12:58.213 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:58 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:12:58.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:12:58.320 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:58 smithi171 ceph-mon[90031]: pgmap v385: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-29T15:12:58.320 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4242603643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:12:58.320 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:58 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:12:58.331 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:58.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:58.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:12:58.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:58.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:58.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:58.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:58.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:58.599 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:58.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:58.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:12:58.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:58.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:58.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:12:58.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:58.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:58.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:58.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:12:59.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:59.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:12:59.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 5 1 2024-08-29T15:12:59.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:59.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:59.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:59.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:59.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:12:59.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:59.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:12:59.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:12:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:12:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[98970]: osdmap e129: 8 total, 8 up, 8 in 2024-08-29T15:12:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:12:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[93921]: osdmap e129: 8 total, 8 up, 8 in 2024-08-29T15:12:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:12:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:12:59] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-29T15:12:59.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:59.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:59.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:59.456 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:59 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:12:59.457 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:59 smithi171 ceph-mon[90031]: osdmap e129: 8 total, 8 up, 8 in 2024-08-29T15:12:59.457 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:12:59.457 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:12:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:12:59.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:59.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:59.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:59.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:59.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:12:59.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:59.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:59.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:12:59.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:12:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:12:59.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:12:59.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:12:59.883 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:12:59.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:00.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:00.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:00.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:00.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:00.218 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:13:00.218 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:183571 5.3 e129/111 5:ce225ad1:::benchmark_data_smithi027_135490_object49819:head v 129'11365, mlcod=129'11363)", 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:00.213392+0000", 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0037145260000000001, 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003744505, 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:00.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213392+0000", 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213392+0000", 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213391+0000", 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213401+0000", 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0417e-05 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213422+0000", 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0862000000000001e-05 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213430+0000", 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0450000000000001e-06 2024-08-29T15:13:00.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213495+0000", 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5344000000000003e-05 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.213515+0000", 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9439000000000001e-05 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:183562 5.9 e129/111 5:9dd615d5:::benchmark_data_smithi027_135490_object49810:head v 129'11412, mlcod=129'11410)", 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:00.214817+0000", 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0022892640000000001, 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0023807250000000002, 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214817+0000", 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:00.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214817+0000", 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214816+0000", 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214824+0000", 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6629999999999993e-06 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214840+0000", 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6232e-05 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214847+0000", 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0829999999999998e-06 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214893+0000", 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5899999999999998e-05 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:00.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:00.214919+0000", 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6174999999999999e-05 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 2 2024-08-29T15:13:00.226 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:13:00.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:00.283 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:00 smithi171 ceph-mon[90031]: pgmap v387: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-08-29T15:13:00.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:00 smithi027 ceph-mon[93921]: pgmap v387: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-08-29T15:13:00.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:00 smithi027 ceph-mon[98970]: pgmap v387: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2024-08-29T15:13:00.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:00.416 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:00.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:00.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:00.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:00.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:00.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:00.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:00.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:00.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:00.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:00.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:01.012 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:13:01.066 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:13:01.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:01.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:01.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:01.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:01.264 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:01.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2495446323' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:01 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2495446323' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:01 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2495446323' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:01 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:01.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:01.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:01.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:01.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:01.441 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:13:01.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:01.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:01.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:01.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:01.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:01.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:01.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:01.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:01.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:01.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:01.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:01.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:01.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:01.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:02.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:02.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:02.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:02.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:02.228 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:02.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:02 smithi171 ceph-mon[90031]: pgmap v388: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-08-29T15:13:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:02 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:02 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:13:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:02 smithi171 ceph-mon[90031]: osdmap e130: 8 total, 8 up, 8 in 2024-08-29T15:13:02.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:02.359 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:02.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:02.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:02.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:02.496 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:02.505 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:02.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:02.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[98970]: pgmap v388: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-08-29T15:13:02.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:02.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:13:02.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[98970]: osdmap e130: 8 total, 8 up, 8 in 2024-08-29T15:13:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[93921]: pgmap v388: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2024-08-29T15:13:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:13:02.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:02 smithi027 ceph-mon[93921]: osdmap e130: 8 total, 8 up, 8 in 2024-08-29T15:13:02.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:02.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:02.647 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:02.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:02.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:02.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:02.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:02.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:02.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:02.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:03.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:03.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:03.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:03.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:03.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:03.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:03.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:03.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:03.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:03.447 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:03.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:03.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:03.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:03.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:03.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:03.687 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:13:03.687 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:13:03.687 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:13:03.687 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:13:03.687 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:13:03.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:03.742 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:03.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:03 smithi171 ceph-mon[90031]: pgmap v390: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-29T15:13:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:13:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:13:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[93921]: pgmap v390: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-29T15:13:03.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:13:03.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:13:03.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.791 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[98970]: pgmap v390: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-29T15:13:03.791 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.791 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.791 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:13:03.791 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:13:03.791 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:03.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:13:03.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:03.896 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:03.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:03.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:04.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:04.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:04.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:04.130 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:04.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:04.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:04.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:04.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:04.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:04.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:04.378 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:13:04.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:04.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:04.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:04.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:04.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:04.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:04.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:04.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:04.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:04.722 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:13:04.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:04.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:04.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:04.766 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:13:04.766 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:13:04.766 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:13:04.766 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:13:04.777 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:13:04.778 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:13:04.809 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:13:04.809 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:13:04.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:04.846 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:13:04.847 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:13:04.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:04.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:04.882 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:13:04.882 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:13:04.912 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:13:04.912 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:13:04.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:13:04.944 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:13:04.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:05.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:05.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:05.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:05.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:05.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:05.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:05.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:05.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:05.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:05.413 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:05.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:05.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:05.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:05.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:05.633 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:05 smithi027 ceph-mon[98970]: pgmap v391: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-29T15:13:05.633 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2864166106' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:05.633 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:05 smithi027 ceph-mon[93921]: pgmap v391: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-29T15:13:05.633 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2864166106' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:05.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:05.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:05.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:05.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:05 smithi171 ceph-mon[90031]: pgmap v391: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2024-08-29T15:13:05.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2864166106' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:05.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:05.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:05.885 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:05.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:05.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:05.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:06.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:06.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:06.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:06.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:06.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:06.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:06.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:06.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:06.399 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:06.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:06.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:06.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:06.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:06.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:06.660 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:13:06.660 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:13:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:06.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:06.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:06.784 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:06.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:06.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:06.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:06.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:06.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:07.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:07.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:07.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:07.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:07.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:07.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:07.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:07.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:07.371 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:07.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:07.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:13:07.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:07.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:07.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:07.570 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.5 to 1 (10000) 2024-08-29T15:13:07.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:07.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:07.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:07.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:13:07.704 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:13:07.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:07.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:07 smithi171 ceph-mon[90031]: pgmap v392: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-08-29T15:13:07.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/550810068' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:13:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:07.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:07.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:07.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:07 smithi027 ceph-mon[98970]: pgmap v392: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-08-29T15:13:07.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/550810068' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:13:07.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:07 smithi027 ceph-mon[93921]: pgmap v392: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-08-29T15:13:07.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/550810068' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:13:07.893 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:07.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:07.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:08.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:08.072 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:08.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:08.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:08.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:08.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:08.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:08.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:08.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:08.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 6 1 2024-08-29T15:13:08.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:08.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:08.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:08.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:08.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:08.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:08.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:08.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:08.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:08.766 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:08.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:08.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:08.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:08.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/550810068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:13:08.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:08 smithi027 ceph-mon[98970]: osdmap e131: 8 total, 8 up, 8 in 2024-08-29T15:13:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/550810068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:13:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:08 smithi027 ceph-mon[93921]: osdmap e131: 8 total, 8 up, 8 in 2024-08-29T15:13:08.893 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:08.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:08.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:09.010 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/550810068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:13:09.011 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:08 smithi171 ceph-mon[90031]: osdmap e131: 8 total, 8 up, 8 in 2024-08-29T15:13:09.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:09.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:09.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:09.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:09.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:09.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:09.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:09.301 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:13:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:13:09] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-29T15:13:09.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:09.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:09.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:09.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:09.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:09.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:09.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:09.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:09.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:09.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:09.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:09.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:09.722 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:09 smithi171 ceph-mon[90031]: pgmap v394: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-08-29T15:13:09.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:09.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:09.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:09.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:09 smithi027 ceph-mon[93921]: pgmap v394: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-08-29T15:13:09.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:09 smithi027 ceph-mon[98970]: pgmap v394: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-08-29T15:13:09.867 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:09.882 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:13:09.882 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:13:09.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:09.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:09.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:10.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:10.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:10.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:10.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:10.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:10.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:10.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:10.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:10.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:10.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:10.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:10.450 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:10.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:10.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:10.563 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:10.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:10.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:10.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:13:10.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:10.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:10.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:10.838 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:10.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:10.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:10.960 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:13:10.985 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:11.047 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:11.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:11.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:11.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:11.185 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:11.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:11.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:11.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:11.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:11.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:11.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:11.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:11.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:11.603 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:11.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:11.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:11.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:11.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:11.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:11 smithi171 ceph-mon[90031]: pgmap v395: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:13:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:11 smithi027 ceph-mon[98970]: pgmap v395: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:13:11.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:11 smithi027 ceph-mon[93921]: pgmap v395: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:13:11.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:11.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:11.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:11.936 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:11.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:12.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:12.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:12.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:12.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:12.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:12.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:12.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:12.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:12.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:12.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:12.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:12.450 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:12.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:12.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:12.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:12.562 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:13:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:12.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:12.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:12.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:12.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:12.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:12.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1143889903' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:12.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:12 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:12.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1143889903' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:12.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:12 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1143889903' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:12 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:12.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:12.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:12.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:12.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:12.924 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:13:12.925 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:13:12.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:13.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:13.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:13.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:13.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:13.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:13.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:13.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:13.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:13.416 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:13.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:13.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:13.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:13.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:13.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:13.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[93921]: pgmap v396: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-29T15:13:13.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:13.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:13:13.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[93921]: osdmap e132: 8 total, 8 up, 8 in 2024-08-29T15:13:13.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[98970]: pgmap v396: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-29T15:13:13.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:13.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:13:13.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:13 smithi027 ceph-mon[98970]: osdmap e132: 8 total, 8 up, 8 in 2024-08-29T15:13:13.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:13.836 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:13 smithi171 ceph-mon[90031]: pgmap v396: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-29T15:13:13.836 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:13 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:13.836 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:13:13.836 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:13 smithi171 ceph-mon[90031]: osdmap e132: 8 total, 8 up, 8 in 2024-08-29T15:13:13.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:13.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:13.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:13.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:14.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:13:14.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:14.097 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:14.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:14.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:14.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:14.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:14.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:14.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:14.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:14.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:14.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:14.394 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:13:14.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:14.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:14.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:14.605 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:13:14.605 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:13:14.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:14.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:13:14.654 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:13:14.659 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:14.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:14.693 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:13:14.693 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:13:14.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:14.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:14.694 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/94583536' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:14.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:14.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:14.694 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/94583536' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:14.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:13:14.727 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:13:14.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:14.764 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:13:14.764 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:13:14.792 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:13:14.793 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:13:14.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:14.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:14.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:14.822 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:13:14.822 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:13:14.852 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:14.852 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:14.852 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/94583536' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:14.860 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:13:14.861 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:13:14.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:14.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:14.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:14.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:15.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:15.099 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:15.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:15.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:15.238 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:15.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:15.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:15.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:15.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:15.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:15.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:15.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:15.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:15.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:15.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:15.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:15.790 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:15.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:15.892 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:15.893 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:15 smithi171 ceph-mon[90031]: pgmap v398: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-29T15:13:15.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:15.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:15.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:15.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:15 smithi027 ceph-mon[93921]: pgmap v398: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-29T15:13:15.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:15 smithi027 ceph-mon[98970]: pgmap v398: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-29T15:13:16.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:16.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:16.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:16.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:16.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:16.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:16.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:16.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:13:16.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:16.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:16.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:23209 5.4 5:24099454:::benchmark_data_smithi027_135490_object23208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:40.957442+0000", 2024-08-29T15:13:16.396 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 155.43354277399999, 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13668750399999999, 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 23209 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957442+0000", 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957442+0000", 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957441+0000", 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957483+0000", 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1879999999999999e-05 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957498+0000", 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5175e-05 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957506+0000", 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3560000000000004e-06 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957555+0000", 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9413000000000003e-05 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957687+0000", 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000131707 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957804+0000", 2024-08-29T15:13:16.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011721499999999999 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.959256+0000", 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014519629999999999 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.959307+0000", 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1561999999999999e-05 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.094046+0000", 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13473817199999999 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.094089+0000", 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3547999999999997e-05 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.094130+0000", 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0639999999999997e-05 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.402 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:27113 5.4 5:2f19f843:::benchmark_data_smithi027_135490_object27112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:42.720352+0000", 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 153.67063347199999, 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.073854800999999998, 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 27113 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.403 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720352+0000", 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720352+0000", 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720350+0000", 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720431+0000", 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0708e-05 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720450+0000", 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9690999999999998e-05 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720459+0000", 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6789999999999994e-06 2024-08-29T15:13:16.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720493+0000", 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4224999999999998e-05 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720571+0000", 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7613000000000004e-05 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.720649+0000", 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7756000000000006e-05 2024-08-29T15:13:16.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.722235+0000", 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00158624 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.722243+0000", 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9440000000000005e-06 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.794115+0000", 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.071872288000000006 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.794165+0000", 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0433999999999999e-05 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:42.794206+0000", 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0859999999999998e-05 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:69875 5.4 5:2ad4ce38:::benchmark_data_smithi027_135490_object69874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:16.408 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:05.651432+0000", 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 130.739552981, 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.094221108999999997, 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 69875 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651432+0000", 2024-08-29T15:13:16.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651432+0000", 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651431+0000", 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651500+0000", 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9097999999999995e-05 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651516+0000", 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6260999999999999e-05 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651524+0000", 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7789999999999996e-06 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651560+0000", 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6767000000000002e-05 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651646+0000", 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5337e-05 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.651751+0000", 2024-08-29T15:13:16.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010511200000000001 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.653649+0000", 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001898091 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.654632+0000", 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00098260300000000008 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.745548+0000", 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090916697000000005 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.745614+0000", 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5837e-05 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.745653+0000", 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9016e-05 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.414 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71202 5.9 5:9a462713:::benchmark_data_smithi027_135490_object71201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.835893+0000", 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.55509166600001, 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095293849, 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71202 2024-08-29T15:13:16.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.835893+0000", 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.835893+0000", 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.835892+0000", 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836001+0000", 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000108497 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836012+0000", 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1538e-05 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836019+0000", 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7719999999999997e-06 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836069+0000", 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0416e-05 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836180+0000", 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000110587 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836496+0000", 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031648499999999998 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.841779+0000", 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0052822549999999996 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.930927+0000", 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.089148580000000005 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931071+0000", 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014423800000000001 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931147+0000", 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5685000000000005e-05 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931187+0000", 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0086000000000002e-05 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.420 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71209 5.4 5:275284da:::benchmark_data_smithi027_135490_object71208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.840201+0000", 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.55078368, 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095483967000000003, 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71209 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840201+0000", 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840201+0000", 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840200+0000", 2024-08-29T15:13:16.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840231+0000", 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1541999999999997e-05 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840243+0000", 2024-08-29T15:13:16.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2094e-05 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840250+0000", 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5749999999999997e-06 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840301+0000", 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1406999999999998e-05 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840412+0000", 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011022 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.840541+0000", 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012950400000000001 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.843255+0000", 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027136780000000002 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846588+0000", 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033337169999999999 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.935421+0000", 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088832499999999995 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.935514+0000", 2024-08-29T15:13:16.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2743000000000003e-05 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.935685+0000", 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017156499999999999 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71215 5.9 5:96572047:::benchmark_data_smithi027_135490_object71214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.842393+0000", 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.54859163899999, 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.089029504999999995, 2024-08-29T15:13:16.427 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71215 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842393+0000", 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842393+0000", 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842392+0000", 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842465+0000", 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2782999999999998e-05 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842480+0000", 2024-08-29T15:13:16.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5698000000000001e-05 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842487+0000", 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7610000000000002e-06 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842523+0000", 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5960999999999997e-05 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842646+0000", 2024-08-29T15:13:16.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000123275 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.842788+0000", 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014133799999999999 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928798+0000", 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.086010004000000001 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931041+0000", 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022429020000000002 2024-08-29T15:13:16.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931248+0000", 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00020735600000000001 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931387+0000", 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013887 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931423+0000", 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6040000000000001e-05 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71214 5.1 e117/111 5:82269f1e:::benchmark_data_smithi027_135490_object71213:head v 117'4427, mlcod=117'4425)", 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.846528+0000", 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.54445653600001, 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.084842231000000004, 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846528+0000", 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846528+0000", 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846528+0000", 2024-08-29T15:13:16.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846554+0000", 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6454000000000001e-05 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846565+0000", 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0548e-05 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846571+0000", 2024-08-29T15:13:16.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5780000000000004e-06 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928822+0000", 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082251123999999995 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928842+0000", 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0018e-05 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931302+0000", 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0024601139999999998 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.931371+0000", 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8348000000000004e-05 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:71219 5.d e117/111 5:be85b71d:::benchmark_data_smithi027_135490_object71218:head v 117'4505, mlcod=117'4503)", 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.846626+0000", 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.54435907199999, 2024-08-29T15:13:16.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.087818227999999998, 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846626+0000", 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846626+0000", 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846625+0000", 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846660+0000", 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4450999999999999e-05 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846670+0000", 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8819999999999996e-06 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846675+0000", 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4970000000000001e-06 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846740+0000", 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4729999999999999e-05 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846759+0000", 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9184000000000001e-05 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934296+0000", 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.087537101000000006 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.934444+0000", 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014802399999999999 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.441 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:71215 5.9 e117/111 ondisk, result = 0)", 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.846728+0000", 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.544257362, 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082072863999999995, 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846728+0000", 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846728+0000", 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846727+0000", 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846731+0000", 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8550000000000004e-06 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846738+0000", 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.486e-06 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.846745+0000", 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1670000000000004e-06 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928785+0000", 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082039972000000003 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928792+0000", 2024-08-29T15:13:16.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4909999999999999e-06 2024-08-29T15:13:16.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.928801+0000", 2024-08-29T15:13:16.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4940000000000003e-06 2024-08-29T15:13:16.448 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.448 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.448 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.448 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:74748 5.4 5:2b7299df:::benchmark_data_smithi027_135490_object74747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:09.561081+0000", 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 126.82990371299999, 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.070135140999999998, 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 74748 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.449 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561081+0000", 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561081+0000", 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561080+0000", 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561126+0000", 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6239999999999998e-05 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561137+0000", 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1147e-05 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561143+0000", 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5759999999999996e-06 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561175+0000", 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2341000000000001e-05 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561265+0000", 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9858999999999995e-05 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:13:16.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.561347+0000", 2024-08-29T15:13:16.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1982000000000002e-05 2024-08-29T15:13:16.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:16.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.563341+0000", 2024-08-29T15:13:16.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019937280000000002 2024-08-29T15:13:16.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.563577+0000", 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00023591700000000001 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.631125+0000", 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067547998999999997 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.631178+0000", 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2778000000000002e-05 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:09.631216+0000", 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8809000000000001e-05 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82171 5.9 5:952386eb:::benchmark_data_smithi027_135490_object82170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.322146+0000", 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.06883871300001, 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095313933000000003, 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.456 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82171 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322146+0000", 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322146+0000", 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322145+0000", 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322173+0000", 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8504e-05 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322186+0000", 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2119999999999999e-05 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322192+0000", 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8630000000000002e-06 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323091+0000", 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00089827099999999997 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323204+0000", 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000113417 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323328+0000", 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012403300000000001 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326304+0000", 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029754410000000001 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326535+0000", 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00023155199999999999 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417377+0000", 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090841663000000003 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417425+0000", 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8429e-05 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417460+0000", 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4968000000000001e-05 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82176 5.9 5:962339e0:::benchmark_data_smithi027_135490_object82175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.324514+0000", 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.066470748, 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093057419000000002, 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.462 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82176 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.324514+0000", 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.324514+0000", 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.324513+0000", 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.324569+0000", 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6542999999999997e-05 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.324580+0000", 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0241000000000001e-05 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.324586+0000", 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2210000000000002e-06 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326570+0000", 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019840159999999999 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326685+0000", 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000115023 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326809+0000", 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012389399999999999 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.332032+0000", 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0052232279999999999 2024-08-29T15:13:16.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334655+0000", 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026234320000000002 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417504+0000", 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082848062 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417544+0000", 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0030000000000001e-05 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417572+0000", 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8147e-05 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82178 5.9 5:90281a6a:::benchmark_data_smithi027_135490_object82177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.326571+0000", 2024-08-29T15:13:16.468 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.064414124, 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.091097629999999999, 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82178 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326571+0000", 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326571+0000", 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326570+0000", 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326649+0000", 2024-08-29T15:13:16.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9516000000000002e-05 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326659+0000", 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7200000000000001e-06 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326673+0000", 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3725e-05 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329236+0000", 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002563306 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329350+0000", 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011372800000000001 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329479+0000", 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000128713 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334762+0000", 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0052837170000000003 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336394+0000", 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001631665 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417603+0000", 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.081209144999999996 2024-08-29T15:13:16.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417640+0000", 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6925999999999998e-05 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417669+0000", 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8388e-05 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.474 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82180 5.9 5:906f714e:::benchmark_data_smithi027_135490_object82179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.328305+0000", 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.062680341, 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.089512578999999995, 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82180 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.328305+0000", 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.328305+0000", 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.328303+0000", 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.328343+0000", 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9415999999999997e-05 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.328355+0000", 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2588999999999999e-05 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.328362+0000", 2024-08-29T15:13:16.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9739999999999996e-06 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.332053+0000", 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00369053 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.332152+0000", 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8997000000000001e-05 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:13:16.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.332267+0000", 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000115116 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.334869+0000", 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026023420000000001 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.336419+0000", 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001550205 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417748+0000", 2024-08-29T15:13:16.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.081329109999999996 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417787+0000", 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8435e-05 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417817+0000", 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0343000000000001e-05 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82188 5.d e118/111 5:baa32811:::benchmark_data_smithi027_135490_object82187:head v 118'5219, mlcod=118'5217)", 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.335030+0000", 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.05595524899999, 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078466956000000004, 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335030+0000", 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335030+0000", 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335029+0000", 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335059+0000", 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9858000000000001e-05 2024-08-29T15:13:16.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335069+0000", 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0689999999999999e-05 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335076+0000", 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3090000000000001e-06 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335145+0000", 2024-08-29T15:13:16.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9441000000000001e-05 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335163+0000", 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7472000000000001e-05 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413456+0000", 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078293023000000003 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413497+0000", 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1100000000000003e-05 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82192 5.6 e118/111 5:684214d4:::benchmark_data_smithi027_135490_object82191:head v 118'5172, mlcod=118'5170)", 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.337431+0000", 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.053553632, 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.076492450000000003, 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337431+0000", 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337431+0000", 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337431+0000", 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337460+0000", 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8943999999999999e-05 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337471+0000", 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1473e-05 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337477+0000", 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0430000000000004e-06 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337535+0000", 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7528999999999998e-05 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337554+0000", 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9675999999999999e-05 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413822+0000", 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.076267919000000003 2024-08-29T15:13:16.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413924+0000", 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000101576 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82193 5.9 5:9843b4f0:::benchmark_data_smithi027_135490_object82192:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.337954+0000", 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.05303146200001, 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.084218854999999995, 2024-08-29T15:13:16.489 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82193 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337954+0000", 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337954+0000", 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337952+0000", 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337983+0000", 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0902000000000003e-05 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.337994+0000", 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1671000000000001e-05 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338001+0000", 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5270000000000004e-06 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338043+0000", 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2205e-05 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338168+0000", 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000124707 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,6", 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.338305+0000", 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013732699999999999 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.415823+0000", 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.077518225999999996 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416940+0000", 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0011165960000000001 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.421902+0000", 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049623749999999998 2024-08-29T15:13:16.494 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.422032+0000", 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013021800000000001 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.422172+0000", 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013998299999999999 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82196 5.4 5:2f53dbea:::benchmark_data_smithi027_135490_object82195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.339558+0000", 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 121.051426959, 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080735196999999995, 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82196 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.339558+0000", 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.339558+0000", 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.339557+0000", 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:13:16.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.339614+0000", 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7021000000000002e-05 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.412683+0000", 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.073069590000000004 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.412697+0000", 2024-08-29T15:13:16.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3782e-05 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.412838+0000", 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000140836 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.412982+0000", 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014354600000000001 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:13:16.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413114+0000", 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000132559 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416290+0000", 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031759549999999998 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.418138+0000", 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001847986 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.420044+0000", 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019055910000000001 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.420171+0000", 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012715400000000001 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.420293+0000", 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012236899999999999 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.501 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:184638 5.6 e130/111 5:6b6bb42c:::benchmark_data_smithi027_135490_object50886:head v 130'11590, mlcod=130'11588)", 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:01.655662+0000", 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 14.735323196, 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.074632452000000002, 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.655662+0000", 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.655662+0000", 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.655661+0000", 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.655667+0000", 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8989999999999998e-06 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.655676+0000", 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8060000000000001e-06 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.655685+0000", 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9849999999999995e-06 2024-08-29T15:13:16.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.661246+0000", 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055612659999999996 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.661302+0000", 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5831999999999997e-05 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.730272+0000", 2024-08-29T15:13:16.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.068969849 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.730294+0000", 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2498e-05 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:184641 5.6 e130/111 5:6b2eecc8:::benchmark_data_smithi027_135490_object50889:head v 130'11591, mlcod=130'11588)", 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:01.660768+0000", 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 14.730216702, 2024-08-29T15:13:16.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.069538902, 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.660768+0000", 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.660768+0000", 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.660768+0000", 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.660773+0000", 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7450000000000003e-06 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.660782+0000", 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1609999999999992e-06 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.660786+0000", 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9199999999999997e-06 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.673773+0000", 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012987434000000001 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.673791+0000", 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8107000000000001e-05 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.730297+0000", 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.056505165000000003 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:01.730307+0000", 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0669000000000001e-05 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:13:16.510 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:13:16.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:16.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:16.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:16.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:16.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:16.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:16.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:16.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:16.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:16.877 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:16.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:16.936 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:16.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:17.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:17.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:17.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:17.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:17.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:17.225 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:17.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:17.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:17.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:17.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:17.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:17.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:17.607 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:17.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:13:17.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:17.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:17.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:17.700 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.6 to 1 (10000) 2024-08-29T15:13:17.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:17.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:17.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:17.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:17.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:17.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:17.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:17.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:13:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:17 smithi171 ceph-mon[90031]: pgmap v399: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 909 op/s 2024-08-29T15:13:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1974088481' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:13:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:17 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:13:18.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:18.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:17 smithi027 ceph-mon[93921]: pgmap v399: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 909 op/s 2024-08-29T15:13:18.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1974088481' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:13:18.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:17 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:13:18.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:17 smithi027 ceph-mon[98970]: pgmap v399: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 909 op/s 2024-08-29T15:13:18.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1974088481' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:13:18.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:17 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:13:18.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:18.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:18.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:18.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:18.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:18.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:18.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:18.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:18.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:18.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:18.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:18.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:18.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:18.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:18.601 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:18.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:18.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:18.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 7 1 2024-08-29T15:13:18.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:18.768 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:18.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:18.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:18.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:18.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:18.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:18.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:19.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:19.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:18 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:13:19.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:18 smithi171 ceph-mon[90031]: osdmap e133: 8 total, 8 up, 8 in 2024-08-29T15:13:19.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:18 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:13:19.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:18 smithi027 ceph-mon[93921]: osdmap e133: 8 total, 8 up, 8 in 2024-08-29T15:13:19.018 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:18 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:13:19.018 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:18 smithi027 ceph-mon[98970]: osdmap e133: 8 total, 8 up, 8 in 2024-08-29T15:13:19.018 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:13:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:13:19] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.51.0" 2024-08-29T15:13:19.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:19.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:19.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:19.132 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:19.159 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:19.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:19.195 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:19.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:19.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:19.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:19.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:19.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:19.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:19.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:19.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:19.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:19.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:19.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:19.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:19.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:19.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:19.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:19.829 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:19 smithi027 ceph-mon[93921]: pgmap v401: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2024-08-29T15:13:19.829 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:19 smithi027 ceph-mon[98970]: pgmap v401: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2024-08-29T15:13:19.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:19.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:19.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:19.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:19 smithi171 ceph-mon[90031]: pgmap v401: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2024-08-29T15:13:20.023 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:20.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:20.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:20.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:20.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:20.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:20.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:20.355 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:20.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:20.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:20.500 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:20.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:20.602 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:20.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:20.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:20.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:20.778 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:20.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:20.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:20.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:20.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:20.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:20.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:21.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:21.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:21.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:21.150 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:21.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:21.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:21.277 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:21.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:21.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:21.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:21.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:21.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:21.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:21.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:21.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:21.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:21.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:21.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:21.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:21.752 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:13:21.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:21.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:21.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:21.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:21.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:21.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:21.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:21.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:21 smithi171 ceph-mon[90031]: pgmap v402: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-29T15:13:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2535671342' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:13:22.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:21 smithi027 ceph-mon[93921]: pgmap v402: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-29T15:13:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2535671342' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:13:22.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:21 smithi027 ceph-mon[98970]: pgmap v402: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-29T15:13:22.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2535671342' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:13:22.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:22.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:22.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:22.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:22.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:22.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:22.424 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:22.526 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:22.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.559 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:22.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:22.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:22.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:22.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:22.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:22.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:22.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:22.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:22.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:22.990 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:13:22.990 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:13:22.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:22 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:22.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2535671342' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:13:22.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:22 smithi027 ceph-mon[93921]: osdmap e134: 8 total, 8 up, 8 in 2024-08-29T15:13:22.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:22 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:22.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2535671342' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:13:22.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:22 smithi027 ceph-mon[98970]: osdmap e134: 8 total, 8 up, 8 in 2024-08-29T15:13:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:22 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2535671342' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:13:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:22 smithi171 ceph-mon[90031]: osdmap e134: 8 total, 8 up, 8 in 2024-08-29T15:13:23.022 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:13:23.022 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:13:23.058 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:13:23.059 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:13:23.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:23.090 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:13:23.090 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:13:23.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:23.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:23.124 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:13:23.124 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:13:23.162 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:13:23.162 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:13:23.195 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:13:23.195 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:13:23.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:23.229 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:13:23.229 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:13:23.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:23.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:23.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:23.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:23.386 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:23.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:23.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:23.488 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:23.522 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:23.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:23.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:23.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:23.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:23.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:23.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:23.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:23.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:23.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:23.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:23 smithi171 ceph-mon[90031]: pgmap v404: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-29T15:13:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1775740550' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:24.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:23 smithi027 ceph-mon[93921]: pgmap v404: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-29T15:13:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1775740550' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:23 smithi027 ceph-mon[98970]: pgmap v404: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-08-29T15:13:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1775740550' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:24.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:24.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:24.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:24.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:24.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:24.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:24.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:24.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:24.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:24.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:24.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:24.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:24.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:24.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:24.620 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:24.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:24.721 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:24.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:24.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:24.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:24.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:24.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:24.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:25.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:25.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:25.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:25.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:25.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:25.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:25.322 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:25.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:25.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:25.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:25.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:25.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:25.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:25.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:25.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:25.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:25.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.863 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:25.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:25.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:25.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:25.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:25 smithi171 ceph-mon[90031]: pgmap v405: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-08-29T15:13:26.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:25 smithi027 ceph-mon[98970]: pgmap v405: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-08-29T15:13:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:25 smithi027 ceph-mon[93921]: pgmap v405: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-08-29T15:13:26.101 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:26.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:26.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:26.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:13:26.225 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:26.260 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:26.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:26.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:26.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:26.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:26.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:26.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:26.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:26.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:26.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:26.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:26.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:26.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:26.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:26.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:26.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:26.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:26.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:26.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:26.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:27.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:27.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:27.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:27.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:27.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:27.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:27.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:27.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:27.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:27.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:27.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:27.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:27.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:27.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:27.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:27.630 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:27.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:27.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:13:27.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:27.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:27.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:27.688 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:13:27.689 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:13:27.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:27.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:27.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:27.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:27.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:27.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:27.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:27.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:27 smithi171 ceph-mon[90031]: pgmap v406: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-08-29T15:13:28.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:27 smithi027 ceph-mon[93921]: pgmap v406: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-08-29T15:13:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:27 smithi027 ceph-mon[98970]: pgmap v406: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-08-29T15:13:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:28.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:28.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:28.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:28.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:28.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:28.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:28.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:28.349 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:28.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:28.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:28.488 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:28.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:28.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:28.590 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:28.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:28.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:28.727 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:28.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:28.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:28.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:28.782 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.7 to 1 (10000) 2024-08-29T15:13:28.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:28.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:28.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:29.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/37452191' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:13:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:28 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:13:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:29.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/37452191' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:13:29.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:28 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:13:29.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:29.035 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:29.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:29.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:13:29.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:29.160 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:29.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:29.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:29.262 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/37452191' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:13:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:28 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:13:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:29.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:29.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:29.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:13:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:13:29] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-29T15:13:29.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:29.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:29.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:29.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:29.575 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:29.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:29.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:29.679 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-29T15:13:29.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:13:29.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:29.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:29 smithi027 ceph-mon[98970]: pgmap v407: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-08-29T15:13:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:29 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:13:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:29 smithi027 ceph-mon[98970]: osdmap e135: 8 total, 8 up, 8 in 2024-08-29T15:13:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:29 smithi027 ceph-mon[93921]: pgmap v407: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-08-29T15:13:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:29 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:13:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:29 smithi027 ceph-mon[93921]: osdmap e135: 8 total, 8 up, 8 in 2024-08-29T15:13:29.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:29.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:29.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:29.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:30.032 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:30.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:30.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:30.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:30.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:30.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:29 smithi171 ceph-mon[90031]: pgmap v407: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-08-29T15:13:30.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:29 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:13:30.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:29 smithi171 ceph-mon[90031]: osdmap e135: 8 total, 8 up, 8 in 2024-08-29T15:13:30.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:30.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:30.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:30.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:30.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:30.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:30.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:30.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:30.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:30.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:30.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:30.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:30.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:30.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:30.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:30.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:30.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:30.966 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:30.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:30.977 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:30.990 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:31.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:31.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:31.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:31.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:31.272 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:31.374 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:31.414 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:31.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:31.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:31.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:31.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:31.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.682 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:31.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:31.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:31.803 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:13:31.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:31.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:31.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:31.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:31.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:31.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:32.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:32.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:31 smithi027 ceph-mon[93921]: pgmap v409: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-08-29T15:13:32.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3064875849' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:32.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:31 smithi027 ceph-mon[98970]: pgmap v409: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-08-29T15:13:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3064875849' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:32.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:32.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:32.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:32.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:32.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:32.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:32.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:31 smithi171 ceph-mon[90031]: pgmap v409: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-08-29T15:13:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3064875849' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:13:32.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:32.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:32.377 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:32.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:32.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:32.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:32.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:32.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:32.619 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:32.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:32.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:32.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:32.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:32.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:32.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:32.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:32.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:32.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:33.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:13:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:13:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:13:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[93921]: osdmap e136: 8 total, 8 up, 8 in 2024-08-29T15:13:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:13:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:13:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:13:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:32 smithi027 ceph-mon[98970]: osdmap e136: 8 total, 8 up, 8 in 2024-08-29T15:13:33.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:33.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:33.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:33.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:33.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:33.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:32 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:13:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:32 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:13:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:32 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:13:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:32 smithi171 ceph-mon[90031]: osdmap e136: 8 total, 8 up, 8 in 2024-08-29T15:13:33.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:33.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:33.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:33.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:33.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:33.477 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:33.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:33.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:33.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:33.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:33.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:33.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:33.751 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:33.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:33.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:33.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:33.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:33.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:34.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:34.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:33 smithi027 ceph-mon[93921]: pgmap v411: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-08-29T15:13:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:33 smithi027 ceph-mon[98970]: pgmap v411: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-08-29T15:13:34.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:34.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:34.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:33 smithi171 ceph-mon[90031]: pgmap v411: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-08-29T15:13:34.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:34.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:34.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:34.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:34.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:34.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:34.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:34.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:34.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:34.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:34.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:34.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:34.724 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:34.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:34.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:34.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:34.863 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:34.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:34.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:34.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:35.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:35.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:35.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:35.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:35.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:35.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:35.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:35.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:35.551 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:35.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:35.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:35.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:35.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:35.822 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:35.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:35.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:35.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:35.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:35 smithi027 ceph-mon[98970]: pgmap v412: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-29T15:13:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:35 smithi027 ceph-mon[93921]: pgmap v412: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-29T15:13:36.076 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:36.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:36.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:36.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:36.250 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:36.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:35 smithi171 ceph-mon[90031]: pgmap v412: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-29T15:13:36.352 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:36.388 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:36.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:36.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:36.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:36.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:36.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:36.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:36.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:36.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:36.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:36.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:36.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:36.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:36.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:36.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:13:36.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:36.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:36.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:36.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:37.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:37.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:37.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:37.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:37.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:37.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:37.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:37.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:37.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:37.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:37.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:37.476 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:37.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:37.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:37.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:37.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:37.652 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:37.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:37.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:37.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:37.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:37.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:37.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:37.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:37.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:37.991 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:13:37.991 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:13:38.049 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:13:38.050 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:13:38.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:38.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:13:38.093 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:13:38.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:37 smithi027 ceph-mon[98970]: pgmap v413: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-29T15:13:38.094 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:37 smithi027 ceph-mon[93921]: pgmap v413: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-29T15:13:38.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:38.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:38.138 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:13:38.139 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:13:38.155 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:38.185 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:13:38.185 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:13:38.196 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:13:38.196 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:13:38.196 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:13:38.196 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:13:38.196 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:13:38.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:38.224 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:13:38.224 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:13:38.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:38.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:38.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:38.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:13:38.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:13:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:37 smithi171 ceph-mon[90031]: pgmap v413: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-29T15:13:38.294 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:13:38.294 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:13:38.347 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:38.376 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:38.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:38.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:38.506 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:38.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:38.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:38.608 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:38.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:38.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:38.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:38.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:38.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:38.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:38.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:38.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:39.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:39.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:39.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:39.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:39.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:39.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:39.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:39.227 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1661549835' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:39.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1661549835' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:39.227 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:13:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:13:39] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-29T15:13:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1661549835' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:39.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:39.357 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:39.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:39.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:39.488 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:39.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:39.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:39.628 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:39.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:39.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:39.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:39.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:39.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:39.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:39.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:39.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:13:40.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:40.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:40.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:40.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:40.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:40.186 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:39 smithi027 ceph-mon[93921]: pgmap v414: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-29T15:13:40.187 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:39 smithi027 ceph-mon[98970]: pgmap v414: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-29T15:13:40.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:40.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:40.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:39 smithi171 ceph-mon[90031]: pgmap v414: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-08-29T15:13:40.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:40.359 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:40.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:40.462 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:40.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:40.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:40.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:40.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:40.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:40.702 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:13:40.704 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:13:40.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:40.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:40.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:40.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:40.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:40.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:41.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:41.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:41.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:41.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:41.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:41.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:41.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:13:41.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:41.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:41.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:41.366 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:41.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:41.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:41.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:41.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:41.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:41.612 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:41.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:41.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:41.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:41.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:41.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:41.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:41.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:41.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:41.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:41.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:42.023 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:42.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:42.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:42.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:42.149 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:41 smithi171 ceph-mon[90031]: pgmap v415: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-29T15:13:42.149 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:41 smithi171 ceph-mon[90031]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:13:42.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:41 smithi027 ceph-mon[93921]: pgmap v415: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-29T15:13:42.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:41 smithi027 ceph-mon[93921]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:13:42.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:41 smithi027 ceph-mon[98970]: pgmap v415: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-29T15:13:42.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:41 smithi027 ceph-mon[98970]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:13:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:42.160 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:42.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:42.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:42.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:42.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:42.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:42.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:42.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:42.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:42.513 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":415,"stamp":"2024-08-29T15:13:40.403016+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3346601981,"num_objects":51021,"num_object_clones":0,"num_object_copies":153063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51021,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":217630,"num_write_kb":8574978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147889,"ondisk_log_size":147889,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10786084,"kb_used_data":9861860,"kb_used_omap":34,"kb_used_meta":923997,"kb_avail":739175132,"statfs":{"total":767960285184,"available":756915335168,"internally_reserved":0,"allocated":10098544640,"data_stored":10092945199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34969,"internal_metadata":946173799},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-561512448,"num_objects":-8568,"num_object_clones":0,"num_object_copies":-25704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8568,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004113"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":142,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822298+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:13:31.822298+0000","last_peered":"2024-08-29T15:13:31.822298+0000","last_clean":"2024-08-29T15:13:31.822298+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:13:31.822298+0000","last_undegraded":"2024-08-29T15:13:31.822298+0000","last_fullsized":"2024-08-29T15:13:31.822298+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.793045+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:13:28.793045+0000","last_peered":"2024-08-29T15:13:28.793045+0000","last_clean":"2024-08-29T15:13:28.793045+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:13:28.793045+0000","last_undegraded":"2024-08-29T15:13:28.793045+0000","last_fullsized":"2024-08-29T15:13:28.793045+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.784721+0000","last_change":"2024-08-29T15:08:58.887518+0000","last_active":"2024-08-29T15:13:28.784721+0000","last_peered":"2024-08-29T15:13:28.784721+0000","last_clean":"2024-08-29T15:13:28.784721+0000","last_became_active":"2024-08-29T15:08:58.887049+0000","last_became_peered":"2024-08-29T15:08:58.887049+0000","last_unstale":"2024-08-29T15:13:28.784721+0000","last_undegraded":"2024-08-29T15:13:28.784721+0000","last_fullsized":"2024-08-29T15:13:28.784721+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:23:07.638407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.804505+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:13:31.804505+0000","last_peered":"2024-08-29T15:13:31.804505+0000","last_clean":"2024-08-29T15:13:31.804505+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:13:31.804505+0000","last_undegraded":"2024-08-29T15:13:31.804505+0000","last_fullsized":"2024-08-29T15:13:31.804505+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.805067+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:13:31.805067+0000","last_peered":"2024-08-29T15:13:31.805067+0000","last_clean":"2024-08-29T15:13:31.805067+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:13:31.805067+0000","last_undegraded":"2024-08-29T15:13:31.805067+0000","last_fullsized":"2024-08-29T15:13:31.805067+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":84,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.795030+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T15:13:28.795030+0000","last_peered":"2024-08-29T15:13:28.795030+0000","last_clean":"2024-08-29T15:13:28.795030+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T15:13:28.795030+0000","last_undegraded":"2024-08-29T15:13:28.795030+0000","last_fullsized":"2024-08-29T15:13:28.795030+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":128,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822829+0000","last_change":"2024-08-29T15:08:58.884675+0000","last_active":"2024-08-29T15:13:31.822829+0000","last_peered":"2024-08-29T15:13:31.822829+0000","last_clean":"2024-08-29T15:13:31.822829+0000","last_became_active":"2024-08-29T15:08:58.884007+0000","last_became_peered":"2024-08-29T15:08:58.884007+0000","last_unstale":"2024-08-29T15:13:31.822829+0000","last_undegraded":"2024-08-29T15:13:31.822829+0000","last_fullsized":"2024-08-29T15:13:31.822829+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:12:03.629717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":108,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.789201+0000","last_change":"2024-08-29T15:07:20.812313+0000","last_active":"2024-08-29T15:13:28.789201+0000","last_peered":"2024-08-29T15:13:28.789201+0000","last_clean":"2024-08-29T15:13:28.789201+0000","last_became_active":"2024-08-29T15:07:20.811973+0000","last_became_peered":"2024-08-29T15:07:20.811973+0000","last_unstale":"2024-08-29T15:13:28.789201+0000","last_undegraded":"2024-08-29T15:13:28.789201+0000","last_fullsized":"2024-08-29T15:13:28.789201+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:37.283470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"136'13417","reported_seq":13451,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.473421+0000","last_change":"2024-08-29T15:10:22.057568+0000","last_active":"2024-08-29T15:13:39.473421+0000","last_peered":"2024-08-29T15:13:39.473421+0000","last_clean":"2024-08-29T15:13:39.473421+0000","last_became_active":"2024-08-29T15:10:22.056935+0000","last_became_peered":"2024-08-29T15:10:22.056935+0000","last_unstale":"2024-08-29T15:13:39.473421+0000","last_undegraded":"2024-08-29T15:13:39.473421+0000","last_fullsized":"2024-08-29T15:13:39.473421+0000","mapping_epoch":111,"log_start":"117'4800","ondisk_log_start":"117'4800","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8617,"log_dups_size":0,"ondisk_log_size":8617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:33:23.576258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211353600,"num_objects":3225,"num_object_clones":0,"num_object_copies":9675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13417,"num_write_kb":532544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816983+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:13:31.816983+0000","last_peered":"2024-08-29T15:13:31.816983+0000","last_clean":"2024-08-29T15:13:31.816983+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:13:31.816983+0000","last_undegraded":"2024-08-29T15:13:31.816983+0000","last_fullsized":"2024-08-29T15:13:31.816983+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":84,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.784107+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T15:13:28.784107+0000","last_peered":"2024-08-29T15:13:28.784107+0000","last_clean":"2024-08-29T15:13:28.784107+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T15:13:28.784107+0000","last_undegraded":"2024-08-29T15:13:28.784107+0000","last_fullsized":"2024-08-29T15:13:28.784107+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"136'13326","reported_seq":13388,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:38.602730+0000","last_change":"2024-08-29T15:10:22.061175+0000","last_active":"2024-08-29T15:13:38.602730+0000","last_peered":"2024-08-29T15:13:38.602730+0000","last_clean":"2024-08-29T15:13:38.602730+0000","last_became_active":"2024-08-29T15:10:22.060889+0000","last_became_peered":"2024-08-29T15:10:22.060889+0000","last_unstale":"2024-08-29T15:13:38.602730+0000","last_undegraded":"2024-08-29T15:13:38.602730+0000","last_fullsized":"2024-08-29T15:13:38.602730+0000","mapping_epoch":111,"log_start":"117'3300","ondisk_log_start":"117'3300","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:59:27.186639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206045184,"num_objects":3144,"num_object_clones":0,"num_object_copies":9432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13326,"num_write_kb":527040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.817176+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:13:31.817176+0000","last_peered":"2024-08-29T15:13:31.817176+0000","last_clean":"2024-08-29T15:13:31.817176+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:13:31.817176+0000","last_undegraded":"2024-08-29T15:13:31.817176+0000","last_fullsized":"2024-08-29T15:13:31.817176+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":113,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.789251+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:13:28.789251+0000","last_peered":"2024-08-29T15:13:28.789251+0000","last_clean":"2024-08-29T15:13:28.789251+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:13:28.789251+0000","last_undegraded":"2024-08-29T15:13:28.789251+0000","last_fullsized":"2024-08-29T15:13:28.789251+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"136'13582","reported_seq":13642,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:38.590460+0000","last_change":"2024-08-29T15:10:22.060048+0000","last_active":"2024-08-29T15:13:38.590460+0000","last_peered":"2024-08-29T15:13:38.590460+0000","last_clean":"2024-08-29T15:13:38.590460+0000","last_became_active":"2024-08-29T15:10:22.059752+0000","last_became_peered":"2024-08-29T15:10:22.059752+0000","last_unstale":"2024-08-29T15:13:38.590460+0000","last_undegraded":"2024-08-29T15:13:38.590460+0000","last_fullsized":"2024-08-29T15:13:38.590460+0000","mapping_epoch":111,"log_start":"117'3500","ondisk_log_start":"117'3500","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10082,"log_dups_size":0,"ondisk_log_size":10082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:17:39.363676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210239488,"num_objects":3208,"num_object_clones":0,"num_object_copies":9624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13582,"num_write_kb":537280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816216+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T15:13:31.816216+0000","last_peered":"2024-08-29T15:13:31.816216+0000","last_clean":"2024-08-29T15:13:31.816216+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T15:13:31.816216+0000","last_undegraded":"2024-08-29T15:13:31.816216+0000","last_fullsized":"2024-08-29T15:13:31.816216+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":100,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.789877+0000","last_change":"2024-08-29T15:07:52.136104+0000","last_active":"2024-08-29T15:13:28.789877+0000","last_peered":"2024-08-29T15:13:28.789877+0000","last_clean":"2024-08-29T15:13:28.789877+0000","last_became_active":"2024-08-29T15:07:52.135781+0000","last_became_peered":"2024-08-29T15:07:52.135781+0000","last_unstale":"2024-08-29T15:13:28.789877+0000","last_undegraded":"2024-08-29T15:13:28.789877+0000","last_fullsized":"2024-08-29T15:13:28.789877+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:26.381499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"136'13722","reported_seq":13784,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:38.637353+0000","last_change":"2024-08-29T15:10:22.059900+0000","last_active":"2024-08-29T15:13:38.637353+0000","last_peered":"2024-08-29T15:13:38.637353+0000","last_clean":"2024-08-29T15:13:38.637353+0000","last_became_active":"2024-08-29T15:10:22.059601+0000","last_became_peered":"2024-08-29T15:10:22.059601+0000","last_unstale":"2024-08-29T15:13:38.637353+0000","last_undegraded":"2024-08-29T15:13:38.637353+0000","last_fullsized":"2024-08-29T15:13:38.637353+0000","mapping_epoch":111,"log_start":"117'3700","ondisk_log_start":"117'3700","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:19:28.744391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216399872,"num_objects":3302,"num_object_clones":0,"num_object_copies":9906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13722,"num_write_kb":544768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.815322+0000","last_change":"2024-08-29T15:08:58.887669+0000","last_active":"2024-08-29T15:13:31.815322+0000","last_peered":"2024-08-29T15:13:31.815322+0000","last_clean":"2024-08-29T15:13:31.815322+0000","last_became_active":"2024-08-29T15:08:58.887343+0000","last_became_peered":"2024-08-29T15:08:58.887343+0000","last_unstale":"2024-08-29T15:13:31.815322+0000","last_undegraded":"2024-08-29T15:13:31.815322+0000","last_fullsized":"2024-08-29T15:13:31.815322+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:32.280756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.804923+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:13:31.804923+0000","last_peered":"2024-08-29T15:13:31.804923+0000","last_clean":"2024-08-29T15:13:31.804923+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:13:31.804923+0000","last_undegraded":"2024-08-29T15:13:31.804923+0000","last_fullsized":"2024-08-29T15:13:31.804923+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"136'13559","reported_seq":13622,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:38.615512+0000","last_change":"2024-08-29T15:10:22.059154+0000","last_active":"2024-08-29T15:13:38.615512+0000","last_peered":"2024-08-29T15:13:38.615512+0000","last_clean":"2024-08-29T15:13:38.615512+0000","last_became_active":"2024-08-29T15:10:22.058770+0000","last_became_peered":"2024-08-29T15:10:22.058770+0000","last_unstale":"2024-08-29T15:13:38.615512+0000","last_undegraded":"2024-08-29T15:13:38.615512+0000","last_fullsized":"2024-08-29T15:13:38.615512+0000","mapping_epoch":111,"log_start":"117'3500","ondisk_log_start":"117'3500","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10059,"log_dups_size":0,"ondisk_log_size":10059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:03.961012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202113048,"num_objects":3085,"num_object_clones":0,"num_object_copies":9255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3085,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13559,"num_write_kb":532545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":127,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822192+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:13:31.822192+0000","last_peered":"2024-08-29T15:13:31.822192+0000","last_clean":"2024-08-29T15:13:31.822192+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:13:31.822192+0000","last_undegraded":"2024-08-29T15:13:31.822192+0000","last_fullsized":"2024-08-29T15:13:31.822192+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.817364+0000","last_change":"2024-08-29T15:08:58.886369+0000","last_active":"2024-08-29T15:13:31.817364+0000","last_peered":"2024-08-29T15:13:31.817364+0000","last_clean":"2024-08-29T15:13:31.817364+0000","last_became_active":"2024-08-29T15:08:58.886022+0000","last_became_peered":"2024-08-29T15:08:58.886022+0000","last_unstale":"2024-08-29T15:13:31.817364+0000","last_undegraded":"2024-08-29T15:13:31.817364+0000","last_fullsized":"2024-08-29T15:13:31.817364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:36.451849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"136'13612","reported_seq":13673,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:40.361194+0000","last_change":"2024-08-29T15:10:22.060607+0000","last_active":"2024-08-29T15:13:40.361194+0000","last_peered":"2024-08-29T15:13:40.361194+0000","last_clean":"2024-08-29T15:13:40.361194+0000","last_became_active":"2024-08-29T15:10:22.060299+0000","last_became_peered":"2024-08-29T15:10:22.060299+0000","last_unstale":"2024-08-29T15:13:40.361194+0000","last_undegraded":"2024-08-29T15:13:40.361194+0000","last_fullsized":"2024-08-29T15:13:40.361194+0000","mapping_epoch":111,"log_start":"118'5500","ondisk_log_start":"118'5500","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8112,"log_dups_size":0,"ondisk_log_size":8112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:54:02.847420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205258752,"num_objects":3132,"num_object_clones":0,"num_object_copies":9396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13612,"num_write_kb":535808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.789995+0000","last_change":"2024-08-29T15:07:52.134960+0000","last_active":"2024-08-29T15:13:28.789995+0000","last_peered":"2024-08-29T15:13:28.789995+0000","last_clean":"2024-08-29T15:13:28.789995+0000","last_became_active":"2024-08-29T15:07:52.134608+0000","last_became_peered":"2024-08-29T15:07:52.134608+0000","last_unstale":"2024-08-29T15:13:28.789995+0000","last_undegraded":"2024-08-29T15:13:28.789995+0000","last_fullsized":"2024-08-29T15:13:28.789995+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:42.154504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.817981+0000","last_change":"2024-08-29T15:08:25.191304+0000","last_active":"2024-08-29T15:13:31.817981+0000","last_peered":"2024-08-29T15:13:31.817981+0000","last_clean":"2024-08-29T15:13:31.817981+0000","last_became_active":"2024-08-29T15:08:25.191022+0000","last_became_peered":"2024-08-29T15:08:25.191022+0000","last_unstale":"2024-08-29T15:13:31.817981+0000","last_undegraded":"2024-08-29T15:13:31.817981+0000","last_fullsized":"2024-08-29T15:13:31.817981+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:07.608759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"136'13716","reported_seq":13750,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.678838+0000","last_change":"2024-08-29T15:10:22.060350+0000","last_active":"2024-08-29T15:13:39.678838+0000","last_peered":"2024-08-29T15:13:39.678838+0000","last_clean":"2024-08-29T15:13:39.678838+0000","last_became_active":"2024-08-29T15:10:22.060063+0000","last_became_peered":"2024-08-29T15:10:22.060063+0000","last_unstale":"2024-08-29T15:13:39.678838+0000","last_undegraded":"2024-08-29T15:13:39.678838+0000","last_fullsized":"2024-08-29T15:13:39.678838+0000","mapping_epoch":111,"log_start":"117'3700","ondisk_log_start":"117'3700","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10016,"log_dups_size":0,"ondisk_log_size":10016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:20:03.508888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208797696,"num_objects":3186,"num_object_clones":0,"num_object_copies":9558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13716,"num_write_kb":540864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.789330+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:13:28.789330+0000","last_peered":"2024-08-29T15:13:28.789330+0000","last_clean":"2024-08-29T15:13:28.789330+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:13:28.789330+0000","last_undegraded":"2024-08-29T15:13:28.789330+0000","last_fullsized":"2024-08-29T15:13:28.789330+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.817363+0000","last_change":"2024-08-29T15:07:52.142297+0000","last_active":"2024-08-29T15:13:31.817363+0000","last_peered":"2024-08-29T15:13:31.817363+0000","last_clean":"2024-08-29T15:13:31.817363+0000","last_became_active":"2024-08-29T15:07:52.141836+0000","last_became_peered":"2024-08-29T15:07:52.141836+0000","last_unstale":"2024-08-29T15:13:31.817363+0000","last_undegraded":"2024-08-29T15:13:31.817363+0000","last_fullsized":"2024-08-29T15:13:31.817363+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:58:01.221672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"136'13706","reported_seq":13740,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.511644+0000","last_change":"2024-08-29T15:10:22.063109+0000","last_active":"2024-08-29T15:13:39.511644+0000","last_peered":"2024-08-29T15:13:39.511644+0000","last_clean":"2024-08-29T15:13:39.511644+0000","last_became_active":"2024-08-29T15:10:22.062773+0000","last_became_peered":"2024-08-29T15:10:22.062773+0000","last_unstale":"2024-08-29T15:13:39.511644+0000","last_undegraded":"2024-08-29T15:13:39.511644+0000","last_fullsized":"2024-08-29T15:13:39.511644+0000","mapping_epoch":111,"log_start":"119'6300","ondisk_log_start":"119'6300","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":7406,"log_dups_size":0,"ondisk_log_size":7406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:01:43.837643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207749120,"num_objects":3170,"num_object_clones":0,"num_object_copies":9510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13706,"num_write_kb":540032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":130,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.823378+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:13:31.823378+0000","last_peered":"2024-08-29T15:13:31.823378+0000","last_clean":"2024-08-29T15:13:31.823378+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:13:31.823378+0000","last_undegraded":"2024-08-29T15:13:31.823378+0000","last_fullsized":"2024-08-29T15:13:31.823378+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":108,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.803193+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:13:31.803193+0000","last_peered":"2024-08-29T15:13:31.803193+0000","last_clean":"2024-08-29T15:13:31.803193+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:13:31.803193+0000","last_undegraded":"2024-08-29T15:13:31.803193+0000","last_fullsized":"2024-08-29T15:13:31.803193+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"136'13685","reported_seq":13719,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.481560+0000","last_change":"2024-08-29T15:10:22.062609+0000","last_active":"2024-08-29T15:13:39.481560+0000","last_peered":"2024-08-29T15:13:39.481560+0000","last_clean":"2024-08-29T15:13:39.481560+0000","last_became_active":"2024-08-29T15:10:22.062277+0000","last_became_peered":"2024-08-29T15:10:22.062277+0000","last_unstale":"2024-08-29T15:13:39.481560+0000","last_undegraded":"2024-08-29T15:13:39.481560+0000","last_fullsized":"2024-08-29T15:13:39.481560+0000","mapping_epoch":111,"log_start":"118'5100","ondisk_log_start":"118'5100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8585,"log_dups_size":0,"ondisk_log_size":8585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:48:06.231404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208076800,"num_objects":3175,"num_object_clones":0,"num_object_copies":9525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13685,"num_write_kb":539520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":430,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.792860+0000","last_change":"2024-08-29T15:07:20.817595+0000","last_active":"2024-08-29T15:13:28.792860+0000","last_peered":"2024-08-29T15:13:28.792860+0000","last_clean":"2024-08-29T15:13:28.792860+0000","last_became_active":"2024-08-29T15:07:20.817130+0000","last_became_peered":"2024-08-29T15:07:20.817130+0000","last_unstale":"2024-08-29T15:13:28.792860+0000","last_undegraded":"2024-08-29T15:13:28.792860+0000","last_fullsized":"2024-08-29T15:13:28.792860+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:28:33.151864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":622,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.783073+0000","last_change":"2024-08-29T15:08:25.185728+0000","last_active":"2024-08-29T15:13:28.783073+0000","last_peered":"2024-08-29T15:13:28.783073+0000","last_clean":"2024-08-29T15:13:28.783073+0000","last_became_active":"2024-08-29T15:08:25.185219+0000","last_became_peered":"2024-08-29T15:08:25.185219+0000","last_unstale":"2024-08-29T15:13:28.783073+0000","last_undegraded":"2024-08-29T15:13:28.783073+0000","last_fullsized":"2024-08-29T15:13:28.783073+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:16:25.013137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822218+0000","last_change":"2024-08-29T15:08:58.890060+0000","last_active":"2024-08-29T15:13:31.822218+0000","last_peered":"2024-08-29T15:13:31.822218+0000","last_clean":"2024-08-29T15:13:31.822218+0000","last_became_active":"2024-08-29T15:08:58.889714+0000","last_became_peered":"2024-08-29T15:08:58.889714+0000","last_unstale":"2024-08-29T15:13:31.822218+0000","last_undegraded":"2024-08-29T15:13:31.822218+0000","last_fullsized":"2024-08-29T15:13:31.822218+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:50.659662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":85,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.790479+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T15:13:28.790479+0000","last_peered":"2024-08-29T15:13:28.790479+0000","last_clean":"2024-08-29T15:13:28.790479+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T15:13:28.790479+0000","last_undegraded":"2024-08-29T15:13:28.790479+0000","last_fullsized":"2024-08-29T15:13:28.790479+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"136'13692","reported_seq":13726,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.662360+0000","last_change":"2024-08-29T15:10:22.063647+0000","last_active":"2024-08-29T15:13:39.662360+0000","last_peered":"2024-08-29T15:13:39.662360+0000","last_clean":"2024-08-29T15:13:39.662360+0000","last_became_active":"2024-08-29T15:10:22.063401+0000","last_became_peered":"2024-08-29T15:10:22.063401+0000","last_unstale":"2024-08-29T15:13:39.662360+0000","last_undegraded":"2024-08-29T15:13:39.662360+0000","last_fullsized":"2024-08-29T15:13:39.662360+0000","mapping_epoch":111,"log_start":"117'3600","ondisk_log_start":"117'3600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:14.052876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208928768,"num_objects":3188,"num_object_clones":0,"num_object_copies":9564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13692,"num_write_kb":540160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":827,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.792855+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:13:28.792855+0000","last_peered":"2024-08-29T15:13:28.792855+0000","last_clean":"2024-08-29T15:13:28.792855+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:13:28.792855+0000","last_undegraded":"2024-08-29T15:13:28.792855+0000","last_fullsized":"2024-08-29T15:13:28.792855+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":119,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.804740+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:13:31.804740+0000","last_peered":"2024-08-29T15:13:31.804740+0000","last_clean":"2024-08-29T15:13:31.804740+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:13:31.804740+0000","last_undegraded":"2024-08-29T15:13:31.804740+0000","last_fullsized":"2024-08-29T15:13:31.804740+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":129,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.805222+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:13:31.805222+0000","last_peered":"2024-08-29T15:13:31.805222+0000","last_clean":"2024-08-29T15:13:31.805222+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:13:31.805222+0000","last_undegraded":"2024-08-29T15:13:31.805222+0000","last_fullsized":"2024-08-29T15:13:31.805222+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"136'13563","reported_seq":13625,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:38.615595+0000","last_change":"2024-08-29T15:10:22.063713+0000","last_active":"2024-08-29T15:13:38.615595+0000","last_peered":"2024-08-29T15:13:38.615595+0000","last_clean":"2024-08-29T15:13:38.615595+0000","last_became_active":"2024-08-29T15:10:22.058812+0000","last_became_peered":"2024-08-29T15:10:22.058812+0000","last_unstale":"2024-08-29T15:13:38.615595+0000","last_undegraded":"2024-08-29T15:13:38.615595+0000","last_fullsized":"2024-08-29T15:13:38.615595+0000","mapping_epoch":111,"log_start":"117'3500","ondisk_log_start":"117'3500","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10063,"log_dups_size":0,"ondisk_log_size":10063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:34:42.633452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209780736,"num_objects":3201,"num_object_clones":0,"num_object_copies":9603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13563,"num_write_kb":536448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":594,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.801073+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:13:31.801073+0000","last_peered":"2024-08-29T15:13:31.801073+0000","last_clean":"2024-08-29T15:13:31.801073+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:13:31.801073+0000","last_undegraded":"2024-08-29T15:13:31.801073+0000","last_fullsized":"2024-08-29T15:13:31.801073+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.801919+0000","last_change":"2024-08-29T15:08:25.188186+0000","last_active":"2024-08-29T15:13:31.801919+0000","last_peered":"2024-08-29T15:13:31.801919+0000","last_clean":"2024-08-29T15:13:31.801919+0000","last_became_active":"2024-08-29T15:08:25.187830+0000","last_became_peered":"2024-08-29T15:08:25.187830+0000","last_unstale":"2024-08-29T15:13:31.801919+0000","last_undegraded":"2024-08-29T15:13:31.801919+0000","last_fullsized":"2024-08-29T15:13:31.801919+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:00:12.780470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.802999+0000","last_change":"2024-08-29T15:07:52.139273+0000","last_active":"2024-08-29T15:13:31.802999+0000","last_peered":"2024-08-29T15:13:31.802999+0000","last_clean":"2024-08-29T15:13:31.802999+0000","last_became_active":"2024-08-29T15:07:52.138052+0000","last_became_peered":"2024-08-29T15:07:52.138052+0000","last_unstale":"2024-08-29T15:13:31.802999+0000","last_undegraded":"2024-08-29T15:13:31.802999+0000","last_fullsized":"2024-08-29T15:13:31.802999+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:01.705167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"136'13533","reported_seq":13593,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:38.659728+0000","last_change":"2024-08-29T15:10:22.064573+0000","last_active":"2024-08-29T15:13:38.659728+0000","last_peered":"2024-08-29T15:13:38.659728+0000","last_clean":"2024-08-29T15:13:38.659728+0000","last_became_active":"2024-08-29T15:10:22.064260+0000","last_became_peered":"2024-08-29T15:10:22.064260+0000","last_unstale":"2024-08-29T15:13:38.659728+0000","last_undegraded":"2024-08-29T15:13:38.659728+0000","last_fullsized":"2024-08-29T15:13:38.659728+0000","mapping_epoch":111,"log_start":"117'4700","ondisk_log_start":"117'4700","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8833,"log_dups_size":0,"ondisk_log_size":8833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:28.889508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214761472,"num_objects":3277,"num_object_clones":0,"num_object_copies":9831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13533,"num_write_kb":537920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":553,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.783619+0000","last_change":"2024-08-29T15:07:52.138323+0000","last_active":"2024-08-29T15:13:28.783619+0000","last_peered":"2024-08-29T15:13:28.783619+0000","last_clean":"2024-08-29T15:13:28.783619+0000","last_became_active":"2024-08-29T15:07:52.137781+0000","last_became_peered":"2024-08-29T15:07:52.137781+0000","last_unstale":"2024-08-29T15:13:28.783619+0000","last_undegraded":"2024-08-29T15:13:28.783619+0000","last_fullsized":"2024-08-29T15:13:28.783619+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:11:27.075260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":579,"num_read_kb":2582,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.786466+0000","last_change":"2024-08-29T15:07:20.816485+0000","last_active":"2024-08-29T15:13:28.786466+0000","last_peered":"2024-08-29T15:13:28.786466+0000","last_clean":"2024-08-29T15:13:28.786466+0000","last_became_active":"2024-08-29T15:07:20.816190+0000","last_became_peered":"2024-08-29T15:07:20.816190+0000","last_unstale":"2024-08-29T15:13:28.786466+0000","last_undegraded":"2024-08-29T15:13:28.786466+0000","last_fullsized":"2024-08-29T15:13:28.786466+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:06.731598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":119,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.801911+0000","last_change":"2024-08-29T15:08:58.883318+0000","last_active":"2024-08-29T15:13:31.801911+0000","last_peered":"2024-08-29T15:13:31.801911+0000","last_clean":"2024-08-29T15:13:31.801911+0000","last_became_active":"2024-08-29T15:08:58.882978+0000","last_became_peered":"2024-08-29T15:08:58.882978+0000","last_unstale":"2024-08-29T15:13:31.801911+0000","last_undegraded":"2024-08-29T15:13:31.801911+0000","last_fullsized":"2024-08-29T15:13:31.801911+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:08.675766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"136'13455","reported_seq":13516,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:38.579620+0000","last_change":"2024-08-29T15:10:22.060106+0000","last_active":"2024-08-29T15:13:38.579620+0000","last_peered":"2024-08-29T15:13:38.579620+0000","last_clean":"2024-08-29T15:13:38.579620+0000","last_became_active":"2024-08-29T15:10:22.059746+0000","last_became_peered":"2024-08-29T15:10:22.059746+0000","last_unstale":"2024-08-29T15:13:38.579620+0000","last_undegraded":"2024-08-29T15:13:38.579620+0000","last_fullsized":"2024-08-29T15:13:38.579620+0000","mapping_epoch":111,"log_start":"117'4600","ondisk_log_start":"117'4600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8855,"log_dups_size":0,"ondisk_log_size":8855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:25:15.955899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207552512,"num_objects":3167,"num_object_clones":0,"num_object_copies":9501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13455,"num_write_kb":531904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":115,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.784880+0000","last_change":"2024-08-29T15:08:58.887791+0000","last_active":"2024-08-29T15:13:28.784880+0000","last_peered":"2024-08-29T15:13:28.784880+0000","last_clean":"2024-08-29T15:13:28.784880+0000","last_became_active":"2024-08-29T15:08:58.887472+0000","last_became_peered":"2024-08-29T15:08:58.887472+0000","last_unstale":"2024-08-29T15:13:28.784880+0000","last_undegraded":"2024-08-29T15:13:28.784880+0000","last_fullsized":"2024-08-29T15:13:28.784880+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:35:58.690131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":559,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.802634+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:13:31.802634+0000","last_peered":"2024-08-29T15:13:31.802634+0000","last_clean":"2024-08-29T15:13:31.802634+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:13:31.802634+0000","last_undegraded":"2024-08-29T15:13:31.802634+0000","last_fullsized":"2024-08-29T15:13:31.802634+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":97,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.802233+0000","last_change":"2024-08-29T15:07:52.139543+0000","last_active":"2024-08-29T15:13:31.802233+0000","last_peered":"2024-08-29T15:13:31.802233+0000","last_clean":"2024-08-29T15:13:31.802233+0000","last_became_active":"2024-08-29T15:07:52.138499+0000","last_became_peered":"2024-08-29T15:07:52.138499+0000","last_unstale":"2024-08-29T15:13:31.802233+0000","last_undegraded":"2024-08-29T15:13:31.802233+0000","last_fullsized":"2024-08-29T15:13:31.802233+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:41:05.588954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"136'13434","reported_seq":13469,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.382557+0000","last_change":"2024-08-29T15:10:22.060236+0000","last_active":"2024-08-29T15:13:39.382557+0000","last_peered":"2024-08-29T15:13:39.382557+0000","last_clean":"2024-08-29T15:13:39.382557+0000","last_became_active":"2024-08-29T15:10:22.059894+0000","last_became_peered":"2024-08-29T15:10:22.059894+0000","last_unstale":"2024-08-29T15:13:39.382557+0000","last_undegraded":"2024-08-29T15:13:39.382557+0000","last_fullsized":"2024-08-29T15:13:39.382557+0000","mapping_epoch":111,"log_start":"117'3700","ondisk_log_start":"117'3700","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":9734,"log_dups_size":0,"ondisk_log_size":9734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:54:04.521598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204079104,"num_objects":3114,"num_object_clones":0,"num_object_copies":9342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13434,"num_write_kb":529536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.801995+0000","last_change":"2024-08-29T15:08:25.186307+0000","last_active":"2024-08-29T15:13:31.801995+0000","last_peered":"2024-08-29T15:13:31.801995+0000","last_clean":"2024-08-29T15:13:31.801995+0000","last_became_active":"2024-08-29T15:08:25.185788+0000","last_became_peered":"2024-08-29T15:08:25.185788+0000","last_unstale":"2024-08-29T15:13:31.801995+0000","last_undegraded":"2024-08-29T15:13:31.801995+0000","last_fullsized":"2024-08-29T15:13:31.801995+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:58:08.511525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":524,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.793753+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:13:28.793753+0000","last_peered":"2024-08-29T15:13:28.793753+0000","last_clean":"2024-08-29T15:13:28.793753+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:13:28.793753+0000","last_undegraded":"2024-08-29T15:13:28.793753+0000","last_fullsized":"2024-08-29T15:13:28.793753+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":108,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.804898+0000","last_change":"2024-08-29T15:07:20.813592+0000","last_active":"2024-08-29T15:13:31.804898+0000","last_peered":"2024-08-29T15:13:31.804898+0000","last_clean":"2024-08-29T15:13:31.804898+0000","last_became_active":"2024-08-29T15:07:20.812970+0000","last_became_peered":"2024-08-29T15:07:20.812970+0000","last_unstale":"2024-08-29T15:13:31.804898+0000","last_undegraded":"2024-08-29T15:13:31.804898+0000","last_fullsized":"2024-08-29T15:13:31.804898+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:54.850894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"136'13445","reported_seq":13479,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.490186+0000","last_change":"2024-08-29T15:10:22.060701+0000","last_active":"2024-08-29T15:13:39.490186+0000","last_peered":"2024-08-29T15:13:39.490186+0000","last_clean":"2024-08-29T15:13:39.490186+0000","last_became_active":"2024-08-29T15:10:22.060185+0000","last_became_peered":"2024-08-29T15:10:22.060185+0000","last_unstale":"2024-08-29T15:13:39.490186+0000","last_undegraded":"2024-08-29T15:13:39.490186+0000","last_fullsized":"2024-08-29T15:13:39.490186+0000","mapping_epoch":111,"log_start":"119'6100","ondisk_log_start":"119'6100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:10:46.794683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204144640,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13445,"num_write_kb":529920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":127,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.823019+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:13:31.823019+0000","last_peered":"2024-08-29T15:13:31.823019+0000","last_clean":"2024-08-29T15:13:31.823019+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:13:31.823019+0000","last_undegraded":"2024-08-29T15:13:31.823019+0000","last_fullsized":"2024-08-29T15:13:31.823019+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":650,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.802844+0000","last_change":"2024-08-29T15:07:52.139196+0000","last_active":"2024-08-29T15:13:31.802844+0000","last_peered":"2024-08-29T15:13:31.802844+0000","last_clean":"2024-08-29T15:13:31.802844+0000","last_became_active":"2024-08-29T15:07:52.138669+0000","last_became_peered":"2024-08-29T15:07:52.138669+0000","last_unstale":"2024-08-29T15:13:31.802844+0000","last_undegraded":"2024-08-29T15:13:31.802844+0000","last_fullsized":"2024-08-29T15:13:31.802844+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:43:54.949715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":124,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.823207+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:13:31.823207+0000","last_peered":"2024-08-29T15:13:31.823207+0000","last_clean":"2024-08-29T15:13:31.823207+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:13:31.823207+0000","last_undegraded":"2024-08-29T15:13:31.823207+0000","last_fullsized":"2024-08-29T15:13:31.823207+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"136'13060","reported_seq":13121,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:36.531484+0000","last_change":"2024-08-29T15:10:22.058096+0000","last_active":"2024-08-29T15:13:36.531484+0000","last_peered":"2024-08-29T15:13:36.531484+0000","last_clean":"2024-08-29T15:13:36.531484+0000","last_became_active":"2024-08-29T15:10:22.057728+0000","last_became_peered":"2024-08-29T15:10:22.057728+0000","last_unstale":"2024-08-29T15:13:36.531484+0000","last_undegraded":"2024-08-29T15:13:36.531484+0000","last_fullsized":"2024-08-29T15:13:36.531484+0000","mapping_epoch":111,"log_start":"117'4200","ondisk_log_start":"117'4200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8860,"log_dups_size":0,"ondisk_log_size":8860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:10:07.157182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216530944,"num_objects":3304,"num_object_clones":0,"num_object_copies":9912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13060,"num_write_kb":523648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":122,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.804977+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:13:31.804977+0000","last_peered":"2024-08-29T15:13:31.804977+0000","last_clean":"2024-08-29T15:13:31.804977+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:13:31.804977+0000","last_undegraded":"2024-08-29T15:13:31.804977+0000","last_fullsized":"2024-08-29T15:13:31.804977+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":364,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.790575+0000","last_change":"2024-08-29T15:08:25.186819+0000","last_active":"2024-08-29T15:13:28.790575+0000","last_peered":"2024-08-29T15:13:28.790575+0000","last_clean":"2024-08-29T15:13:28.790575+0000","last_became_active":"2024-08-29T15:08:25.186401+0000","last_became_peered":"2024-08-29T15:08:25.186401+0000","last_unstale":"2024-08-29T15:13:28.790575+0000","last_undegraded":"2024-08-29T15:13:28.790575+0000","last_fullsized":"2024-08-29T15:13:28.790575+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:45.101824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.789069+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T15:13:28.789069+0000","last_peered":"2024-08-29T15:13:28.789069+0000","last_clean":"2024-08-29T15:13:28.789069+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T15:13:28.789069+0000","last_undegraded":"2024-08-29T15:13:28.789069+0000","last_fullsized":"2024-08-29T15:13:28.789069+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":106,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.786284+0000","last_change":"2024-08-29T15:08:25.185452+0000","last_active":"2024-08-29T15:13:28.786284+0000","last_peered":"2024-08-29T15:13:28.786284+0000","last_clean":"2024-08-29T15:13:28.786284+0000","last_became_active":"2024-08-29T15:08:25.185095+0000","last_became_peered":"2024-08-29T15:08:25.185095+0000","last_unstale":"2024-08-29T15:13:28.786284+0000","last_undegraded":"2024-08-29T15:13:28.786284+0000","last_fullsized":"2024-08-29T15:13:28.786284+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:19:03.238217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":106,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.786750+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T15:13:28.786750+0000","last_peered":"2024-08-29T15:13:28.786750+0000","last_clean":"2024-08-29T15:13:28.786750+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T15:13:28.786750+0000","last_undegraded":"2024-08-29T15:13:28.786750+0000","last_fullsized":"2024-08-29T15:13:28.786750+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":108,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.802098+0000","last_change":"2024-08-29T15:08:25.186368+0000","last_active":"2024-08-29T15:13:31.802098+0000","last_peered":"2024-08-29T15:13:31.802098+0000","last_clean":"2024-08-29T15:13:31.802098+0000","last_became_active":"2024-08-29T15:08:25.185912+0000","last_became_peered":"2024-08-29T15:08:25.185912+0000","last_unstale":"2024-08-29T15:13:31.802098+0000","last_undegraded":"2024-08-29T15:13:31.802098+0000","last_fullsized":"2024-08-29T15:13:31.802098+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:23.124995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.793563+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:13:28.793563+0000","last_peered":"2024-08-29T15:13:28.793563+0000","last_clean":"2024-08-29T15:13:28.793563+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:13:28.793563+0000","last_undegraded":"2024-08-29T15:13:28.793563+0000","last_fullsized":"2024-08-29T15:13:28.793563+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.796020+0000","last_change":"2024-08-29T15:07:52.136541+0000","last_active":"2024-08-29T15:13:28.796020+0000","last_peered":"2024-08-29T15:13:28.796020+0000","last_clean":"2024-08-29T15:13:28.796020+0000","last_became_active":"2024-08-29T15:07:52.136202+0000","last_became_peered":"2024-08-29T15:07:52.136202+0000","last_unstale":"2024-08-29T15:13:28.796020+0000","last_undegraded":"2024-08-29T15:13:28.796020+0000","last_fullsized":"2024-08-29T15:13:28.796020+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:01.747963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":77,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822224+0000","last_change":"2024-08-29T15:08:58.888848+0000","last_active":"2024-08-29T15:13:31.822224+0000","last_peered":"2024-08-29T15:13:31.822224+0000","last_clean":"2024-08-29T15:13:31.822224+0000","last_became_active":"2024-08-29T15:08:58.886702+0000","last_became_peered":"2024-08-29T15:08:58.886702+0000","last_unstale":"2024-08-29T15:13:31.822224+0000","last_undegraded":"2024-08-29T15:13:31.822224+0000","last_fullsized":"2024-08-29T15:13:31.822224+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:56.189439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.805082+0000","last_change":"2024-08-29T15:08:58.887151+0000","last_active":"2024-08-29T15:13:31.805082+0000","last_peered":"2024-08-29T15:13:31.805082+0000","last_clean":"2024-08-29T15:13:31.805082+0000","last_became_active":"2024-08-29T15:08:58.886918+0000","last_became_peered":"2024-08-29T15:08:58.886918+0000","last_unstale":"2024-08-29T15:13:31.805082+0000","last_undegraded":"2024-08-29T15:13:31.805082+0000","last_fullsized":"2024-08-29T15:13:31.805082+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:37.557892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":129,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.823158+0000","last_change":"2024-08-29T15:08:58.889700+0000","last_active":"2024-08-29T15:13:31.823158+0000","last_peered":"2024-08-29T15:13:31.823158+0000","last_clean":"2024-08-29T15:13:31.823158+0000","last_became_active":"2024-08-29T15:08:58.887521+0000","last_became_peered":"2024-08-29T15:08:58.887521+0000","last_unstale":"2024-08-29T15:13:31.823158+0000","last_undegraded":"2024-08-29T15:13:31.823158+0000","last_fullsized":"2024-08-29T15:13:31.823158+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:32.997619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.790413+0000","last_change":"2024-08-29T15:07:52.135092+0000","last_active":"2024-08-29T15:13:28.790413+0000","last_peered":"2024-08-29T15:13:28.790413+0000","last_clean":"2024-08-29T15:13:28.790413+0000","last_became_active":"2024-08-29T15:07:52.134625+0000","last_became_peered":"2024-08-29T15:07:52.134625+0000","last_unstale":"2024-08-29T15:13:28.790413+0000","last_undegraded":"2024-08-29T15:13:28.790413+0000","last_fullsized":"2024-08-29T15:13:28.790413+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:22:35.250433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":144,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.823290+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:13:31.823290+0000","last_peered":"2024-08-29T15:13:31.823290+0000","last_clean":"2024-08-29T15:13:31.823290+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:13:31.823290+0000","last_undegraded":"2024-08-29T15:13:31.823290+0000","last_fullsized":"2024-08-29T15:13:31.823290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":119,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.805148+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:13:31.805148+0000","last_peered":"2024-08-29T15:13:31.805148+0000","last_clean":"2024-08-29T15:13:31.805148+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:13:31.805148+0000","last_undegraded":"2024-08-29T15:13:31.805148+0000","last_fullsized":"2024-08-29T15:13:31.805148+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.801866+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:13:31.801866+0000","last_peered":"2024-08-29T15:13:31.801866+0000","last_clean":"2024-08-29T15:13:31.801866+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:13:31.801866+0000","last_undegraded":"2024-08-29T15:13:31.801866+0000","last_fullsized":"2024-08-29T15:13:31.801866+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":127,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822410+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:13:31.822410+0000","last_peered":"2024-08-29T15:13:31.822410+0000","last_clean":"2024-08-29T15:13:31.822410+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:13:31.822410+0000","last_undegraded":"2024-08-29T15:13:31.822410+0000","last_fullsized":"2024-08-29T15:13:31.822410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":114,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.787992+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:13:28.787992+0000","last_peered":"2024-08-29T15:13:28.787992+0000","last_clean":"2024-08-29T15:13:28.787992+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:13:28.787992+0000","last_undegraded":"2024-08-29T15:13:28.787992+0000","last_fullsized":"2024-08-29T15:13:28.787992+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":148,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.801453+0000","last_change":"2024-08-29T15:08:58.884121+0000","last_active":"2024-08-29T15:13:31.801453+0000","last_peered":"2024-08-29T15:13:31.801453+0000","last_clean":"2024-08-29T15:13:31.801453+0000","last_became_active":"2024-08-29T15:08:58.883669+0000","last_became_peered":"2024-08-29T15:08:58.883669+0000","last_unstale":"2024-08-29T15:13:31.801453+0000","last_undegraded":"2024-08-29T15:13:31.801453+0000","last_fullsized":"2024-08-29T15:13:31.801453+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:32.165590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":111,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.804784+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:13:31.804784+0000","last_peered":"2024-08-29T15:13:31.804784+0000","last_clean":"2024-08-29T15:13:31.804784+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:13:31.804784+0000","last_undegraded":"2024-08-29T15:13:31.804784+0000","last_fullsized":"2024-08-29T15:13:31.804784+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.786214+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:13:28.786214+0000","last_peered":"2024-08-29T15:13:28.786214+0000","last_clean":"2024-08-29T15:13:28.786214+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:13:28.786214+0000","last_undegraded":"2024-08-29T15:13:28.786214+0000","last_fullsized":"2024-08-29T15:13:28.786214+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":101,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.783297+0000","last_change":"2024-08-29T15:07:52.134597+0000","last_active":"2024-08-29T15:13:28.783297+0000","last_peered":"2024-08-29T15:13:28.783297+0000","last_clean":"2024-08-29T15:13:28.783297+0000","last_became_active":"2024-08-29T15:07:52.133893+0000","last_became_peered":"2024-08-29T15:07:52.133893+0000","last_unstale":"2024-08-29T15:13:28.783297+0000","last_undegraded":"2024-08-29T15:13:28.783297+0000","last_fullsized":"2024-08-29T15:13:28.783297+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:14.240866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.787857+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:13:28.787857+0000","last_peered":"2024-08-29T15:13:28.787857+0000","last_clean":"2024-08-29T15:13:28.787857+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:13:28.787857+0000","last_undegraded":"2024-08-29T15:13:28.787857+0000","last_fullsized":"2024-08-29T15:13:28.787857+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":86,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.784946+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T15:13:28.784946+0000","last_peered":"2024-08-29T15:13:28.784946+0000","last_clean":"2024-08-29T15:13:28.784946+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T15:13:28.784946+0000","last_undegraded":"2024-08-29T15:13:28.784946+0000","last_fullsized":"2024-08-29T15:13:28.784946+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":147,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822478+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:13:31.822478+0000","last_peered":"2024-08-29T15:13:31.822478+0000","last_clean":"2024-08-29T15:13:31.822478+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:13:31.822478+0000","last_undegraded":"2024-08-29T15:13:31.822478+0000","last_fullsized":"2024-08-29T15:13:31.822478+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":136,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.822700+0000","last_change":"2024-08-29T15:08:58.888653+0000","last_active":"2024-08-29T15:13:31.822700+0000","last_peered":"2024-08-29T15:13:31.822700+0000","last_clean":"2024-08-29T15:13:31.822700+0000","last_became_active":"2024-08-29T15:08:58.888133+0000","last_became_peered":"2024-08-29T15:08:58.888133+0000","last_unstale":"2024-08-29T15:13:31.822700+0000","last_undegraded":"2024-08-29T15:13:31.822700+0000","last_fullsized":"2024-08-29T15:13:31.822700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:24.144974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":97,"reported_epoch":135,"state":"active+clean","last_fresh":"2024-08-29T15:13:28.790762+0000","last_change":"2024-08-29T15:07:52.135649+0000","last_active":"2024-08-29T15:13:28.790762+0000","last_peered":"2024-08-29T15:13:28.790762+0000","last_clean":"2024-08-29T15:13:28.790762+0000","last_became_active":"2024-08-29T15:07:52.135327+0000","last_became_peered":"2024-08-29T15:07:52.135327+0000","last_unstale":"2024-08-29T15:13:28.790762+0000","last_undegraded":"2024-08-29T15:13:28.790762+0000","last_fullsized":"2024-08-29T15:13:28.790762+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:05.402462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":123,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.805071+0000","last_change":"2024-08-29T15:08:25.187422+0000","last_active":"2024-08-29T15:13:31.805071+0000","last_peered":"2024-08-29T15:13:31.805071+0000","last_clean":"2024-08-29T15:13:31.805071+0000","last_became_active":"2024-08-29T15:08:25.187062+0000","last_became_peered":"2024-08-29T15:08:25.187062+0000","last_unstale":"2024-08-29T15:13:31.805071+0000","last_undegraded":"2024-08-29T15:13:31.805071+0000","last_fullsized":"2024-08-29T15:13:31.805071+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:18:20.745348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":3341811736,"num_objects":50993,"num_object_clones":0,"num_object_copies":152979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50993,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":216507,"num_write_kb":8559937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9995890688,"data_stored":9995878472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146707,"ondisk_log_size":146707,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234228,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1123396,"kb_used_data":1015996,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":92621756,"statfs":{"total":95995035648,"available":94844678144,"internally_reserved":0,"allocated":1040379904,"data_stored":1039659587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":109967819},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495864,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896628,"kb_used_data":821228,"kb_used_omap":1,"kb_used_meta":75390,"kb_avail":92848524,"statfs":{"total":95995035648,"available":95076888576,"internally_reserved":0,"allocated":840937472,"data_stored":840233115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":77199815},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757501,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1979948,"kb_used_data":1814308,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":91765204,"statfs":{"total":95995035648,"available":93967568896,"internally_reserved":0,"allocated":1857851392,"data_stored":1857150475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":169605574},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019138,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1793524,"kb_used_data":1654124,"kb_used_omap":1,"kb_used_meta":139390,"kb_avail":91951628,"statfs":{"total":95995035648,"available":94158467072,"internally_reserved":0,"allocated":1693822976,"data_stored":1693118132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":142735814},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435322,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":712996,"kb_used_data":641116,"kb_used_omap":4,"kb_used_meta":71867,"kb_avail":93032156,"statfs":{"total":95995035648,"available":95264927744,"internally_reserved":0,"allocated":656502784,"data_stored":655809797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":73592073},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598849,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353440,"kb_used_data":1241176,"kb_used_omap":7,"kb_used_meta":112184,"kb_avail":92391712,"statfs":{"total":95995035648,"available":94609113088,"internally_reserved":0,"allocated":1270964224,"data_stored":1270274633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":114876751},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762376,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1111588,"kb_used_data":1007708,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":92633564,"statfs":{"total":95995035648,"available":94856769536,"internally_reserved":0,"allocated":1031892992,"data_stored":1031199934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":106357090},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925902,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1814564,"kb_used_data":1666204,"kb_used_omap":7,"kb_used_meta":148280,"kb_avail":91930588,"statfs":{"total":95995035648,"available":94136922112,"internally_reserved":0,"allocated":1706192896,"data_stored":1705499526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":151838863},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1679360000,"data_stored":1679360000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1024065536,"data_stored":1024065536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1255673856,"data_stored":1255669784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":654180352,"data_stored":654180352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1670053888,"data_stored":1670053888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1851985920,"data_stored":1851981848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":830603264,"data_stored":830603264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1029967872,"data_stored":1029963800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:13:42.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:13:42.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:42.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:42.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:42.704 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:42.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:42.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:42.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:42.890 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:42.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:42.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:43.020 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:43.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:43.122 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:43.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:43.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.300 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:43.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:43.400 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:43.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:43.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:43.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:43.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:43.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:43.720 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:43.824 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:43.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:43.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:43.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:43.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:43.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:43 smithi027 ceph-mon[98970]: pgmap v416: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-29T15:13:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:43 smithi027 ceph-mon[93921]: pgmap v416: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-29T15:13:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:44.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:44.119 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:44.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:44.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:44.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:44.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:44.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:43 smithi171 ceph-mon[90031]: pgmap v416: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-29T15:13:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:13:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:13:44.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:44.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:44.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:44.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:44.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:44.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:44.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:44.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:44.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:44.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:44.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:44.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:44.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:44.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:44.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:44.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:44.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:44.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:45.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:45.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:45.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:45.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:45.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:45.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:45.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:45.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:45.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:45.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:45.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:45.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:45.490 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:45.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:45.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:45.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:45.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:45.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:45.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:45.764 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:45.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:45.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:45.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:45.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:45.932 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:45.932 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:45.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:46.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:46.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:46.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:46.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:45 smithi027 ceph-mon[93921]: pgmap v417: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-08-29T15:13:46.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:45 smithi027 ceph-mon[98970]: pgmap v417: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-08-29T15:13:46.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:46.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:46.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:45 smithi171 ceph-mon[90031]: pgmap v417: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-08-29T15:13:46.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:46.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:46.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:46.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:46.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:46.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:46.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:46.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:46.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:46.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:46.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:46.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:46.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:46.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:46.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:46.879 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:13:46.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:46.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:46.923 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23213 5.5 5:abfc38ee:::benchmark_data_smithi027_135490_object23212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.959676+0000", 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 185.96094390600001, 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13413031, 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.924 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23213 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959676+0000", 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959676+0000", 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959677+0000", 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.508e-06 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959731+0000", 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3182e-05 2024-08-29T15:13:46.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959744+0000", 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3794999999999999e-05 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959752+0000", 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6580000000000002e-06 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959815+0000", 2024-08-29T15:13:46.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3294999999999996e-05 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.959979+0000", 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016389099999999999 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960134+0000", 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000154661 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961191+0000", 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010571910000000001 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961722+0000", 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00053042699999999998 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093703+0000", 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13198111700000001 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093762+0000", 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9286999999999997e-05 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093806+0000", 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4298000000000002e-05 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:23215 5.e e115/111 5:7c274c1d:::benchmark_data_smithi027_135490_object23214:head v 115'1485, mlcod=115'1483)", 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.960868+0000", 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 185.95975187600001, 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13105096899999999, 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960868+0000", 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960868+0000", 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960869+0000", 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2600000000000001e-07 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960931+0000", 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2082999999999995e-05 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960942+0000", 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0729e-05 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960948+0000", 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9390000000000002e-06 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.960998+0000", 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0087999999999999e-05 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961015+0000", 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7478000000000001e-05 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.091867+0000", 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.130851896 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.091919+0000", 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1929999999999999e-05 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.934 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23216 5.3 5:c08b9121:::benchmark_data_smithi027_135490_object23215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.961706+0000", 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 185.958914015, 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.137816141, 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23216 2024-08-29T15:13:46.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961706+0000", 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961706+0000", 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961707+0000", 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.031e-06 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961728+0000", 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1568000000000001e-05 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961738+0000", 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5410000000000006e-06 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961745+0000", 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3379999999999999e-06 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093391+0000", 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13164532300000001 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093512+0000", 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120995 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093632+0000", 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012078099999999999 2024-08-29T15:13:46.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.096349+0000", 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002716825 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.096525+0000", 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017614099999999999 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.098729+0000", 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022036999999999998 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.099473+0000", 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00074389300000000003 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.099522+0000", 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9005000000000003e-05 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.941 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62785 5.3 5:c3da9dee:::benchmark_data_smithi027_135490_object62784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 166.22255841399999, 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.23727498499999999, 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.942 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62785 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698064+0000", 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2879999999999999e-06 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698106+0000", 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2039999999999997e-05 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698128+0000", 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2064000000000001e-05 2024-08-29T15:13:46.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698136+0000", 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3790000000000006e-06 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.700871+0000", 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002734972 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.701013+0000", 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141607 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.701141+0000", 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012853499999999999 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.704979+0000", 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038374239999999999 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.716497+0000", 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011518188 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935036+0000", 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.218538912 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935095+0000", 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8857000000000002e-05 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935336+0000", 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024171899999999999 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62814 5.3 5:c9d52f91:::benchmark_data_smithi027_135490_object62813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 166.19413501899999, 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.210827087, 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.948 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62814 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726486+0000", 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5e-06 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726507+0000", 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0245e-05 2024-08-29T15:13:46.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726519+0000", 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2074e-05 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726526+0000", 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7980000000000006e-06 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726593+0000", 2024-08-29T15:13:46.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6962e-05 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726740+0000", 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014641200000000001 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726858+0000", 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011843 2024-08-29T15:13:46.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.728600+0000", 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017419340000000001 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.728790+0000", 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018932899999999999 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937147+0000", 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20835720599999999 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937263+0000", 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116499 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937312+0000", 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8698e-05 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62826 5.3 5:c75bdcb4:::benchmark_data_smithi027_135490_object62825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 166.16931759799999, 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.188340805, 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62826 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.955 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751303+0000", 2024-08-29T15:13:46.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.201e-06 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751324+0000", 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0271e-05 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751356+0000", 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2428999999999997e-05 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751364+0000", 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9039999999999992e-06 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751427+0000", 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3304000000000002e-05 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751579+0000", 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015205000000000001 2024-08-29T15:13:46.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751757+0000", 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017760700000000001 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.754471+0000", 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027138589999999999 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.756577+0000", 2024-08-29T15:13:46.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021064460000000001 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939494+0000", 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18291691700000001 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939598+0000", 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010330700000000001 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939643+0000", 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5510000000000003e-05 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62843 5.3 5:c10518e0:::benchmark_data_smithi027_135490_object62842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 166.158347187, 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17756435800000001, 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.961 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62843 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762274+0000", 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.116e-06 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762294+0000", 2024-08-29T15:13:46.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9743000000000001e-05 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762308+0000", 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4959e-05 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762316+0000", 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1829999999999997e-06 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762372+0000", 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6172999999999999e-05 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762497+0000", 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012541700000000001 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762631+0000", 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013369200000000001 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.765559+0000", 2024-08-29T15:13:46.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029280959999999998 2024-08-29T15:13:46.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.765658+0000", 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8919999999999995e-05 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939688+0000", 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17403020299999999 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939785+0000", 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6971000000000004e-05 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939837+0000", 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1885000000000002e-05 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.968 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62883 5.3 5:c29effb9:::benchmark_data_smithi027_135490_object62882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 166.13187829899999, 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15126415900000001, 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62883 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788744+0000", 2024-08-29T15:13:46.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5919999999999999e-06 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788770+0000", 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5687999999999999e-05 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788788+0000", 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8099000000000001e-05 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788798+0000", 2024-08-29T15:13:46.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7659999999999993e-06 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788890+0000", 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2724000000000002e-05 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.789064+0000", 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000174048 2024-08-29T15:13:46.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.789215+0000", 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015093600000000001 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.791500+0000", 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00228502 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.791885+0000", 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038479299999999997 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939879+0000", 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14799395700000001 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939965+0000", 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6304999999999994e-05 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940006+0000", 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0231000000000002e-05 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62891 5.3 5:cec157a3:::benchmark_data_smithi027_135490_object62890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.797135+0000", 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 166.12348472400001, 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14327178700000001, 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.975 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62891 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797135+0000", 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797135+0000", 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797137+0000", 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0619999999999999e-06 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797160+0000", 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2721000000000002e-05 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797170+0000", 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9359999999999994e-06 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797175+0000", 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3430000000000003e-06 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797229+0000", 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.342e-05 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797374+0000", 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014522499999999999 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797534+0000", 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000159679 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.799392+0000", 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018581100000000001 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.799781+0000", 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038978999999999999 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940045+0000", 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14026403400000001 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940364+0000", 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031812799999999999 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940407+0000", 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3339e-05 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62899 5.3 5:c4e1726c:::benchmark_data_smithi027_135490_object62898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.814956+0000", 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 166.10566426700001, 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12571695299999999, 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.981 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62899 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.814956+0000", 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.814956+0000", 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.814957+0000", 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7129999999999999e-06 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.814980+0000", 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2684000000000002e-05 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.814994+0000", 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3991e-05 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.815001+0000", 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1539999999999996e-06 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.815056+0000", 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4437000000000001e-05 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.815155+0000", 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9402999999999997e-05 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.815249+0000", 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3712e-05 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.816963+0000", 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001713883 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.817191+0000", 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000228798 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940450+0000", 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12325836 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940634+0000", 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184155 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940673+0000", 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8662999999999999e-05 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.987 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68893 5.5 5:a47e561b:::benchmark_data_smithi027_135490_object68892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.01906717400001, 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14711858999999999, 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68893 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901554+0000", 2024-08-29T15:13:46.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.24e-06 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901576+0000", 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2155999999999999e-05 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901587+0000", 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1258e-05 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901596+0000", 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3930000000000008e-06 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904408+0000", 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028119130000000001 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904518+0000", 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010994500000000001 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904644+0000", 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012621299999999999 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907190+0000", 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025458640000000001 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048542+0000", 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.141352177 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048577+0000", 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4686e-05 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048622+0000", 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5382e-05 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048671+0000", 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9363000000000002e-05 2024-08-29T15:13:46.993 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68895 5.3 5:c72cbd73:::benchmark_data_smithi027_135490_object68894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.01844219200001, 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.151393162, 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:46.994 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68895 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:13:46.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902179+0000", 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8700000000000004e-07 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902204+0000", 2024-08-29T15:13:46.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5474999999999999e-05 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2199e-05 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902224+0000", 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1289999999999996e-06 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904177+0000", 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00195218 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904303+0000", 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012648 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904431+0000", 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012815400000000001 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909174+0000", 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047423450000000002 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909243+0000", 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9497000000000002e-05 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053476+0000", 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.144233477 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:46.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053526+0000", 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9477000000000001e-05 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053571+0000", 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4861999999999998e-05 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.000 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68897 5.5 5:adfd8055:::benchmark_data_smithi027_135490_object68896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.01834025900001, 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.147127652, 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68897 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7599999999999997e-07 2024-08-29T15:13:47.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902303+0000", 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2676000000000001e-05 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902313+0000", 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7529999999999996e-06 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902319+0000", 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6290000000000004e-06 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907215+0000", 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048957189999999998 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907355+0000", 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139775 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:13:47.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907500+0000", 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014554800000000001 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049212+0000", 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14171125700000001 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049280+0000", 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8319999999999996e-05 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049292+0000", 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1849e-05 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049363+0000", 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1616e-05 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049407+0000", 2024-08-29T15:13:47.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3934000000000003e-05 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:68893 5.5 e117/111 ondisk, result = 0)", 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.908863+0000", 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.011756679, 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13968623199999999, 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908863+0000", 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908863+0000", 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908864+0000", 2024-08-29T15:13:47.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0200000000000002e-07 2024-08-29T15:13:47.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1570161903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:47.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1570161903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908867+0000", 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9239999999999999e-06 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908876+0000", 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5230000000000001e-06 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908882+0000", 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9100000000000002e-06 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048526+0000", 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13964390500000001 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048535+0000", 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2369999999999998e-06 2024-08-29T15:13:47.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048549+0000", 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4131000000000001e-05 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68907 5.3 5:cf461b57:::benchmark_data_smithi027_135490_object68906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.908921+0000", 2024-08-29T15:13:47.012 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.01169902500001, 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.145667885, 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68907 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908921+0000", 2024-08-29T15:13:47.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908921+0000", 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908922+0000", 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.147e-06 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908942+0000", 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0259e-05 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908951+0000", 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2099999999999999e-06 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908957+0000", 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.468e-06 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909276+0000", 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031944699999999999 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048670+0000", 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.139393394 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048861+0000", 2024-08-29T15:13:47.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000190982 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053583+0000", 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047221590000000001 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053644+0000", 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1029000000000001e-05 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:13:47.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054502+0000", 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00085849299999999999 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054552+0000", 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9160999999999999e-05 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054589+0000", 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7135999999999998e-05 2024-08-29T15:13:47.018 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:68906 5.e e117/111 5:73dcaf78:::benchmark_data_smithi027_135490_object68905:head v 117'4372, mlcod=117'4370)", 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.909133+0000", 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.01148657499999, 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13943607699999999, 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.019 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909133+0000", 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909133+0000", 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.020 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909134+0000", 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8400000000000004e-07 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909164+0000", 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9853999999999999e-05 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909172+0000", 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4999999999999999e-06 2024-08-29T15:13:47.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909178+0000", 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9059999999999999e-06 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909239+0000", 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1291999999999995e-05 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909267+0000", 2024-08-29T15:13:47.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7735000000000001e-05 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048521+0000", 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13925380900000001 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048569+0000", 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8396999999999998e-05 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.023 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:68896 5.b e117/111 5:d4a8571b:::benchmark_data_smithi027_135490_object68895:head v 117'4385, mlcod=117'4382)", 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.910870+0000", 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.00975010799999, 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.142751935, 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910870+0000", 2024-08-29T15:13:47.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910870+0000", 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910871+0000", 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4099999999999997e-07 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910895+0000", 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4634000000000002e-05 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048477+0000", 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.137582223 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048489+0000", 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1814000000000001e-05 2024-08-29T15:13:47.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048682+0000", 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019234800000000001 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048706+0000", 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4893e-05 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:47.027 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053584+0000", 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048777220000000001 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053622+0000", 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7459999999999997e-05 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.028 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:68893 5.5 e117/111 ondisk, result = 0)", 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.910871+0000", 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.00974883200001, 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.137806919, 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910871+0000", 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.029 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910871+0000", 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910872+0000", 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6799999999999999e-07 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910877+0000", 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8749999999999999e-06 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048388+0000", 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.137510997 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048411+0000", 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2918999999999999e-05 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048568+0000", 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000157357 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048573+0000", 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9810000000000003e-06 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048678+0000", 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105022 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:68909 5.7 e117/111 5:eeb8103f:::benchmark_data_smithi027_135490_object68908:head v 117'4363, mlcod=117'4360)", 2024-08-29T15:13:47.032 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.912355+0000", 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 162.00826487399999, 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.143279975, 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912355+0000", 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.033 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912355+0000", 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912356+0000", 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9800000000000003e-07 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912422+0000", 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5881000000000002e-05 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048786+0000", 2024-08-29T15:13:47.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.136363978 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048794+0000", 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8300000000000002e-06 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048860+0000", 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6100999999999996e-05 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048881+0000", 2024-08-29T15:13:47.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0682999999999998e-05 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055514+0000", 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006632827 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055635+0000", 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120977 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.036 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:220846 5.7 e136/111 5:ef61a58d:::benchmark_data_smithi027_135490_object87094:head v 136'13929, mlcod=136'13927)", 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:13:44.079484+0000", 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 2.8411356319999999, 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11845694800000001, 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.079484+0000", 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.079484+0000", 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.079485+0000", 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1299999999999999e-07 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.079491+0000", 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3759999999999996e-06 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.166898+0000", 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.087406782000000002 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.166908+0000", 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9699999999999994e-06 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.167404+0000", 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000495854 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.167423+0000", 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9295000000000001e-05 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.197761+0000", 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030337981 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:13:47.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:44.197941+0000", 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000179877 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:13:47.041 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:13:47.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:47.076 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:47.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:47.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:47.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:47.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:47.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1570161903' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:13:47.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:47.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:47.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:47.444 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:47.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:47.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:47.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:47.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:47.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:47.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:47.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:47.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:47.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:47.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:47.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:47.883 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:47.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:47.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:48.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[93921]: pgmap v418: 89 pgs: 89 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2024-08-29T15:13:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1570161903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:13:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[93921]: osdmap e137: 8 total, 8 up, 8 in 2024-08-29T15:13:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[98970]: pgmap v418: 89 pgs: 89 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2024-08-29T15:13:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1570161903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:13:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:47 smithi027 ceph-mon[98970]: osdmap e137: 8 total, 8 up, 8 in 2024-08-29T15:13:48.115 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:48.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:48.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:48.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:47 smithi171 ceph-mon[90031]: pgmap v418: 89 pgs: 89 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2024-08-29T15:13:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:47 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:13:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1570161903' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:13:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:47 smithi171 ceph-mon[90031]: osdmap e137: 8 total, 8 up, 8 in 2024-08-29T15:13:48.288 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:48.389 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:48.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:48.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:48.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:48.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:48.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:48.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:48.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:48.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:48.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:48.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:48.975 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:48.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:49.077 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:49.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:49.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:49.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:49.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:49.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:49.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:13:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:13:49] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-29T15:13:49.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:49.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:49.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:49.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:13:49.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:49.572 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:49.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:49.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:49.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:49.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:49.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:49.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:49.848 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:49.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:49.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:49.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:49.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:50.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:49 smithi027 ceph-mon[93921]: pgmap v420: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-08-29T15:13:50.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:49 smithi027 ceph-mon[98970]: pgmap v420: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-08-29T15:13:50.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:50.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:50.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:50.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:50.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:50.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:50.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:49 smithi171 ceph-mon[90031]: pgmap v420: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-08-29T15:13:50.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:50.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:50.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:50.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:50.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:50.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:50.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:50.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:50.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:50.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:50.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:50.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:50.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:50.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:50.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:50.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:50.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:50.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:51.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:51.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:51.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:51.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:51.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:51.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:51.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:51.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:51.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:51.346 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:51.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:51.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:51.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:51.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:51.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:51.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:51.630 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:51.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:51.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:51.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:51.773 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:51.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:51.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:51.906 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:51.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:52.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:52.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:52.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:52.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:52.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:52.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:51 smithi171 ceph-mon[90031]: pgmap v421: 89 pgs: 89 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-08-29T15:13:52.276 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:51 smithi027 ceph-mon[93921]: pgmap v421: 89 pgs: 89 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-08-29T15:13:52.276 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:51 smithi027 ceph-mon[98970]: pgmap v421: 89 pgs: 89 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-08-29T15:13:52.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:52.323 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:52.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:52.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:52.472 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:52.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:52.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:52.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:52.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:52.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:52.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:52.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:52.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:52.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:52.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:52.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:52.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:52.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:13:53.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:53.019 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:13:53.019 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:13:53.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:13:53.055 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:13:53.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:53.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:53.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:53.109 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:13:53.109 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:13:53.156 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:13:53.156 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:13:53.180 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:53.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:53.207 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:13:53.207 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:13:53.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:53.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:53.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:53.255 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:13:53.255 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:13:53.287 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:13:53.288 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:13:53.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:53.339 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:13:53.339 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:13:53.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:53.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:53.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:53.486 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:53.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:53.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:53.591 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:13:53.593 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:13:53.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:53.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:53.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:53.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:53.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:53.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:53.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:53.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:53.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:53.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:54.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:54.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:54.135 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:54.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:53 smithi171 ceph-mon[90031]: pgmap v422: 89 pgs: 89 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-08-29T15:13:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4159894272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:54.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:54.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:54.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:53 smithi027 ceph-mon[93921]: pgmap v422: 89 pgs: 89 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-08-29T15:13:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4159894272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:53 smithi027 ceph-mon[98970]: pgmap v422: 89 pgs: 89 active+clean; 2.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-08-29T15:13:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4159894272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:13:54.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:54.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:54.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:54.579 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:54.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:54.717 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:54.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:54.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:54.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:54.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:54.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:54.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:54 smithi027 ceph-mon[98970]: from='client.34839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:13:54.966 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:54 smithi027 ceph-mon[93921]: from='client.34839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:13:54.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:54.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:54.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:54.999 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":422,"stamp":"2024-08-29T15:13:52.408741+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2771261437,"num_objects":42242,"num_object_clones":0,"num_object_copies":126726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42242,"num_whiteouts":0,"num_read":6391,"num_read_kb":13846,"num_write":226409,"num_write_kb":8574978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":148068,"ondisk_log_size":148068,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9075024,"kb_used_data":8179268,"kb_used_omap":34,"kb_used_meta":895133,"kb_avail":740886192,"statfs":{"total":767960285184,"available":758667460608,"internally_reserved":0,"allocated":8375570432,"data_stored":8369938511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34969,"internal_metadata":916617063},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,2,7,12,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-557580288,"num_objects":-8508,"num_object_clones":0,"num_object_copies":-25524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8508,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005188"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":144,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.904631+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:13:46.904631+0000","last_peered":"2024-08-29T15:13:46.904631+0000","last_clean":"2024-08-29T15:13:46.904631+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:13:46.904631+0000","last_undegraded":"2024-08-29T15:13:46.904631+0000","last_fullsized":"2024-08-29T15:13:46.904631+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.815944+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:13:31.815944+0000","last_peered":"2024-08-29T15:13:31.815944+0000","last_clean":"2024-08-29T15:13:31.815944+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:13:31.815944+0000","last_undegraded":"2024-08-29T15:13:31.815944+0000","last_fullsized":"2024-08-29T15:13:31.815944+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.811768+0000","last_change":"2024-08-29T15:08:58.887518+0000","last_active":"2024-08-29T15:13:31.811768+0000","last_peered":"2024-08-29T15:13:31.811768+0000","last_clean":"2024-08-29T15:13:31.811768+0000","last_became_active":"2024-08-29T15:08:58.887049+0000","last_became_peered":"2024-08-29T15:08:58.887049+0000","last_unstale":"2024-08-29T15:13:31.811768+0000","last_undegraded":"2024-08-29T15:13:31.811768+0000","last_fullsized":"2024-08-29T15:13:31.811768+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:23:07.638407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895620+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:13:46.895620+0000","last_peered":"2024-08-29T15:13:46.895620+0000","last_clean":"2024-08-29T15:13:46.895620+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:13:46.895620+0000","last_undegraded":"2024-08-29T15:13:46.895620+0000","last_fullsized":"2024-08-29T15:13:46.895620+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895951+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:13:46.895951+0000","last_peered":"2024-08-29T15:13:46.895951+0000","last_clean":"2024-08-29T15:13:46.895951+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:13:46.895951+0000","last_undegraded":"2024-08-29T15:13:46.895951+0000","last_fullsized":"2024-08-29T15:13:46.895951+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":85,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816301+0000","last_change":"2024-08-29T14:55:32.413915+0000","last_active":"2024-08-29T15:13:31.816301+0000","last_peered":"2024-08-29T15:13:31.816301+0000","last_clean":"2024-08-29T15:13:31.816301+0000","last_became_active":"2024-08-29T14:55:32.413485+0000","last_became_peered":"2024-08-29T14:55:32.413485+0000","last_unstale":"2024-08-29T15:13:31.816301+0000","last_undegraded":"2024-08-29T15:13:31.816301+0000","last_fullsized":"2024-08-29T15:13:31.816301+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:10.711579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":131,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.905494+0000","last_change":"2024-08-29T15:08:58.884675+0000","last_active":"2024-08-29T15:13:46.905494+0000","last_peered":"2024-08-29T15:13:46.905494+0000","last_clean":"2024-08-29T15:13:46.905494+0000","last_became_active":"2024-08-29T15:08:58.884007+0000","last_became_peered":"2024-08-29T15:08:58.884007+0000","last_unstale":"2024-08-29T15:13:46.905494+0000","last_undegraded":"2024-08-29T15:13:46.905494+0000","last_fullsized":"2024-08-29T15:13:46.905494+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:12:03.629717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.818091+0000","last_change":"2024-08-29T15:07:20.812313+0000","last_active":"2024-08-29T15:13:31.818091+0000","last_peered":"2024-08-29T15:13:31.818091+0000","last_clean":"2024-08-29T15:13:31.818091+0000","last_became_active":"2024-08-29T15:07:20.811973+0000","last_became_peered":"2024-08-29T15:07:20.811973+0000","last_unstale":"2024-08-29T15:13:31.818091+0000","last_undegraded":"2024-08-29T15:13:31.818091+0000","last_fullsized":"2024-08-29T15:13:31.818091+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:37.283470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"137'13957","reported_seq":13992,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:49.464209+0000","last_change":"2024-08-29T15:10:22.057568+0000","last_active":"2024-08-29T15:13:49.464209+0000","last_peered":"2024-08-29T15:13:49.464209+0000","last_clean":"2024-08-29T15:13:49.464209+0000","last_became_active":"2024-08-29T15:10:22.056935+0000","last_became_peered":"2024-08-29T15:10:22.056935+0000","last_unstale":"2024-08-29T15:13:49.464209+0000","last_undegraded":"2024-08-29T15:13:49.464209+0000","last_fullsized":"2024-08-29T15:13:49.464209+0000","mapping_epoch":111,"log_start":"118'5300","ondisk_log_start":"118'5300","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8657,"log_dups_size":0,"ondisk_log_size":8657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:33:23.576258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175964160,"num_objects":2685,"num_object_clones":0,"num_object_copies":8055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13957,"num_write_kb":532544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:34.513052+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:13:34.513052+0000","last_peered":"2024-08-29T15:13:34.513052+0000","last_clean":"2024-08-29T15:13:34.513052+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:13:34.513052+0000","last_undegraded":"2024-08-29T15:13:34.513052+0000","last_fullsized":"2024-08-29T15:13:34.513052+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.812583+0000","last_change":"2024-08-29T14:55:33.423445+0000","last_active":"2024-08-29T15:13:31.812583+0000","last_peered":"2024-08-29T15:13:31.812583+0000","last_clean":"2024-08-29T15:13:31.812583+0000","last_became_active":"2024-08-29T14:55:33.423176+0000","last_became_peered":"2024-08-29T14:55:33.423176+0000","last_unstale":"2024-08-29T15:13:31.812583+0000","last_undegraded":"2024-08-29T15:13:31.812583+0000","last_fullsized":"2024-08-29T15:13:31.812583+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:44.344082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"137'13847","reported_seq":13910,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:48.639715+0000","last_change":"2024-08-29T15:10:22.061175+0000","last_active":"2024-08-29T15:13:48.639715+0000","last_peered":"2024-08-29T15:13:48.639715+0000","last_clean":"2024-08-29T15:13:48.639715+0000","last_became_active":"2024-08-29T15:10:22.060889+0000","last_became_peered":"2024-08-29T15:10:22.060889+0000","last_unstale":"2024-08-29T15:13:48.639715+0000","last_undegraded":"2024-08-29T15:13:48.639715+0000","last_fullsized":"2024-08-29T15:13:48.639715+0000","mapping_epoch":111,"log_start":"117'3800","ondisk_log_start":"117'3800","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10047,"log_dups_size":0,"ondisk_log_size":10047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:59:27.186639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171900928,"num_objects":2623,"num_object_clones":0,"num_object_copies":7869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13847,"num_write_kb":527040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:34.513035+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:13:34.513035+0000","last_peered":"2024-08-29T15:13:34.513035+0000","last_clean":"2024-08-29T15:13:34.513035+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:13:34.513035+0000","last_undegraded":"2024-08-29T15:13:34.513035+0000","last_fullsized":"2024-08-29T15:13:34.513035+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":114,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816880+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:13:31.816880+0000","last_peered":"2024-08-29T15:13:31.816880+0000","last_clean":"2024-08-29T15:13:31.816880+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:13:31.816880+0000","last_undegraded":"2024-08-29T15:13:31.816880+0000","last_fullsized":"2024-08-29T15:13:31.816880+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"137'14078","reported_seq":14139,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:48.664951+0000","last_change":"2024-08-29T15:10:22.060048+0000","last_active":"2024-08-29T15:13:48.664951+0000","last_peered":"2024-08-29T15:13:48.664951+0000","last_clean":"2024-08-29T15:13:48.664951+0000","last_became_active":"2024-08-29T15:10:22.059752+0000","last_became_peered":"2024-08-29T15:10:22.059752+0000","last_unstale":"2024-08-29T15:13:48.664951+0000","last_undegraded":"2024-08-29T15:13:48.664951+0000","last_fullsized":"2024-08-29T15:13:48.664951+0000","mapping_epoch":111,"log_start":"117'4000","ondisk_log_start":"117'4000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10078,"log_dups_size":0,"ondisk_log_size":10078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:17:39.363676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177733632,"num_objects":2712,"num_object_clones":0,"num_object_copies":8136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14078,"num_write_kb":537280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:34.513008+0000","last_change":"2024-08-29T14:55:32.404805+0000","last_active":"2024-08-29T15:13:34.513008+0000","last_peered":"2024-08-29T15:13:34.513008+0000","last_clean":"2024-08-29T15:13:34.513008+0000","last_became_active":"2024-08-29T14:55:32.404493+0000","last_became_peered":"2024-08-29T14:55:32.404493+0000","last_unstale":"2024-08-29T15:13:34.513008+0000","last_undegraded":"2024-08-29T15:13:34.513008+0000","last_fullsized":"2024-08-29T15:13:34.513008+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:47.378979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.818568+0000","last_change":"2024-08-29T15:07:52.136104+0000","last_active":"2024-08-29T15:13:31.818568+0000","last_peered":"2024-08-29T15:13:31.818568+0000","last_clean":"2024-08-29T15:13:31.818568+0000","last_became_active":"2024-08-29T15:07:52.135781+0000","last_became_peered":"2024-08-29T15:07:52.135781+0000","last_unstale":"2024-08-29T15:13:31.818568+0000","last_undegraded":"2024-08-29T15:13:31.818568+0000","last_fullsized":"2024-08-29T15:13:31.818568+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:31:26.381499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"137'14273","reported_seq":14337,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:48.647487+0000","last_change":"2024-08-29T15:10:22.059900+0000","last_active":"2024-08-29T15:13:48.647487+0000","last_peered":"2024-08-29T15:13:48.647487+0000","last_clean":"2024-08-29T15:13:48.647487+0000","last_became_active":"2024-08-29T15:10:22.059601+0000","last_became_peered":"2024-08-29T15:10:22.059601+0000","last_unstale":"2024-08-29T15:13:48.647487+0000","last_undegraded":"2024-08-29T15:13:48.647487+0000","last_fullsized":"2024-08-29T15:13:48.647487+0000","mapping_epoch":111,"log_start":"117'4200","ondisk_log_start":"117'4200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10073,"log_dups_size":0,"ondisk_log_size":10073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:19:28.744391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180289536,"num_objects":2751,"num_object_clones":0,"num_object_copies":8253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14273,"num_write_kb":544768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:34.669483+0000","last_change":"2024-08-29T15:08:58.887669+0000","last_active":"2024-08-29T15:13:34.669483+0000","last_peered":"2024-08-29T15:13:34.669483+0000","last_clean":"2024-08-29T15:13:34.669483+0000","last_became_active":"2024-08-29T15:08:58.887343+0000","last_became_peered":"2024-08-29T15:08:58.887343+0000","last_unstale":"2024-08-29T15:13:34.669483+0000","last_undegraded":"2024-08-29T15:13:34.669483+0000","last_fullsized":"2024-08-29T15:13:34.669483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:32.280756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896905+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:13:46.896905+0000","last_peered":"2024-08-29T15:13:46.896905+0000","last_clean":"2024-08-29T15:13:46.896905+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:13:46.896905+0000","last_undegraded":"2024-08-29T15:13:46.896905+0000","last_fullsized":"2024-08-29T15:13:46.896905+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"137'14058","reported_seq":14123,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:48.633140+0000","last_change":"2024-08-29T15:10:22.059154+0000","last_active":"2024-08-29T15:13:48.633140+0000","last_peered":"2024-08-29T15:13:48.633140+0000","last_clean":"2024-08-29T15:13:48.633140+0000","last_became_active":"2024-08-29T15:10:22.058770+0000","last_became_peered":"2024-08-29T15:10:22.058770+0000","last_unstale":"2024-08-29T15:13:48.633140+0000","last_undegraded":"2024-08-29T15:13:48.633140+0000","last_fullsized":"2024-08-29T15:13:48.633140+0000","mapping_epoch":111,"log_start":"117'4000","ondisk_log_start":"117'4000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:03.961012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410584,"num_objects":2586,"num_object_clones":0,"num_object_copies":7758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2586,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14058,"num_write_kb":532545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":129,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.904991+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:13:46.904991+0000","last_peered":"2024-08-29T15:13:46.904991+0000","last_clean":"2024-08-29T15:13:46.904991+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:13:46.904991+0000","last_undegraded":"2024-08-29T15:13:46.904991+0000","last_fullsized":"2024-08-29T15:13:46.904991+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.565807+0000","last_change":"2024-08-29T15:08:58.886369+0000","last_active":"2024-08-29T15:13:39.565807+0000","last_peered":"2024-08-29T15:13:39.565807+0000","last_clean":"2024-08-29T15:13:39.565807+0000","last_became_active":"2024-08-29T15:08:58.886022+0000","last_became_peered":"2024-08-29T15:08:58.886022+0000","last_unstale":"2024-08-29T15:13:39.565807+0000","last_undegraded":"2024-08-29T15:13:39.565807+0000","last_fullsized":"2024-08-29T15:13:39.565807+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:36.451849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"137'14126","reported_seq":14189,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:50.329891+0000","last_change":"2024-08-29T15:10:22.060607+0000","last_active":"2024-08-29T15:13:50.329891+0000","last_peered":"2024-08-29T15:13:50.329891+0000","last_clean":"2024-08-29T15:13:50.329891+0000","last_became_active":"2024-08-29T15:10:22.060299+0000","last_became_peered":"2024-08-29T15:10:22.060299+0000","last_unstale":"2024-08-29T15:13:50.329891+0000","last_undegraded":"2024-08-29T15:13:50.329891+0000","last_fullsized":"2024-08-29T15:13:50.329891+0000","mapping_epoch":111,"log_start":"118'6000","ondisk_log_start":"118'6000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:54:02.847420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171573248,"num_objects":2618,"num_object_clones":0,"num_object_copies":7854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14126,"num_write_kb":535808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.819078+0000","last_change":"2024-08-29T15:07:52.134960+0000","last_active":"2024-08-29T15:13:31.819078+0000","last_peered":"2024-08-29T15:13:31.819078+0000","last_clean":"2024-08-29T15:13:31.819078+0000","last_became_active":"2024-08-29T15:07:52.134608+0000","last_became_peered":"2024-08-29T15:07:52.134608+0000","last_unstale":"2024-08-29T15:13:31.819078+0000","last_undegraded":"2024-08-29T15:13:31.819078+0000","last_fullsized":"2024-08-29T15:13:31.819078+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:42.154504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.565835+0000","last_change":"2024-08-29T15:08:25.191304+0000","last_active":"2024-08-29T15:13:39.565835+0000","last_peered":"2024-08-29T15:13:39.565835+0000","last_clean":"2024-08-29T15:13:39.565835+0000","last_became_active":"2024-08-29T15:08:25.191022+0000","last_became_peered":"2024-08-29T15:08:25.191022+0000","last_unstale":"2024-08-29T15:13:39.565835+0000","last_undegraded":"2024-08-29T15:13:39.565835+0000","last_fullsized":"2024-08-29T15:13:39.565835+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:07.608759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"137'14270","reported_seq":14305,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:49.669421+0000","last_change":"2024-08-29T15:10:22.060350+0000","last_active":"2024-08-29T15:13:49.669421+0000","last_peered":"2024-08-29T15:13:49.669421+0000","last_clean":"2024-08-29T15:13:49.669421+0000","last_became_active":"2024-08-29T15:10:22.060063+0000","last_became_peered":"2024-08-29T15:10:22.060063+0000","last_unstale":"2024-08-29T15:13:49.669421+0000","last_undegraded":"2024-08-29T15:13:49.669421+0000","last_fullsized":"2024-08-29T15:13:49.669421+0000","mapping_epoch":111,"log_start":"117'4200","ondisk_log_start":"117'4200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:20:03.508888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172490752,"num_objects":2632,"num_object_clones":0,"num_object_copies":7896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14270,"num_write_kb":540864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816948+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:13:31.816948+0000","last_peered":"2024-08-29T15:13:31.816948+0000","last_clean":"2024-08-29T15:13:31.816948+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:13:31.816948+0000","last_undegraded":"2024-08-29T15:13:31.816948+0000","last_fullsized":"2024-08-29T15:13:31.816948+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:39.565874+0000","last_change":"2024-08-29T15:07:52.142297+0000","last_active":"2024-08-29T15:13:39.565874+0000","last_peered":"2024-08-29T15:13:39.565874+0000","last_clean":"2024-08-29T15:13:39.565874+0000","last_became_active":"2024-08-29T15:07:52.141836+0000","last_became_peered":"2024-08-29T15:07:52.141836+0000","last_unstale":"2024-08-29T15:13:39.565874+0000","last_undegraded":"2024-08-29T15:13:39.565874+0000","last_fullsized":"2024-08-29T15:13:39.565874+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:58:01.221672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"137'14223","reported_seq":14258,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:49.527932+0000","last_change":"2024-08-29T15:10:22.063109+0000","last_active":"2024-08-29T15:13:49.527932+0000","last_peered":"2024-08-29T15:13:49.527932+0000","last_clean":"2024-08-29T15:13:49.527932+0000","last_became_active":"2024-08-29T15:10:22.062773+0000","last_became_peered":"2024-08-29T15:10:22.062773+0000","last_unstale":"2024-08-29T15:13:49.527932+0000","last_undegraded":"2024-08-29T15:13:49.527932+0000","last_fullsized":"2024-08-29T15:13:49.527932+0000","mapping_epoch":111,"log_start":"119'6900","ondisk_log_start":"119'6900","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":7323,"log_dups_size":0,"ondisk_log_size":7323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:01:43.837643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173867008,"num_objects":2653,"num_object_clones":0,"num_object_copies":7959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14223,"num_write_kb":540032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":132,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.907039+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:13:46.907039+0000","last_peered":"2024-08-29T15:13:46.907039+0000","last_clean":"2024-08-29T15:13:46.907039+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:13:46.907039+0000","last_undegraded":"2024-08-29T15:13:46.907039+0000","last_fullsized":"2024-08-29T15:13:46.907039+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896354+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:13:46.896354+0000","last_peered":"2024-08-29T15:13:46.896354+0000","last_clean":"2024-08-29T15:13:46.896354+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:13:46.896354+0000","last_undegraded":"2024-08-29T15:13:46.896354+0000","last_fullsized":"2024-08-29T15:13:46.896354+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"137'14210","reported_seq":14245,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:49.489084+0000","last_change":"2024-08-29T15:10:22.062609+0000","last_active":"2024-08-29T15:13:49.489084+0000","last_peered":"2024-08-29T15:13:49.489084+0000","last_clean":"2024-08-29T15:13:49.489084+0000","last_became_active":"2024-08-29T15:10:22.062277+0000","last_became_peered":"2024-08-29T15:10:22.062277+0000","last_unstale":"2024-08-29T15:13:49.489084+0000","last_undegraded":"2024-08-29T15:13:49.489084+0000","last_fullsized":"2024-08-29T15:13:49.489084+0000","mapping_epoch":111,"log_start":"118'5600","ondisk_log_start":"118'5600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8610,"log_dups_size":0,"ondisk_log_size":8610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:48:06.231404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173670400,"num_objects":2650,"num_object_clones":0,"num_object_copies":7950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14210,"num_write_kb":539520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":431,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.815804+0000","last_change":"2024-08-29T15:07:20.817595+0000","last_active":"2024-08-29T15:13:31.815804+0000","last_peered":"2024-08-29T15:13:31.815804+0000","last_clean":"2024-08-29T15:13:31.815804+0000","last_became_active":"2024-08-29T15:07:20.817130+0000","last_became_peered":"2024-08-29T15:07:20.817130+0000","last_unstale":"2024-08-29T15:13:31.815804+0000","last_undegraded":"2024-08-29T15:13:31.815804+0000","last_fullsized":"2024-08-29T15:13:31.815804+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:28:33.151864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":623,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.814905+0000","last_change":"2024-08-29T15:08:25.185728+0000","last_active":"2024-08-29T15:13:31.814905+0000","last_peered":"2024-08-29T15:13:31.814905+0000","last_clean":"2024-08-29T15:13:31.814905+0000","last_became_active":"2024-08-29T15:08:25.185219+0000","last_became_peered":"2024-08-29T15:08:25.185219+0000","last_unstale":"2024-08-29T15:13:31.814905+0000","last_undegraded":"2024-08-29T15:13:31.814905+0000","last_fullsized":"2024-08-29T15:13:31.814905+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:16:25.013137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.906104+0000","last_change":"2024-08-29T15:08:58.890060+0000","last_active":"2024-08-29T15:13:46.906104+0000","last_peered":"2024-08-29T15:13:46.906104+0000","last_clean":"2024-08-29T15:13:46.906104+0000","last_became_active":"2024-08-29T15:08:58.889714+0000","last_became_peered":"2024-08-29T15:08:58.889714+0000","last_unstale":"2024-08-29T15:13:46.906104+0000","last_undegraded":"2024-08-29T15:13:46.906104+0000","last_fullsized":"2024-08-29T15:13:46.906104+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:50.659662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.818282+0000","last_change":"2024-08-29T14:55:33.429121+0000","last_active":"2024-08-29T15:13:31.818282+0000","last_peered":"2024-08-29T15:13:31.818282+0000","last_clean":"2024-08-29T15:13:31.818282+0000","last_became_active":"2024-08-29T14:55:33.428849+0000","last_became_peered":"2024-08-29T14:55:33.428849+0000","last_unstale":"2024-08-29T15:13:31.818282+0000","last_undegraded":"2024-08-29T15:13:31.818282+0000","last_fullsized":"2024-08-29T15:13:31.818282+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:30.671363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"137'14239","reported_seq":14274,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:49.668064+0000","last_change":"2024-08-29T15:10:22.063647+0000","last_active":"2024-08-29T15:13:49.668064+0000","last_peered":"2024-08-29T15:13:49.668064+0000","last_clean":"2024-08-29T15:13:49.668064+0000","last_became_active":"2024-08-29T15:10:22.063401+0000","last_became_peered":"2024-08-29T15:10:22.063401+0000","last_unstale":"2024-08-29T15:13:49.668064+0000","last_undegraded":"2024-08-29T15:13:49.668064+0000","last_fullsized":"2024-08-29T15:13:49.668064+0000","mapping_epoch":111,"log_start":"117'4200","ondisk_log_start":"117'4200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:14.052876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173080576,"num_objects":2641,"num_object_clones":0,"num_object_copies":7923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14239,"num_write_kb":540160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":828,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816031+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:13:31.816031+0000","last_peered":"2024-08-29T15:13:31.816031+0000","last_clean":"2024-08-29T15:13:31.816031+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:13:31.816031+0000","last_undegraded":"2024-08-29T15:13:31.816031+0000","last_fullsized":"2024-08-29T15:13:31.816031+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895293+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:13:46.895293+0000","last_peered":"2024-08-29T15:13:46.895293+0000","last_clean":"2024-08-29T15:13:46.895293+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:13:46.895293+0000","last_undegraded":"2024-08-29T15:13:46.895293+0000","last_fullsized":"2024-08-29T15:13:46.895293+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896162+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:13:46.896162+0000","last_peered":"2024-08-29T15:13:46.896162+0000","last_clean":"2024-08-29T15:13:46.896162+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:13:46.896162+0000","last_undegraded":"2024-08-29T15:13:46.896162+0000","last_fullsized":"2024-08-29T15:13:46.896162+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"137'14093","reported_seq":14157,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:48.685527+0000","last_change":"2024-08-29T15:10:22.063713+0000","last_active":"2024-08-29T15:13:48.685527+0000","last_peered":"2024-08-29T15:13:48.685527+0000","last_clean":"2024-08-29T15:13:48.685527+0000","last_became_active":"2024-08-29T15:10:22.058812+0000","last_became_peered":"2024-08-29T15:10:22.058812+0000","last_unstale":"2024-08-29T15:13:48.685527+0000","last_undegraded":"2024-08-29T15:13:48.685527+0000","last_fullsized":"2024-08-29T15:13:48.685527+0000","mapping_epoch":111,"log_start":"117'4000","ondisk_log_start":"117'4000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":10093,"log_dups_size":0,"ondisk_log_size":10093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:34:42.633452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175046656,"num_objects":2671,"num_object_clones":0,"num_object_copies":8013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14093,"num_write_kb":536448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":596,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.885921+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:13:46.885921+0000","last_peered":"2024-08-29T15:13:46.885921+0000","last_clean":"2024-08-29T15:13:46.885921+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:13:46.885921+0000","last_undegraded":"2024-08-29T15:13:46.885921+0000","last_fullsized":"2024-08-29T15:13:46.885921+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":112,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.900257+0000","last_change":"2024-08-29T15:08:25.188186+0000","last_active":"2024-08-29T15:13:46.900257+0000","last_peered":"2024-08-29T15:13:46.900257+0000","last_clean":"2024-08-29T15:13:46.900257+0000","last_became_active":"2024-08-29T15:08:25.187830+0000","last_became_peered":"2024-08-29T15:08:25.187830+0000","last_unstale":"2024-08-29T15:13:46.900257+0000","last_undegraded":"2024-08-29T15:13:46.900257+0000","last_fullsized":"2024-08-29T15:13:46.900257+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:00:12.780470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":144,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896160+0000","last_change":"2024-08-29T15:07:52.139273+0000","last_active":"2024-08-29T15:13:46.896160+0000","last_peered":"2024-08-29T15:13:46.896160+0000","last_clean":"2024-08-29T15:13:46.896160+0000","last_became_active":"2024-08-29T15:07:52.138052+0000","last_became_peered":"2024-08-29T15:07:52.138052+0000","last_unstale":"2024-08-29T15:13:46.896160+0000","last_undegraded":"2024-08-29T15:13:46.896160+0000","last_fullsized":"2024-08-29T15:13:46.896160+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:01.705167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"137'14117","reported_seq":14180,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:48.662011+0000","last_change":"2024-08-29T15:10:22.064573+0000","last_active":"2024-08-29T15:13:48.662011+0000","last_peered":"2024-08-29T15:13:48.662011+0000","last_clean":"2024-08-29T15:13:48.662011+0000","last_became_active":"2024-08-29T15:10:22.064260+0000","last_became_peered":"2024-08-29T15:10:22.064260+0000","last_unstale":"2024-08-29T15:13:48.662011+0000","last_undegraded":"2024-08-29T15:13:48.662011+0000","last_fullsized":"2024-08-29T15:13:48.662011+0000","mapping_epoch":111,"log_start":"118'5200","ondisk_log_start":"118'5200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8917,"log_dups_size":0,"ondisk_log_size":8917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:28.889508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176488448,"num_objects":2693,"num_object_clones":0,"num_object_copies":8079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14117,"num_write_kb":537920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":554,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.810691+0000","last_change":"2024-08-29T15:07:52.138323+0000","last_active":"2024-08-29T15:13:31.810691+0000","last_peered":"2024-08-29T15:13:31.810691+0000","last_clean":"2024-08-29T15:13:31.810691+0000","last_became_active":"2024-08-29T15:07:52.137781+0000","last_became_peered":"2024-08-29T15:07:52.137781+0000","last_unstale":"2024-08-29T15:13:31.810691+0000","last_undegraded":"2024-08-29T15:13:31.810691+0000","last_fullsized":"2024-08-29T15:13:31.810691+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:11:27.075260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":579,"num_read_kb":2582,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816902+0000","last_change":"2024-08-29T15:07:20.816485+0000","last_active":"2024-08-29T15:13:31.816902+0000","last_peered":"2024-08-29T15:13:31.816902+0000","last_clean":"2024-08-29T15:13:31.816902+0000","last_became_active":"2024-08-29T15:07:20.816190+0000","last_became_peered":"2024-08-29T15:07:20.816190+0000","last_unstale":"2024-08-29T15:13:31.816902+0000","last_undegraded":"2024-08-29T15:13:31.816902+0000","last_fullsized":"2024-08-29T15:13:31.816902+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:06.731598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895542+0000","last_change":"2024-08-29T15:08:58.883318+0000","last_active":"2024-08-29T15:13:46.895542+0000","last_peered":"2024-08-29T15:13:46.895542+0000","last_clean":"2024-08-29T15:13:46.895542+0000","last_became_active":"2024-08-29T15:08:58.882978+0000","last_became_peered":"2024-08-29T15:08:58.882978+0000","last_unstale":"2024-08-29T15:13:46.895542+0000","last_undegraded":"2024-08-29T15:13:46.895542+0000","last_fullsized":"2024-08-29T15:13:46.895542+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:08.675766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"137'13989","reported_seq":14052,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:48.642035+0000","last_change":"2024-08-29T15:10:22.060106+0000","last_active":"2024-08-29T15:13:48.642035+0000","last_peered":"2024-08-29T15:13:48.642035+0000","last_clean":"2024-08-29T15:13:48.642035+0000","last_became_active":"2024-08-29T15:10:22.059746+0000","last_became_peered":"2024-08-29T15:10:22.059746+0000","last_unstale":"2024-08-29T15:13:48.642035+0000","last_undegraded":"2024-08-29T15:13:48.642035+0000","last_fullsized":"2024-08-29T15:13:48.642035+0000","mapping_epoch":111,"log_start":"118'5100","ondisk_log_start":"118'5100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8889,"log_dups_size":0,"ondisk_log_size":8889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:25:15.955899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172556288,"num_objects":2633,"num_object_clones":0,"num_object_copies":7899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13989,"num_write_kb":531904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.811904+0000","last_change":"2024-08-29T15:08:58.887791+0000","last_active":"2024-08-29T15:13:31.811904+0000","last_peered":"2024-08-29T15:13:31.811904+0000","last_clean":"2024-08-29T15:13:31.811904+0000","last_became_active":"2024-08-29T15:08:58.887472+0000","last_became_peered":"2024-08-29T15:08:58.887472+0000","last_unstale":"2024-08-29T15:13:31.811904+0000","last_undegraded":"2024-08-29T15:13:31.811904+0000","last_fullsized":"2024-08-29T15:13:31.811904+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:35:58.690131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":561,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895734+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:13:46.895734+0000","last_peered":"2024-08-29T15:13:46.895734+0000","last_clean":"2024-08-29T15:13:46.895734+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:13:46.895734+0000","last_undegraded":"2024-08-29T15:13:46.895734+0000","last_fullsized":"2024-08-29T15:13:46.895734+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":99,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.898187+0000","last_change":"2024-08-29T15:07:52.139543+0000","last_active":"2024-08-29T15:13:46.898187+0000","last_peered":"2024-08-29T15:13:46.898187+0000","last_clean":"2024-08-29T15:13:46.898187+0000","last_became_active":"2024-08-29T15:07:52.138499+0000","last_became_peered":"2024-08-29T15:07:52.138499+0000","last_unstale":"2024-08-29T15:13:46.898187+0000","last_undegraded":"2024-08-29T15:13:46.898187+0000","last_fullsized":"2024-08-29T15:13:46.898187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:41:05.588954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"137'13982","reported_seq":14018,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:49.396273+0000","last_change":"2024-08-29T15:10:22.060236+0000","last_active":"2024-08-29T15:13:49.396273+0000","last_peered":"2024-08-29T15:13:49.396273+0000","last_clean":"2024-08-29T15:13:49.396273+0000","last_became_active":"2024-08-29T15:10:22.059894+0000","last_became_peered":"2024-08-29T15:10:22.059894+0000","last_unstale":"2024-08-29T15:13:49.396273+0000","last_undegraded":"2024-08-29T15:13:49.396273+0000","last_fullsized":"2024-08-29T15:13:49.396273+0000","mapping_epoch":111,"log_start":"117'4300","ondisk_log_start":"117'4300","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":9682,"log_dups_size":0,"ondisk_log_size":9682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:54:04.521598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168165376,"num_objects":2566,"num_object_clones":0,"num_object_copies":7698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13982,"num_write_kb":529536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895637+0000","last_change":"2024-08-29T15:08:25.186307+0000","last_active":"2024-08-29T15:13:46.895637+0000","last_peered":"2024-08-29T15:13:46.895637+0000","last_clean":"2024-08-29T15:13:46.895637+0000","last_became_active":"2024-08-29T15:08:25.185788+0000","last_became_peered":"2024-08-29T15:08:25.185788+0000","last_unstale":"2024-08-29T15:13:46.895637+0000","last_undegraded":"2024-08-29T15:13:46.895637+0000","last_fullsized":"2024-08-29T15:13:46.895637+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:58:08.511525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":525,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816287+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:13:31.816287+0000","last_peered":"2024-08-29T15:13:31.816287+0000","last_clean":"2024-08-29T15:13:31.816287+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:13:31.816287+0000","last_undegraded":"2024-08-29T15:13:31.816287+0000","last_fullsized":"2024-08-29T15:13:31.816287+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896810+0000","last_change":"2024-08-29T15:07:20.813592+0000","last_active":"2024-08-29T15:13:46.896810+0000","last_peered":"2024-08-29T15:13:46.896810+0000","last_clean":"2024-08-29T15:13:46.896810+0000","last_became_active":"2024-08-29T15:07:20.812970+0000","last_became_peered":"2024-08-29T15:07:20.812970+0000","last_unstale":"2024-08-29T15:13:46.896810+0000","last_undegraded":"2024-08-29T15:13:46.896810+0000","last_fullsized":"2024-08-29T15:13:46.896810+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:54.850894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"137'13963","reported_seq":13998,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:49.499357+0000","last_change":"2024-08-29T15:10:22.060701+0000","last_active":"2024-08-29T15:13:49.499357+0000","last_peered":"2024-08-29T15:13:49.499357+0000","last_clean":"2024-08-29T15:13:49.499357+0000","last_became_active":"2024-08-29T15:10:22.060185+0000","last_became_peered":"2024-08-29T15:10:22.060185+0000","last_unstale":"2024-08-29T15:13:49.499357+0000","last_undegraded":"2024-08-29T15:13:49.499357+0000","last_fullsized":"2024-08-29T15:13:49.499357+0000","mapping_epoch":111,"log_start":"119'6600","ondisk_log_start":"119'6600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:10:46.794683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170196992,"num_objects":2597,"num_object_clones":0,"num_object_copies":7791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13963,"num_write_kb":529920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":129,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.905667+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:13:46.905667+0000","last_peered":"2024-08-29T15:13:46.905667+0000","last_clean":"2024-08-29T15:13:46.905667+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:13:46.905667+0000","last_undegraded":"2024-08-29T15:13:46.905667+0000","last_fullsized":"2024-08-29T15:13:46.905667+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":652,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895123+0000","last_change":"2024-08-29T15:07:52.139196+0000","last_active":"2024-08-29T15:13:46.895123+0000","last_peered":"2024-08-29T15:13:46.895123+0000","last_clean":"2024-08-29T15:13:46.895123+0000","last_became_active":"2024-08-29T15:07:52.138669+0000","last_became_peered":"2024-08-29T15:07:52.138669+0000","last_unstale":"2024-08-29T15:13:46.895123+0000","last_undegraded":"2024-08-29T15:13:46.895123+0000","last_fullsized":"2024-08-29T15:13:46.895123+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:43:54.949715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":126,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.906844+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:13:46.906844+0000","last_peered":"2024-08-29T15:13:46.906844+0000","last_clean":"2024-08-29T15:13:46.906844+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:13:46.906844+0000","last_undegraded":"2024-08-29T15:13:46.906844+0000","last_fullsized":"2024-08-29T15:13:46.906844+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"137'13861","reported_seq":13925,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:51.495000+0000","last_change":"2024-08-29T15:10:22.058096+0000","last_active":"2024-08-29T15:13:51.495000+0000","last_peered":"2024-08-29T15:13:51.495000+0000","last_clean":"2024-08-29T15:13:51.495000+0000","last_became_active":"2024-08-29T15:10:22.057728+0000","last_became_peered":"2024-08-29T15:10:22.057728+0000","last_unstale":"2024-08-29T15:13:51.495000+0000","last_undegraded":"2024-08-29T15:13:51.495000+0000","last_fullsized":"2024-08-29T15:13:51.495000+0000","mapping_epoch":111,"log_start":"118'5000","ondisk_log_start":"118'5000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:10:21.034680+0000","last_clean_scrub_stamp":"2024-08-29T15:10:21.034680+0000","objects_scrubbed":0,"log_size":8861,"log_dups_size":0,"ondisk_log_size":8861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:10:07.157182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164036608,"num_objects":2503,"num_object_clones":0,"num_object_copies":7509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13861,"num_write_kb":523648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.895895+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:13:46.895895+0000","last_peered":"2024-08-29T15:13:46.895895+0000","last_clean":"2024-08-29T15:13:46.895895+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:13:46.895895+0000","last_undegraded":"2024-08-29T15:13:46.895895+0000","last_fullsized":"2024-08-29T15:13:46.895895+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":366,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.887300+0000","last_change":"2024-08-29T15:08:25.186819+0000","last_active":"2024-08-29T15:13:46.887300+0000","last_peered":"2024-08-29T15:13:46.887300+0000","last_clean":"2024-08-29T15:13:46.887300+0000","last_became_active":"2024-08-29T15:08:25.186401+0000","last_became_peered":"2024-08-29T15:08:25.186401+0000","last_unstale":"2024-08-29T15:13:46.887300+0000","last_undegraded":"2024-08-29T15:13:46.887300+0000","last_fullsized":"2024-08-29T15:13:46.887300+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:45.101824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816717+0000","last_change":"2024-08-29T14:55:33.428058+0000","last_active":"2024-08-29T15:13:31.816717+0000","last_peered":"2024-08-29T15:13:31.816717+0000","last_clean":"2024-08-29T15:13:31.816717+0000","last_became_active":"2024-08-29T14:55:33.427738+0000","last_became_peered":"2024-08-29T14:55:33.427738+0000","last_unstale":"2024-08-29T15:13:31.816717+0000","last_undegraded":"2024-08-29T15:13:31.816717+0000","last_fullsized":"2024-08-29T15:13:31.816717+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:12.995339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":107,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816680+0000","last_change":"2024-08-29T15:08:25.185452+0000","last_active":"2024-08-29T15:13:31.816680+0000","last_peered":"2024-08-29T15:13:31.816680+0000","last_clean":"2024-08-29T15:13:31.816680+0000","last_became_active":"2024-08-29T15:08:25.185095+0000","last_became_peered":"2024-08-29T15:08:25.185095+0000","last_unstale":"2024-08-29T15:13:31.816680+0000","last_undegraded":"2024-08-29T15:13:31.816680+0000","last_fullsized":"2024-08-29T15:13:31.816680+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:19:03.238217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":107,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.817509+0000","last_change":"2024-08-29T14:55:32.413532+0000","last_active":"2024-08-29T15:13:31.817509+0000","last_peered":"2024-08-29T15:13:31.817509+0000","last_clean":"2024-08-29T15:13:31.817509+0000","last_became_active":"2024-08-29T14:55:32.412995+0000","last_became_peered":"2024-08-29T14:55:32.412995+0000","last_unstale":"2024-08-29T15:13:31.817509+0000","last_undegraded":"2024-08-29T15:13:31.817509+0000","last_fullsized":"2024-08-29T15:13:31.817509+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:34:59.219900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.898034+0000","last_change":"2024-08-29T15:08:25.186368+0000","last_active":"2024-08-29T15:13:46.898034+0000","last_peered":"2024-08-29T15:13:46.898034+0000","last_clean":"2024-08-29T15:13:46.898034+0000","last_became_active":"2024-08-29T15:08:25.185912+0000","last_became_peered":"2024-08-29T15:08:25.185912+0000","last_unstale":"2024-08-29T15:13:46.898034+0000","last_undegraded":"2024-08-29T15:13:46.898034+0000","last_fullsized":"2024-08-29T15:13:46.898034+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:23.124995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816091+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:13:31.816091+0000","last_peered":"2024-08-29T15:13:31.816091+0000","last_clean":"2024-08-29T15:13:31.816091+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:13:31.816091+0000","last_undegraded":"2024-08-29T15:13:31.816091+0000","last_fullsized":"2024-08-29T15:13:31.816091+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.814253+0000","last_change":"2024-08-29T15:07:52.136541+0000","last_active":"2024-08-29T15:13:31.814253+0000","last_peered":"2024-08-29T15:13:31.814253+0000","last_clean":"2024-08-29T15:13:31.814253+0000","last_became_active":"2024-08-29T15:07:52.136202+0000","last_became_peered":"2024-08-29T15:07:52.136202+0000","last_unstale":"2024-08-29T15:13:31.814253+0000","last_undegraded":"2024-08-29T15:13:31.814253+0000","last_fullsized":"2024-08-29T15:13:31.814253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:01.747963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.905119+0000","last_change":"2024-08-29T15:08:58.888848+0000","last_active":"2024-08-29T15:13:46.905119+0000","last_peered":"2024-08-29T15:13:46.905119+0000","last_clean":"2024-08-29T15:13:46.905119+0000","last_became_active":"2024-08-29T15:08:58.886702+0000","last_became_peered":"2024-08-29T15:08:58.886702+0000","last_unstale":"2024-08-29T15:13:46.905119+0000","last_undegraded":"2024-08-29T15:13:46.905119+0000","last_fullsized":"2024-08-29T15:13:46.905119+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:56.189439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.897777+0000","last_change":"2024-08-29T15:08:58.887151+0000","last_active":"2024-08-29T15:13:46.897777+0000","last_peered":"2024-08-29T15:13:46.897777+0000","last_clean":"2024-08-29T15:13:46.897777+0000","last_became_active":"2024-08-29T15:08:58.886918+0000","last_became_peered":"2024-08-29T15:08:58.886918+0000","last_unstale":"2024-08-29T15:13:46.897777+0000","last_undegraded":"2024-08-29T15:13:46.897777+0000","last_fullsized":"2024-08-29T15:13:46.897777+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:37.557892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":131,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.906744+0000","last_change":"2024-08-29T15:08:58.889700+0000","last_active":"2024-08-29T15:13:46.906744+0000","last_peered":"2024-08-29T15:13:46.906744+0000","last_clean":"2024-08-29T15:13:46.906744+0000","last_became_active":"2024-08-29T15:08:58.887521+0000","last_became_peered":"2024-08-29T15:08:58.887521+0000","last_unstale":"2024-08-29T15:13:46.906744+0000","last_undegraded":"2024-08-29T15:13:46.906744+0000","last_fullsized":"2024-08-29T15:13:46.906744+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:32.997619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.824625+0000","last_change":"2024-08-29T15:07:52.135092+0000","last_active":"2024-08-29T15:13:31.824625+0000","last_peered":"2024-08-29T15:13:31.824625+0000","last_clean":"2024-08-29T15:13:31.824625+0000","last_became_active":"2024-08-29T15:07:52.134625+0000","last_became_peered":"2024-08-29T15:07:52.134625+0000","last_unstale":"2024-08-29T15:13:31.824625+0000","last_undegraded":"2024-08-29T15:13:31.824625+0000","last_fullsized":"2024-08-29T15:13:31.824625+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:22:35.250433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":147,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.906923+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:13:46.906923+0000","last_peered":"2024-08-29T15:13:46.906923+0000","last_clean":"2024-08-29T15:13:46.906923+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:13:46.906923+0000","last_undegraded":"2024-08-29T15:13:46.906923+0000","last_fullsized":"2024-08-29T15:13:46.906923+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896037+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:13:46.896037+0000","last_peered":"2024-08-29T15:13:46.896037+0000","last_clean":"2024-08-29T15:13:46.896037+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:13:46.896037+0000","last_undegraded":"2024-08-29T15:13:46.896037+0000","last_fullsized":"2024-08-29T15:13:46.896037+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896197+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:13:46.896197+0000","last_peered":"2024-08-29T15:13:46.896197+0000","last_clean":"2024-08-29T15:13:46.896197+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:13:46.896197+0000","last_undegraded":"2024-08-29T15:13:46.896197+0000","last_fullsized":"2024-08-29T15:13:46.896197+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":129,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.905030+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:13:46.905030+0000","last_peered":"2024-08-29T15:13:46.905030+0000","last_clean":"2024-08-29T15:13:46.905030+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:13:46.905030+0000","last_undegraded":"2024-08-29T15:13:46.905030+0000","last_fullsized":"2024-08-29T15:13:46.905030+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.817531+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:13:31.817531+0000","last_peered":"2024-08-29T15:13:31.817531+0000","last_clean":"2024-08-29T15:13:31.817531+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:13:31.817531+0000","last_undegraded":"2024-08-29T15:13:31.817531+0000","last_fullsized":"2024-08-29T15:13:31.817531+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":150,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.889133+0000","last_change":"2024-08-29T15:08:58.884121+0000","last_active":"2024-08-29T15:13:46.889133+0000","last_peered":"2024-08-29T15:13:46.889133+0000","last_clean":"2024-08-29T15:13:46.889133+0000","last_became_active":"2024-08-29T15:08:58.883669+0000","last_became_peered":"2024-08-29T15:08:58.883669+0000","last_unstale":"2024-08-29T15:13:46.889133+0000","last_undegraded":"2024-08-29T15:13:46.889133+0000","last_fullsized":"2024-08-29T15:13:46.889133+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:33:32.165590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.896717+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:13:46.896717+0000","last_peered":"2024-08-29T15:13:46.896717+0000","last_clean":"2024-08-29T15:13:46.896717+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:13:46.896717+0000","last_undegraded":"2024-08-29T15:13:46.896717+0000","last_fullsized":"2024-08-29T15:13:46.896717+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":113,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816593+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:13:31.816593+0000","last_peered":"2024-08-29T15:13:31.816593+0000","last_clean":"2024-08-29T15:13:31.816593+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:13:31.816593+0000","last_undegraded":"2024-08-29T15:13:31.816593+0000","last_fullsized":"2024-08-29T15:13:31.816593+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.815253+0000","last_change":"2024-08-29T15:07:52.134597+0000","last_active":"2024-08-29T15:13:31.815253+0000","last_peered":"2024-08-29T15:13:31.815253+0000","last_clean":"2024-08-29T15:13:31.815253+0000","last_became_active":"2024-08-29T15:07:52.133893+0000","last_became_peered":"2024-08-29T15:07:52.133893+0000","last_unstale":"2024-08-29T15:13:31.815253+0000","last_undegraded":"2024-08-29T15:13:31.815253+0000","last_fullsized":"2024-08-29T15:13:31.815253+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:14.240866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":113,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.817381+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:13:31.817381+0000","last_peered":"2024-08-29T15:13:31.817381+0000","last_clean":"2024-08-29T15:13:31.817381+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:13:31.817381+0000","last_undegraded":"2024-08-29T15:13:31.817381+0000","last_fullsized":"2024-08-29T15:13:31.817381+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.811839+0000","last_change":"2024-08-29T14:55:32.408846+0000","last_active":"2024-08-29T15:13:31.811839+0000","last_peered":"2024-08-29T15:13:31.811839+0000","last_clean":"2024-08-29T15:13:31.811839+0000","last_became_active":"2024-08-29T14:55:32.408585+0000","last_became_peered":"2024-08-29T14:55:32.408585+0000","last_unstale":"2024-08-29T15:13:31.811839+0000","last_undegraded":"2024-08-29T15:13:31.811839+0000","last_fullsized":"2024-08-29T15:13:31.811839+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:05.868638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":150,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.905455+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:13:46.905455+0000","last_peered":"2024-08-29T15:13:46.905455+0000","last_clean":"2024-08-29T15:13:46.905455+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:13:46.905455+0000","last_undegraded":"2024-08-29T15:13:46.905455+0000","last_fullsized":"2024-08-29T15:13:46.905455+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":138,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.905385+0000","last_change":"2024-08-29T15:08:58.888653+0000","last_active":"2024-08-29T15:13:46.905385+0000","last_peered":"2024-08-29T15:13:46.905385+0000","last_clean":"2024-08-29T15:13:46.905385+0000","last_became_active":"2024-08-29T15:08:58.888133+0000","last_became_peered":"2024-08-29T15:08:58.888133+0000","last_unstale":"2024-08-29T15:13:46.905385+0000","last_undegraded":"2024-08-29T15:13:46.905385+0000","last_fullsized":"2024-08-29T15:13:46.905385+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:12:24.144974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-08-29T15:13:31.816582+0000","last_change":"2024-08-29T15:07:52.135649+0000","last_active":"2024-08-29T15:13:31.816582+0000","last_peered":"2024-08-29T15:13:31.816582+0000","last_clean":"2024-08-29T15:13:31.816582+0000","last_became_active":"2024-08-29T15:07:52.135327+0000","last_became_peered":"2024-08-29T15:07:52.135327+0000","last_unstale":"2024-08-29T15:13:31.816582+0000","last_undegraded":"2024-08-29T15:13:31.816582+0000","last_fullsized":"2024-08-29T15:13:31.816582+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:05.402462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-08-29T15:13:46.897678+0000","last_change":"2024-08-29T15:08:25.187422+0000","last_active":"2024-08-29T15:13:46.897678+0000","last_peered":"2024-08-29T15:13:46.897678+0000","last_clean":"2024-08-29T15:13:46.897678+0000","last_became_active":"2024-08-29T15:08:25.187062+0000","last_became_peered":"2024-08-29T15:08:25.187062+0000","last_unstale":"2024-08-29T15:13:46.897678+0000","last_undegraded":"2024-08-29T15:13:46.897678+0000","last_fullsized":"2024-08-29T15:13:46.897678+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:18:20.745348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2766471192,"num_objects":42214,"num_object_clones":0,"num_object_copies":126642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42214,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":225286,"num_write_kb":8559937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8266526720,"data_stored":8266514504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146886,"ondisk_log_size":146886,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5998,"num_read_kb":13011,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234230,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":950292,"kb_used_data":851080,"kb_used_omap":1,"kb_used_meta":99198,"kb_avail":92794860,"statfs":{"total":95995035648,"available":95021936640,"internally_reserved":0,"allocated":871505920,"data_stored":870781543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":101579211},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495866,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":747328,"kb_used_data":680056,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":92997824,"statfs":{"total":95995035648,"available":95229771776,"internally_reserved":0,"allocated":696377344,"data_stored":695668927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":68811207},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757503,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1673912,"kb_used_data":1508272,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":92071240,"statfs":{"total":95995035648,"available":94280949760,"internally_reserved":0,"allocated":1544470528,"data_stored":1543765551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":169605574},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019140,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1519872,"kb_used_data":1380152,"kb_used_omap":1,"kb_used_meta":139390,"kb_avail":92225280,"statfs":{"total":95995035648,"available":94438686720,"internally_reserved":0,"allocated":1413275648,"data_stored":1412566744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":142735814},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435325,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":553264,"kb_used_data":485544,"kb_used_omap":4,"kb_used_meta":67707,"kb_avail":93191888,"statfs":{"total":95995035648,"available":95428493312,"internally_reserved":0,"allocated":497197056,"data_stored":496500009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":69332233},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598851,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1150512,"kb_used_data":1042468,"kb_used_omap":7,"kb_used_meta":108024,"kb_avail":92594640,"statfs":{"total":95995035648,"available":94816911360,"internally_reserved":0,"allocated":1067487232,"data_stored":1066793581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":110616911},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,0,7,10,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762378,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":934896,"kb_used_data":835048,"kb_used_omap":7,"kb_used_meta":99704,"kb_avail":92810256,"statfs":{"total":95995035648,"available":95037702144,"internally_reserved":0,"allocated":855089152,"data_stored":854392034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":102097250},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925904,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1544948,"kb_used_data":1396648,"kb_used_omap":7,"kb_used_meta":148280,"kb_avail":92200204,"statfs":{"total":95995035648,"available":94413008896,"internally_reserved":0,"allocated":1430167552,"data_stored":1429470122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":151838863},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1401683968,"data_stored":1401683968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":846397440,"data_stored":846397440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1050939392,"data_stored":1050935320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":495124480,"data_stored":495124480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1387397120,"data_stored":1387397120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1539248128,"data_stored":1539244056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":687210496,"data_stored":687210496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":858525696,"data_stored":858521624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:13:55.003 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-29T15:13:55.003 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:13:55.004 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:13:55.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd out 5 2024-08-29T15:13:55.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:55.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:55.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:55.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:54 smithi171 ceph-mon[90031]: from='client.34839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:13:55.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:55.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:55.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:55.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:55.403 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:55.419 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:55.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:55.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:55.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:55.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:55.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:55.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:55.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:55.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:55.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:55.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:55.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:55.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:55.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:55.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:55.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:56.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:56.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:56.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:56.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:56.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:56.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:56.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:55 smithi171 ceph-mon[90031]: pgmap v423: 89 pgs: 89 active+clean; 2.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-29T15:13:56.311 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:55 smithi027 ceph-mon[98970]: pgmap v423: 89 pgs: 89 active+clean; 2.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-29T15:13:56.311 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:55 smithi027 ceph-mon[93921]: pgmap v423: 89 pgs: 89 active+clean; 2.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2024-08-29T15:13:56.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:56.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:13:56.383 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:56.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:56.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:56.530 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:56.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:56.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:56.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:56.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:56.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:56.769 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:56.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:56.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:56.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:56.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:56.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:57.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:57.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:57.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:57.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:57.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:57.223 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:57.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:57.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:57.359 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:57.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:57.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:57.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:57.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:57.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:57.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:57.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:57.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:57.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:57.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:57.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:57.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:57.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:57.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:57.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:58.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:58.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:57 smithi027 ceph-mon[98970]: pgmap v424: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-08-29T15:13:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:57 smithi027 ceph-mon[93921]: pgmap v424: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-08-29T15:13:58.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:58.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:58.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:58.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:58.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:58.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:57 smithi171 ceph-mon[90031]: pgmap v424: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-08-29T15:13:58.274 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:58.287 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:13:58.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:58.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:58.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:58.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:13:58.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:58.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:13:58.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:58.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:58.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:58.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:58.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:13:58.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:58.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:13:58.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:58.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:58.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:59.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:13:59.054 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:59.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:13:59.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:13:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:13:59] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-29T15:13:59.127 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:13:59.127 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:13:59.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:13:59.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:59.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:59.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:13:59.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:13:59.316 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:59.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:13:59.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:59.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:59.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:59.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:13:59.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:59.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:59.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:13:59.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:13:59.737 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:13:59.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:59.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:13:59.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:13:59.879 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:13:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:13:59.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:13:59.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:00.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:00.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:59 smithi171 ceph-mon[90031]: pgmap v425: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-08-29T15:14:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:13:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:59 smithi027 ceph-mon[98970]: pgmap v425: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-08-29T15:14:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:13:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:59 smithi027 ceph-mon[93921]: pgmap v425: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-08-29T15:14:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:13:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:00.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:00.141 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:00.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:00.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:00.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:00.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:00.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:00.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:00.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:00.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:00.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:00.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:00.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:00.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:00.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:00.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:00.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:01.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:01.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:01.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:01.152 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:14:01.152 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:14:01.188 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:14:01.188 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:14:01.231 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:14:01.231 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:14:01.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:01.265 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:14:01.266 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:14:01.267 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:01.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:01.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:01.302 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:14:01.303 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:14:01.340 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:14:01.340 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:14:01.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:01.375 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:14:01.375 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:14:01.400 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:01.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:01.417 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:14:01.417 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:14:01.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:01.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:01.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:01.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:01.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:01.673 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:01.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:01.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:01.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:01.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:01.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:01.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:01.949 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:01.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:01.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:01 smithi027 ceph-mon[98970]: pgmap v426: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-29T15:14:01.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3486912646' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:01.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:01 smithi027 ceph-mon[93921]: pgmap v426: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-29T15:14:01.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3486912646' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:01 smithi171 ceph-mon[90031]: pgmap v426: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-29T15:14:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3486912646' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:02.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:02.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:02.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:02.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:02.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:02.226 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:02.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:02.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:02.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:02.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:02.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:02.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:02.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:02.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:02.647 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:02.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:02.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:02.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:02.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:02.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:02.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:02.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:14:02.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:02.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:02.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:03.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:03.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:03.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:03.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:03.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:03.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:03.175 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:03.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:03.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:03.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:03.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:03.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:03.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:03.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:03.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:03.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:03.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:03.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:03.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:03.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:03.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:03.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:03.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:03.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:03.801 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:03 smithi027 ceph-mon[93921]: pgmap v427: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-08-29T15:14:03.801 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:14:03.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:03 smithi027 ceph-mon[98970]: pgmap v427: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-08-29T15:14:03.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:14:03.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:03.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:03.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:03.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:03.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:03 smithi171 ceph-mon[90031]: pgmap v427: 89 pgs: 89 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2024-08-29T15:14:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:14:04.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:04.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:04.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:04.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:04.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:04.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:04.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:04.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:04.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:04.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:04.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:04.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:04.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:04.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:04.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:04.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:04.726 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:14:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:04.745 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:04.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:04.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:04.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:04.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:04.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:05.011 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:05.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4082767789' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:14:05.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4082767789' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:14:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4082767789' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:14:05.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:05.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:05.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:05.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:05.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:05.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:05.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.454 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:05.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:05.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:05.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:05.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:05.636 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:05.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:05.734 INFO:teuthology.orchestra.run.smithi027.stderr:marked out osd.5. 2024-08-29T15:14:05.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:05.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:05.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:05.867 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:05.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[98970]: pgmap v428: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-08-29T15:14:05.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:05.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4082767789' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:14:05.880 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[98970]: osdmap e138: 8 total, 8 up, 8 in 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2497768407' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[98970]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:14:05.615+0000 7f3f0d183640 -1 mon.a@0(leader).osd e138 definitely_dead 0 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[93921]: pgmap v428: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4082767789' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[93921]: osdmap e138: 8 total, 8 up, 8 in 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2497768407' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:14:05.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:05 smithi027 ceph-mon[93921]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-29T15:14:05.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:05.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:05.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:05 smithi171 ceph-mon[90031]: pgmap v428: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-08-29T15:14:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:05 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4082767789' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:14:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:05 smithi171 ceph-mon[90031]: osdmap e138: 8 total, 8 up, 8 in 2024-08-29T15:14:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2497768407' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:14:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:14:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:05 smithi171 ceph-mon[90031]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-29T15:14:06.093 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:06.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:06.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:06.216 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:06.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:06.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:06.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:06.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:06.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:06.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.488 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:06.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:06.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:06.612 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:06.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:14:06.713 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:06.736 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:06.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:06.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:06.858 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:06.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:06.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:06.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:06.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:06 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-29T15:14:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:06 smithi171 ceph-mon[90031]: osdmap e139: 8 total, 8 up, 7 in 2024-08-29T15:14:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:06 smithi171 ceph-mon[90031]: osdmap e140: 8 total, 8 up, 7 in 2024-08-29T15:14:07.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:06 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-29T15:14:07.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:06 smithi027 ceph-mon[93921]: osdmap e139: 8 total, 8 up, 7 in 2024-08-29T15:14:07.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:06 smithi027 ceph-mon[93921]: osdmap e140: 8 total, 8 up, 7 in 2024-08-29T15:14:07.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:06 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-29T15:14:07.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:06 smithi027 ceph-mon[98970]: osdmap e139: 8 total, 8 up, 7 in 2024-08-29T15:14:07.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:06 smithi027 ceph-mon[98970]: osdmap e140: 8 total, 8 up, 7 in 2024-08-29T15:14:07.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:07.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:07.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:07.207 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:07.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:07.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:07.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:07.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:07.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:07.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:07.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:07.475 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:07.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:07.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:07.576 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:07.599 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:07.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:07.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:14:07.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:07.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:07.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:07.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:07.907 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:07.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:07.919 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:07 smithi027 ceph-mon[93921]: pgmap v431: 89 pgs: 89 active+clean; 1.8 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-29T15:14:07.919 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:07 smithi027 ceph-mon[93921]: osdmap e141: 8 total, 8 up, 7 in 2024-08-29T15:14:07.920 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:07 smithi027 ceph-mon[98970]: pgmap v431: 89 pgs: 89 active+clean; 1.8 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-29T15:14:07.920 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:07 smithi027 ceph-mon[98970]: osdmap e141: 8 total, 8 up, 7 in 2024-08-29T15:14:08.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:08.014 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:07 smithi171 ceph-mon[90031]: pgmap v431: 89 pgs: 89 active+clean; 1.8 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-29T15:14:08.014 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:07 smithi171 ceph-mon[90031]: osdmap e141: 8 total, 8 up, 7 in 2024-08-29T15:14:08.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:08.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:08.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:08.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:08.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:08.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:08.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:08.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:08.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:08.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:08.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:08.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:08.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:08.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:08.614 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:08.619 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:08.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:08.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:08.667 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:14:08.667 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:14:08.667 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:14:08.667 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:14:08.667 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:14:08.732 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:08.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:08.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:08.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:08.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:08.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:08.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:09.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 138'14847 2024-08-29T15:14:09.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 138'15067 2024-08-29T15:14:09.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 138'15074 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 138'15035 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 138'14900 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.f starting backfill to osd.2 from (0'0,0'0] MAX to 138'15143 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 138'15000 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 138'14965 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 138'15129 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 138'14847 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 138'15067 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 138'15074 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 138'15035 2024-08-29T15:14:09.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 138'14900 2024-08-29T15:14:09.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.f starting backfill to osd.2 from (0'0,0'0] MAX to 138'15143 2024-08-29T15:14:09.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 138'15000 2024-08-29T15:14:09.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 138'14965 2024-08-29T15:14:09.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 138'15129 2024-08-29T15:14:09.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:09.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:08 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:09.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 138'14847 2024-08-29T15:14:09.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 138'15067 2024-08-29T15:14:09.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 138'15074 2024-08-29T15:14:09.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 138'15035 2024-08-29T15:14:09.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 138'14900 2024-08-29T15:14:09.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.f starting backfill to osd.2 from (0'0,0'0] MAX to 138'15143 2024-08-29T15:14:09.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 138'15000 2024-08-29T15:14:09.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 138'14965 2024-08-29T15:14:09.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 138'15129 2024-08-29T15:14:09.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:09.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:08 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:09.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:09.072 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:09.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:09.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:09.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:09.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:09.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:09.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:09.290 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:14:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:14:09] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-29T15:14:09.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:09.365 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:09.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:09.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:09.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:09.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:09.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:09.660 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:09.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:09.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:09.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:09.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:09.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:09.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:09.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:09.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:09 smithi171 ceph-mon[90031]: pgmap v434: 89 pgs: 3 peering, 86 active+clean; 1.8 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 570 op/s 2024-08-29T15:14:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:09 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:14:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:09 smithi171 ceph-mon[90031]: osdmap e142: 8 total, 8 up, 7 in 2024-08-29T15:14:10.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:10.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:09 smithi027 ceph-mon[98970]: pgmap v434: 89 pgs: 3 peering, 86 active+clean; 1.8 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 570 op/s 2024-08-29T15:14:10.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:09 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:14:10.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:09 smithi027 ceph-mon[98970]: osdmap e142: 8 total, 8 up, 7 in 2024-08-29T15:14:10.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:09 smithi027 ceph-mon[93921]: pgmap v434: 89 pgs: 3 peering, 86 active+clean; 1.8 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 570 op/s 2024-08-29T15:14:10.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:09 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:14:10.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:09 smithi027 ceph-mon[93921]: osdmap e142: 8 total, 8 up, 7 in 2024-08-29T15:14:10.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:10.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:10.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:14:10.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:10.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:10.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:10.371 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:10.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:10.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:10.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:10.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:10.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:10.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:10.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:10.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:10.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:10.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:10.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:10.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:10.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:10.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:10.862 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:10 smithi027 ceph-mon[98970]: osdmap e143: 8 total, 8 up, 7 in 2024-08-29T15:14:10.862 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:10 smithi027 ceph-mon[93921]: osdmap e143: 8 total, 8 up, 7 in 2024-08-29T15:14:10.925 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:10.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:10.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:11.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:11.100 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:11.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:11.201 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:11.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:11.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:10 smithi171 ceph-mon[90031]: osdmap e143: 8 total, 8 up, 7 in 2024-08-29T15:14:11.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:11.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:11.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:11.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:11.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:11.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:11.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:11.601 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:11.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:11.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:11.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:11.776 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:11.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:11.878 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:11.923 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:11.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:12.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:12.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:12.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:11 smithi027 ceph-mon[98970]: pgmap v437: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 4 peering, 76 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 14692/82227 objects misplaced (17.868%); 9.5 MiB/s, 158 objects/s recovering 2024-08-29T15:14:12.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:11 smithi027 ceph-mon[93921]: pgmap v437: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 4 peering, 76 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 14692/82227 objects misplaced (17.868%); 9.5 MiB/s, 158 objects/s recovering 2024-08-29T15:14:12.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:12.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:12.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:11 smithi171 ceph-mon[90031]: pgmap v437: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 4 peering, 76 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 14692/82227 objects misplaced (17.868%); 9.5 MiB/s, 158 objects/s recovering 2024-08-29T15:14:12.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:12.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:12.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:12.455 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:12.493 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:12.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:12.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:12.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:12.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:12.740 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:12.772 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:12.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:12.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:12.895 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:12.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:12.962 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:14:12.962 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:14:12.963 INFO:tasks.thrashosds.thrasher:Adding osd 5 2024-08-29T15:14:12.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd in 5 2024-08-29T15:14:12.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:12 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:14:12.989 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:12 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:14:12.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:13.024 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:13.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:13.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:13.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:13.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:13.188 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:14:13.188 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:14:13.226 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:14:13.226 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:14:13.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:13.262 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:14:13.262 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:14:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:12 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:14:13.297 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:14:13.297 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:14:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:13.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:13.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:13.336 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:14:13.336 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:14:13.377 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:14:13.377 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:14:13.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:13.412 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:14:13.412 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:14:13.445 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:14:13.445 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:14:13.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:13.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:13.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:13.529 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:13.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:13.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:13.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:13.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:13.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:13.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:13.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:13.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:13.867 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:14:13.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:13.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:14.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:14.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:14.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:14.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:14.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:14.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:14.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:13 smithi171 ceph-mon[90031]: pgmap v438: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 peering, 79 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 14692/81738 objects misplaced (17.975%); 6.6 MiB/s, 111 objects/s recovering 2024-08-29T15:14:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3032863893' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:14:14.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:14:14.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2453410494' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:14.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:14.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[98970]: pgmap v438: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 peering, 79 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 14692/81738 objects misplaced (17.975%); 6.6 MiB/s, 111 objects/s recovering 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3032863893' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2453410494' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[93921]: pgmap v438: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 1 peering, 79 active+clean; 1.7 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 14692/81738 objects misplaced (17.975%); 6.6 MiB/s, 111 objects/s recovering 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3032863893' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:14:14.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:14:14.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2453410494' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:14.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:14.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:14.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:14.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:14.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:14.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:14.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:14.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:14.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:14.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:14.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:14.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:14.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:14.756 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:14.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:14.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:14.858 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:14.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:14.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:14.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:14.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:15.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:15.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:14 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:14 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:14:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:14 smithi027 ceph-mon[98970]: osdmap e144: 8 total, 8 up, 7 in 2024-08-29T15:14:15.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:14 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:15.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:14 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:14:15.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:14 smithi027 ceph-mon[93921]: osdmap e144: 8 total, 8 up, 7 in 2024-08-29T15:14:15.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:15.162 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:15.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:15.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:15.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:14 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:15.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:14 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:14:15.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:14 smithi171 ceph-mon[90031]: osdmap e144: 8 total, 8 up, 7 in 2024-08-29T15:14:15.306 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:15.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:15.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:15.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:15.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:15.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:15.706 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:15.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:15.846 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:15.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:15.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:15 smithi027 ceph-mon[93921]: pgmap v440: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.6 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 13372/77178 objects misplaced (17.326%); 11 MiB/s, 188 objects/s recovering 2024-08-29T15:14:15.899 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:15 smithi027 ceph-mon[98970]: pgmap v440: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.6 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 13372/77178 objects misplaced (17.326%); 11 MiB/s, 188 objects/s recovering 2024-08-29T15:14:15.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:15.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:15.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:16.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:16.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:16.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:16.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:16.186 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:16.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:15 smithi171 ceph-mon[90031]: pgmap v440: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.6 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 13372/77178 objects misplaced (17.326%); 11 MiB/s, 188 objects/s recovering 2024-08-29T15:14:16.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:16.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:16.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:16.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:14:16.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:16.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:16.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:16.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:16.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:16.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:16.581 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:16.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:16.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:16.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:16.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:16.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:16.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:16.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:16.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:16.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:16.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:17.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:17.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:17.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:17.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:17.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:17.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:17.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:17.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:17.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:17.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:17.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:17.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:17.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:17.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:17.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:17.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:17.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:14:17.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:17.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:17.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:17.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:17.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:17.866 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:17 smithi027 ceph-mon[93921]: pgmap v441: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.5 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 11022/71499 objects misplaced (15.416%); 20 MiB/s, 321 objects/s recovering 2024-08-29T15:14:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:17.880 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:17.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:17.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:18.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:18.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:18.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:18.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:18.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:18.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:18.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:18.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:17 smithi171 ceph-mon[90031]: pgmap v441: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.5 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 11022/71499 objects misplaced (15.416%); 20 MiB/s, 321 objects/s recovering 2024-08-29T15:14:18.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:17 smithi027 ceph-mon[98970]: pgmap v441: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.5 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 708 op/s; 11022/71499 objects misplaced (15.416%); 20 MiB/s, 321 objects/s recovering 2024-08-29T15:14:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:18.325 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:18.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:18.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:18.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:18.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:18.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:18.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:18.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:18.641 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:14:18.641 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:14:18.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:18.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:18.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:18.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:18.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:18.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:18.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:18.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:18.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:19.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:19.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:19.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:19.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:19.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:19.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:19.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:19.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:19.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:19.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:14:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:14:19] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.51.0" 2024-08-29T15:14:19.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:19.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:19.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:19.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:19.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:19.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:19.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:19.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:19.706 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:19.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:19.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:19.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:19.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:19.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:19.871 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:19.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:19.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:19.999 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:20.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:20.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:20.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:14:20.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:19 smithi027 ceph-mon[98970]: pgmap v442: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.4 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 11022/70773 objects misplaced (15.574%); 15 MiB/s, 246 objects/s recovering 2024-08-29T15:14:20.101 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:19 smithi027 ceph-mon[93921]: pgmap v442: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.4 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 11022/70773 objects misplaced (15.574%); 15 MiB/s, 246 objects/s recovering 2024-08-29T15:14:20.101 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:20.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:20.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:20.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:20.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:19 smithi171 ceph-mon[90031]: pgmap v442: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.4 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 11022/70773 objects misplaced (15.574%); 15 MiB/s, 246 objects/s recovering 2024-08-29T15:14:20.290 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:20.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:20.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:20.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:20.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:20.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:20.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:20.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:20.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:20.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:20.731 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:20.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:20.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:20.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:20.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:20.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:20.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:21.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:21.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:21.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:21.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:21.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:21.216 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:21.222 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:21.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:21.309 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:21.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:21.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:21.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:21.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:21.549 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:21.601 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:21.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:21.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:21.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:21.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:21.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:21.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:21.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:21.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:21.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:21.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:21 smithi027 ceph-mon[93921]: pgmap v443: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.2 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 7477/60030 objects misplaced (12.455%); 23 MiB/s, 374 objects/s recovering 2024-08-29T15:14:21.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:21 smithi027 ceph-mon[93921]: osdmap e145: 8 total, 8 up, 7 in 2024-08-29T15:14:21.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:21 smithi027 ceph-mon[98970]: pgmap v443: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.2 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 7477/60030 objects misplaced (12.455%); 23 MiB/s, 374 objects/s recovering 2024-08-29T15:14:21.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:21 smithi027 ceph-mon[98970]: osdmap e145: 8 total, 8 up, 7 in 2024-08-29T15:14:21.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:22.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:22.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:22.115 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:22.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:22.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:22.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:22.263 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:21 smithi171 ceph-mon[90031]: pgmap v443: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.2 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 739 op/s; 7477/60030 objects misplaced (12.455%); 23 MiB/s, 374 objects/s recovering 2024-08-29T15:14:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:21 smithi171 ceph-mon[90031]: osdmap e145: 8 total, 8 up, 7 in 2024-08-29T15:14:22.365 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:22.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:22.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:22.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:22.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:22.637 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:22.739 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:22.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:22.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:22.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:22.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:22.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:23.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:23.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:23.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:23.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:23.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:23.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:23.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:23.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:23.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:23.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:23.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:23.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:23.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:23.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:23.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:23.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:23.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:23.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:23.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[98970]: pgmap v445: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.2 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 7477/59370 objects misplaced (12.594%); 24 MiB/s, 379 objects/s recovering 2024-08-29T15:14:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[98970]: osdmap e146: 8 total, 8 up, 7 in 2024-08-29T15:14:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1878912205' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:14:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:14:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:14:23.506+0000 7f3f0d183640 -1 mon.a@0(leader).osd e146 definitely_dead 0 2024-08-29T15:14:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[93921]: pgmap v445: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.2 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 7477/59370 objects misplaced (12.594%); 24 MiB/s, 379 objects/s recovering 2024-08-29T15:14:23.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[93921]: osdmap e146: 8 total, 8 up, 7 in 2024-08-29T15:14:23.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1878912205' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:14:23.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:23 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:14:23.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:23.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:23.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:24.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:23 smithi171 ceph-mon[90031]: pgmap v445: 89 pgs: 5 active+remapped+backfill_wait, 4 active+remapped+backfilling, 80 active+clean; 1.2 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 693 op/s; 7477/59370 objects misplaced (12.594%); 24 MiB/s, 379 objects/s recovering 2024-08-29T15:14:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:23 smithi171 ceph-mon[90031]: osdmap e146: 8 total, 8 up, 7 in 2024-08-29T15:14:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1878912205' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:14:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:14:24.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:24.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:24.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:24.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:24.199 INFO:teuthology.orchestra.run.smithi027.stderr:marked in osd.5. 2024-08-29T15:14:24.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:24.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:24.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:24.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:24.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:24.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:24.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:24.534 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:24.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:24.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:24.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:24.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:24.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:24.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:24.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:24.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:24.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:24.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:24 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-29T15:14:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:24 smithi171 ceph-mon[90031]: osdmap e147: 8 total, 8 up, 8 in 2024-08-29T15:14:25.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:25.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:24 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-29T15:14:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:24 smithi027 ceph-mon[93921]: osdmap e147: 8 total, 8 up, 8 in 2024-08-29T15:14:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:24 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-29T15:14:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:24 smithi027 ceph-mon[98970]: osdmap e147: 8 total, 8 up, 8 in 2024-08-29T15:14:25.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:25.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:25.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:25.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:25.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:25.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:25.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:25.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:25.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:25.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:25.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:25.664 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:25.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:25.791 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:25.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:25.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:25.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:25.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:25 smithi171 ceph-mon[90031]: pgmap v448: 89 pgs: 1 remapped+peering, 6 peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 77 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 3416/56079 objects misplaced (6.091%); 31 MiB/s, 500 objects/s recovering 2024-08-29T15:14:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:25 smithi171 ceph-mon[90031]: osdmap e148: 8 total, 8 up, 8 in 2024-08-29T15:14:26.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:26.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:26.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:26.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:25 smithi027 ceph-mon[93921]: pgmap v448: 89 pgs: 1 remapped+peering, 6 peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 77 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 3416/56079 objects misplaced (6.091%); 31 MiB/s, 500 objects/s recovering 2024-08-29T15:14:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:25 smithi027 ceph-mon[93921]: osdmap e148: 8 total, 8 up, 8 in 2024-08-29T15:14:26.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:25 smithi027 ceph-mon[98970]: pgmap v448: 89 pgs: 1 remapped+peering, 6 peering, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 77 active+clean; 1.1 GiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 3416/56079 objects misplaced (6.091%); 31 MiB/s, 500 objects/s recovering 2024-08-29T15:14:26.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:25 smithi027 ceph-mon[98970]: osdmap e148: 8 total, 8 up, 8 in 2024-08-29T15:14:26.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:26.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:26.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:26.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:26.322 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:26.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:26.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:26.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:26.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:26.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:26.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:26.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:26.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:26.723 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:26.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:26.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:26.858 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:26.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:26.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:26.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:27.005 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:27.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[98970]: osdmap e149: 8 total, 8 up, 8 in 2024-08-29T15:14:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[98970]: 5.7 starting backfill to osd.5 from (121'8412,144'15773] MIN to 146'15795 2024-08-29T15:14:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[98970]: 5.3 starting backfill to osd.5 from (121'8183,144'15555] MIN to 146'15565 2024-08-29T15:14:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[98970]: osdmap e150: 8 total, 8 up, 8 in 2024-08-29T15:14:27.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[93921]: osdmap e149: 8 total, 8 up, 8 in 2024-08-29T15:14:27.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[93921]: 5.7 starting backfill to osd.5 from (121'8412,144'15773] MIN to 146'15795 2024-08-29T15:14:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[93921]: 5.3 starting backfill to osd.5 from (121'8183,144'15555] MIN to 146'15565 2024-08-29T15:14:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:26 smithi027 ceph-mon[93921]: osdmap e150: 8 total, 8 up, 8 in 2024-08-29T15:14:27.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:27.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:27.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:27.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:27.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:26 smithi171 ceph-mon[90031]: osdmap e149: 8 total, 8 up, 8 in 2024-08-29T15:14:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:26 smithi171 ceph-mon[90031]: 5.7 starting backfill to osd.5 from (121'8412,144'15773] MIN to 146'15795 2024-08-29T15:14:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:26 smithi171 ceph-mon[90031]: 5.3 starting backfill to osd.5 from (121'8183,144'15555] MIN to 146'15565 2024-08-29T15:14:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:26 smithi171 ceph-mon[90031]: osdmap e150: 8 total, 8 up, 8 in 2024-08-29T15:14:27.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:27.391 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:27.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:27.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:27.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:27.658 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:27.760 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:27.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:27.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:27.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:27.910 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:27.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:28.012 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:28.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:28.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:28.041 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:27 smithi027 ceph-mon[93921]: pgmap v451: 89 pgs: 1 activating+degraded, 1 active, 2 active+recovery_wait+degraded, 3 remapped+peering, 12 peering, 1 active+remapped+backfilling, 69 active+clean; 1.1 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 10/53661 objects degraded (0.019%); 106/53661 objects misplaced (0.198%); 40 MiB/s, 642 objects/s recovering 2024-08-29T15:14:28.041 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:27 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 10/53661 objects degraded (0.019%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:14:28.041 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:27 smithi027 ceph-mon[93921]: osdmap e151: 8 total, 8 up, 8 in 2024-08-29T15:14:28.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:27 smithi027 ceph-mon[98970]: pgmap v451: 89 pgs: 1 activating+degraded, 1 active, 2 active+recovery_wait+degraded, 3 remapped+peering, 12 peering, 1 active+remapped+backfilling, 69 active+clean; 1.1 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 10/53661 objects degraded (0.019%); 106/53661 objects misplaced (0.198%); 40 MiB/s, 642 objects/s recovering 2024-08-29T15:14:28.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:27 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 10/53661 objects degraded (0.019%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:14:28.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:27 smithi027 ceph-mon[98970]: osdmap e151: 8 total, 8 up, 8 in 2024-08-29T15:14:28.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:28.160 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:28.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:28.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:27 smithi171 ceph-mon[90031]: pgmap v451: 89 pgs: 1 activating+degraded, 1 active, 2 active+recovery_wait+degraded, 3 remapped+peering, 12 peering, 1 active+remapped+backfilling, 69 active+clean; 1.1 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 10/53661 objects degraded (0.019%); 106/53661 objects misplaced (0.198%); 40 MiB/s, 642 objects/s recovering 2024-08-29T15:14:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:27 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 10/53661 objects degraded (0.019%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:14:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:27 smithi171 ceph-mon[90031]: osdmap e151: 8 total, 8 up, 8 in 2024-08-29T15:14:28.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:28.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:28.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:28.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:28.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:28.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:28.534 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:28.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:28.658 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:28.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:28.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:28.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:28.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:28.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:28.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:28.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:29.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:29.080 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:29.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:29.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:29.203 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:29.208 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:14:29] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.51.0" 2024-08-29T15:14:29.229 INFO:tasks.thrashosds.thrasher:Added osd 5 2024-08-29T15:14:29.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:29.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:29.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:29.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:29.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:29.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:29.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:29.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:29.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:29.681 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:29.683 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:14:29.683 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:14:29.704 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:14:29.704 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:14:29.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:29.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:29.721 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:14:29.721 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:14:29.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:14:29.737 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:14:29.756 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:14:29.756 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:14:29.787 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:14:29.787 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:14:29.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:29.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:14:29.818 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:14:29.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:29.857 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:14:29.857 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[98970]: pgmap v454: 89 pgs: 1 activating+degraded, 1 active, 2 active+recovery_wait+degraded, 3 remapped+peering, 12 peering, 1 active+remapped+backfilling, 69 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 10/53490 objects degraded (0.019%); 106/53490 objects misplaced (0.198%); 23 MiB/s, 370 objects/s recovering 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[98970]: osdmap e152: 8 total, 8 up, 8 in 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1305458717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[93921]: pgmap v454: 89 pgs: 1 activating+degraded, 1 active, 2 active+recovery_wait+degraded, 3 remapped+peering, 12 peering, 1 active+remapped+backfilling, 69 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 10/53490 objects degraded (0.019%); 106/53490 objects misplaced (0.198%); 23 MiB/s, 370 objects/s recovering 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:29.858 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:29.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[93921]: osdmap e152: 8 total, 8 up, 8 in 2024-08-29T15:14:29.859 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1305458717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:29.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:29.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:29.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:29 smithi171 ceph-mon[90031]: pgmap v454: 89 pgs: 1 activating+degraded, 1 active, 2 active+recovery_wait+degraded, 3 remapped+peering, 12 peering, 1 active+remapped+backfilling, 69 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 215 op/s; 10/53490 objects degraded (0.019%); 106/53490 objects misplaced (0.198%); 23 MiB/s, 370 objects/s recovering 2024-08-29T15:14:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:29 smithi171 ceph-mon[90031]: osdmap e152: 8 total, 8 up, 8 in 2024-08-29T15:14:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1305458717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:30.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:30.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:30.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:30.215 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:30.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:30.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:30.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:30.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:30.380 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:30.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:30.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:30.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:30.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:30.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:30.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:30.656 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:30.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:30.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:30.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:30.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:30.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:30.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:30.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:30.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:30 smithi171 ceph-mon[90031]: osdmap e153: 8 total, 8 up, 8 in 2024-08-29T15:14:31.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:31.067 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:31.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:31.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:30 smithi027 ceph-mon[98970]: osdmap e153: 8 total, 8 up, 8 in 2024-08-29T15:14:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:30 smithi027 ceph-mon[93921]: osdmap e153: 8 total, 8 up, 8 in 2024-08-29T15:14:31.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:31.201 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:31.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:31.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:31.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:31.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:31.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:31.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:31.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:31.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:31.621 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:31.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:31.624 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:31.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:31.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:31.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:31.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:31.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:31.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:31.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:32.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:31 smithi171 ceph-mon[90031]: pgmap v457: 89 pgs: 89 active+clean; 1014 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 2.4 MiB/s, 45 objects/s recovering 2024-08-29T15:14:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:31 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/53490 objects degraded (0.019%), 3 pgs degraded) 2024-08-29T15:14:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.035 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:32.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:32.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[98970]: pgmap v457: 89 pgs: 89 active+clean; 1014 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 2.4 MiB/s, 45 objects/s recovering 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/53490 objects degraded (0.019%), 3 pgs degraded) 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[93921]: pgmap v457: 89 pgs: 89 active+clean; 1014 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 2.4 MiB/s, 45 objects/s recovering 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/53490 objects degraded (0.019%), 3 pgs degraded) 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.137 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:32.171 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:32.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:32.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:32.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:32.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:32.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:32.421 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:32.460 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:32.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:32.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:32.594 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:32.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:32.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:32.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:32.730 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:32.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:32.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:14:32.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:32.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:32.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:32.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:32.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:32 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:14:32.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:32 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:14:32.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:32 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:32 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:14:32.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:32 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:14:32.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:32 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:32.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:32 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:14:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:32 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:14:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:32 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:33.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:33.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:33.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.160 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:33.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:33.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:33.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:33.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:33.424 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:33.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:33.526 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:33.560 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:33.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:33.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:33.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:33.701 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:14:33.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:33.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:33.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:33.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:33.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:33.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:33 smithi171 ceph-mon[90031]: pgmap v458: 89 pgs: 89 active+clean; 1003 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 317 op/s; 1.7 MiB/s, 32 objects/s recovering 2024-08-29T15:14:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4294693811' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:14:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:33 smithi027 ceph-mon[93921]: pgmap v458: 89 pgs: 89 active+clean; 1003 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 317 op/s; 1.7 MiB/s, 32 objects/s recovering 2024-08-29T15:14:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4294693811' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:14:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:33 smithi027 ceph-mon[98970]: pgmap v458: 89 pgs: 89 active+clean; 1003 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 317 op/s; 1.7 MiB/s, 32 objects/s recovering 2024-08-29T15:14:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4294693811' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:14:34.072 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:34.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:34.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:34.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:34.208 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:34.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:34.230 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7, 5] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:14:34.230 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:14:34.230 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 6, 7, 5] 2024-08-29T15:14:34.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd out 6 2024-08-29T15:14:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:34.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:34.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:34.352 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:34.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:34.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:34.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:34.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:34.522 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:34.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:34.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:34.658 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:34.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:34.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:34.760 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:34.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:34.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:34.892 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:34.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:34.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:34.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:34.931 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:14:34.931 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:14:34.932 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4294693811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:14:34.932 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[93921]: osdmap e154: 8 total, 8 up, 8 in 2024-08-29T15:14:34.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:14:34.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:14:34.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4294693811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:14:34.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:34 smithi027 ceph-mon[98970]: osdmap e154: 8 total, 8 up, 8 in 2024-08-29T15:14:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:34 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:14:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:34 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:14:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4294693811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:14:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:34 smithi171 ceph-mon[90031]: osdmap e154: 8 total, 8 up, 8 in 2024-08-29T15:14:35.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:35.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:35.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:35.166 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:35.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:35.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:35.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:35.337 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:35.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:35.424 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:35.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:35.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:35.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:35.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:256468 5.8 5:1b378de5:::benchmark_data_smithi027_135490_object122716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:35.474978+0000", 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0018066239999999999, 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018352049999999999, 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:14:35.479 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 256468 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.474978+0000", 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:35.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.474978+0000", 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.474980+0000", 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5820000000000001e-06 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.474985+0000", 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0679999999999996e-06 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475000+0000", 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5336999999999999e-05 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475007+0000", 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9349999999999997e-06 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:35.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475089+0000", 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1988000000000002e-05 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475208+0000", 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119492 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475322+0000", 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113538 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.483 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:256471 5.6 5:623622fc:::benchmark_data_smithi027_135490_object122719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:35.475982+0000", 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00080234000000000002, 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00090684000000000001, 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:35.484 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 256471 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475982+0000", 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475982+0000", 2024-08-29T15:14:35.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475983+0000", 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9000000000000005e-07 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.475988+0000", 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5589999999999999e-06 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.476002+0000", 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4011e-05 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.476009+0000", 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6149999999999999e-06 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.476060+0000", 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0062999999999998e-05 2024-08-29T15:14:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.476166+0000", 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010602099999999999 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.476266+0000", 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100154 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:256460 5.b 5:d9b397cf:::benchmark_data_smithi027_135490_object122708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:35.464265+0000", 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012520124000000001, 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012686563, 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 256460 2024-08-29T15:14:35.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464265+0000", 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464265+0000", 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:35.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464266+0000", 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3340000000000001e-06 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464270+0000", 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5449999999999997e-06 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464280+0000", 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8539999999999992e-06 2024-08-29T15:14:35.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464287+0000", 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1690000000000003e-06 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.467868+0000", 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035807310000000002 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.468230+0000", 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00036162400000000001 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.468686+0000", 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045586599999999998 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:256463 5.a 5:57f8bb6f:::benchmark_data_smithi027_135490_object122711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:35.464784+0000", 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012001016999999999, 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012226941, 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:35.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 256463 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464784+0000", 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464784+0000", 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:35.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464785+0000", 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1659999999999999e-06 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464789+0000", 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6990000000000002e-06 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:35.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464803+0000", 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3655999999999999e-05 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464811+0000", 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2020000000000004e-06 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.464902+0000", 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1138999999999996e-05 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.465031+0000", 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128818 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.465200+0000", 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016829599999999999 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.470051+0000", 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048515260000000001 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:35.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:35.474797+0000", 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047458630000000003 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 4 2024-08-29T15:14:35.499 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:14:35.579 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:35.614 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:35.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:35.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:35.716 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:35.756 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:35.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:35.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:35.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:35.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:35.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:35.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:36.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:35 smithi171 ceph-mon[90031]: pgmap v460: 89 pgs: 89 active+clean; 901 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 1.6 MiB/s, 30 objects/s recovering 2024-08-29T15:14:36.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:36.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:36.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:35 smithi027 ceph-mon[93921]: pgmap v460: 89 pgs: 89 active+clean; 901 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 1.6 MiB/s, 30 objects/s recovering 2024-08-29T15:14:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:35 smithi027 ceph-mon[98970]: pgmap v460: 89 pgs: 89 active+clean; 901 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 1.6 MiB/s, 30 objects/s recovering 2024-08-29T15:14:36.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:36.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:36.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:36.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:36.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:36.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:36.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:36.400 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:36.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:36.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:36.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:14:36.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:36.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:36.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:36.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:14:36.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:36.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:36.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:36.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:36.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:36.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:37.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:37.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:37.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:37.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:37.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:37.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:37.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:37.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:37.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:37.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:37.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:37.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:37.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:37.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:37.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:37.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:37.603 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:37.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:37.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:37.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:37.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:37.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:37.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:37.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:37 smithi027 ceph-mon[98970]: pgmap v461: 89 pgs: 89 active+clean; 765 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1.3 MiB/s, 23 objects/s recovering 2024-08-29T15:14:37.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:37 smithi027 ceph-mon[93921]: pgmap v461: 89 pgs: 89 active+clean; 765 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1.3 MiB/s, 23 objects/s recovering 2024-08-29T15:14:37.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:37.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:38.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:37 smithi171 ceph-mon[90031]: pgmap v461: 89 pgs: 89 active+clean; 765 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1.3 MiB/s, 23 objects/s recovering 2024-08-29T15:14:38.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:38.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:38.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:38.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:38.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:38.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:38.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:38.304 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:38.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:38.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:38.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:38.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:38.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:38.518 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:38.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:38.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:38.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:38.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:38.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:38.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:38.728 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:38.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:38.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:38.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:38.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:38.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:39.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:39.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:39.103 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:39.104 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:39.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:39.190 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:14:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:14:39] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.51.0" 2024-08-29T15:14:39.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:39.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:39.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:39.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:39.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:39.519 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:39.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:39.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:39.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:39.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:39.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:39.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:39.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:39.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:39.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:39.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:39 smithi027 ceph-mon[98970]: pgmap v462: 89 pgs: 89 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 923 KiB/s, 16 objects/s recovering 2024-08-29T15:14:39.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:39 smithi027 ceph-mon[93921]: pgmap v462: 89 pgs: 89 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 923 KiB/s, 16 objects/s recovering 2024-08-29T15:14:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:39 smithi171 ceph-mon[90031]: pgmap v462: 89 pgs: 89 active+clean; 749 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 923 KiB/s, 16 objects/s recovering 2024-08-29T15:14:40.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:40.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:40.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:40.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:40.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:40.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:40.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:40.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:40.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:40.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:40.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:40.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:40.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:40.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:40.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:40.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:40.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:40.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:40.753 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:14:40.753 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:14:40.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:40.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:40.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:40.793 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:14:40.794 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:14:40.833 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:14:40.833 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:14:40.855 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:14:40.855 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:14:40.873 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:14:40.874 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:14:40.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:40.894 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:14:40.894 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:14:40.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3499289814' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-29T15:14:40.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-29T15:14:40.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[98970]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-29T15:14:40.895 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1517093699' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:40.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:14:40.631+0000 7f3f0d183640 -1 mon.a@0(leader).osd e154 definitely_dead 0 2024-08-29T15:14:40.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3499289814' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-29T15:14:40.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-29T15:14:40.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[93921]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-29T15:14:40.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1517093699' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:40.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:40.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:40.912 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:14:40.912 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:14:40.929 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:14:40.929 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:14:41.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3499289814' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-29T15:14:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:40 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-29T15:14:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:40 smithi171 ceph-mon[90031]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-29T15:14:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1517093699' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:41.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:41.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:41.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:41.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:41.280 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:41.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:41.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:41.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:41.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:41.530 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:41.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:41.654 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:41.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:41.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:41.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:41.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:41.835 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[98970]: pgmap v463: 89 pgs: 89 active+clean; 499 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-29T15:14:41.835 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-29T15:14:41.835 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[98970]: osdmap e155: 8 total, 8 up, 7 in 2024-08-29T15:14:41.835 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[98970]: osdmap e156: 8 total, 8 up, 7 in 2024-08-29T15:14:41.836 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[93921]: pgmap v463: 89 pgs: 89 active+clean; 499 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-29T15:14:41.836 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-29T15:14:41.836 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[93921]: osdmap e155: 8 total, 8 up, 7 in 2024-08-29T15:14:41.836 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:41 smithi027 ceph-mon[93921]: osdmap e156: 8 total, 8 up, 7 in 2024-08-29T15:14:41.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:41.902 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:41.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:42.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:41 smithi171 ceph-mon[90031]: pgmap v463: 89 pgs: 89 active+clean; 499 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-29T15:14:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:41 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-29T15:14:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:41 smithi171 ceph-mon[90031]: osdmap e155: 8 total, 8 up, 7 in 2024-08-29T15:14:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:41 smithi171 ceph-mon[90031]: osdmap e156: 8 total, 8 up, 7 in 2024-08-29T15:14:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:42.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:42.117 INFO:teuthology.orchestra.run.smithi027.stderr:marked out osd.6. 2024-08-29T15:14:42.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:42.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:42.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:42.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:42.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:42.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:42.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:42.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:42.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:42.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:42.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:42.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:42.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:42.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:42.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:42.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:42.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:42.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:42.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:42.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:43.020 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:43.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:43.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:43.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:43.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:43.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:43.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:43.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:43.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:43.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:43.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:43.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:43.711 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:43.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:43.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:43.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:43.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:43.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:43.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:14:43.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:43.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: pgmap v466: 89 pgs: 4 peering, 85 active+clean; 464 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: 5.f starting backfill to osd.2 from (0'0,0'0] MAX to 154'16545 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: osdmap e157: 8 total, 8 up, 7 in 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: 5.9 starting backfill to osd.1 from (0'0,0'0] MAX to 154'16232 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 154'16167 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 154'16231 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: pgmap v466: 89 pgs: 4 peering, 85 active+clean; 464 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:14:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: 5.f starting backfill to osd.2 from (0'0,0'0] MAX to 154'16545 2024-08-29T15:14:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: osdmap e157: 8 total, 8 up, 7 in 2024-08-29T15:14:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: 5.9 starting backfill to osd.1 from (0'0,0'0] MAX to 154'16232 2024-08-29T15:14:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 154'16167 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 154'16231 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: pgmap v466: 89 pgs: 4 peering, 85 active+clean; 464 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: 5.f starting backfill to osd.2 from (0'0,0'0] MAX to 154'16545 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: osdmap e157: 8 total, 8 up, 7 in 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: 5.9 starting backfill to osd.1 from (0'0,0'0] MAX to 154'16232 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 154'16167 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: 5.3 starting backfill to osd.3 from (0'0,0'0] MAX to 154'16231 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:44.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:44.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:44.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:44.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:44.231 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:44.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:44.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:44.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:44.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:44.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:44.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:44.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:44.527 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:44.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:44.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:44.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:44.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:44.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:44.717 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:14:44.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:44.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:44.816 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:44.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:44.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:44 smithi027 ceph-mon[93921]: osdmap e158: 8 total, 8 up, 7 in 2024-08-29T15:14:44.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3522878074' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:14:44.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:44 smithi027 ceph-mon[98970]: osdmap e158: 8 total, 8 up, 7 in 2024-08-29T15:14:44.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3522878074' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:14:44.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:44.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:44.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:44.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:44 smithi171 ceph-mon[90031]: osdmap e158: 8 total, 8 up, 7 in 2024-08-29T15:14:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3522878074' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:14:45.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:45.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:45.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:45.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:45.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:45.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:45.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:45.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:45.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:45.388 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:45.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:45.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:45.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:45.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:45.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:45.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:45.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:45.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:45.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:45.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:45.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:45.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:45.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:45.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:45.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:45.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[93921]: pgmap v469: 89 pgs: 2 active+remapped+backfill_wait, 4 peering, 83 active+clean; 392 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 680/18675 objects misplaced (3.641%) 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3522878074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[93921]: osdmap e159: 8 total, 8 up, 7 in 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[98970]: pgmap v469: 89 pgs: 2 active+remapped+backfill_wait, 4 peering, 83 active+clean; 392 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 680/18675 objects misplaced (3.641%) 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3522878074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:14:45.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:45 smithi027 ceph-mon[98970]: osdmap e159: 8 total, 8 up, 7 in 2024-08-29T15:14:46.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:45 smithi171 ceph-mon[90031]: pgmap v469: 89 pgs: 2 active+remapped+backfill_wait, 4 peering, 83 active+clean; 392 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 680/18675 objects misplaced (3.641%) 2024-08-29T15:14:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:45 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:14:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3522878074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:14:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:45 smithi171 ceph-mon[90031]: osdmap e159: 8 total, 8 up, 7 in 2024-08-29T15:14:46.032 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:46.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:46.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:46.247 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:14:46.247 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:14:46.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:46.286 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:46.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:46.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:46.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:46.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:46.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:46.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:46.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:46.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:46.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:46.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:46.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:46.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:46.898 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:46.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:46.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:46 smithi171 ceph-mon[90031]: osdmap e160: 8 total, 8 up, 7 in 2024-08-29T15:14:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:46 smithi171 ceph-mon[90031]: osdmap e161: 8 total, 8 up, 7 in 2024-08-29T15:14:47.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:47.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:47.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:46 smithi027 ceph-mon[93921]: osdmap e160: 8 total, 8 up, 7 in 2024-08-29T15:14:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:46 smithi027 ceph-mon[93921]: osdmap e161: 8 total, 8 up, 7 in 2024-08-29T15:14:47.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:46 smithi027 ceph-mon[98970]: osdmap e160: 8 total, 8 up, 7 in 2024-08-29T15:14:47.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:46 smithi027 ceph-mon[98970]: osdmap e161: 8 total, 8 up, 7 in 2024-08-29T15:14:47.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:47.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:14:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:47.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:47.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:47.213 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:47.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:47.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:47.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:47.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:47.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:47.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:47.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:47.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:47.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:47.655 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:47.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:47.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:47.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:47.748 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:14:47.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 0 1 2024-08-29T15:14:47.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:47.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:47.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:47.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:48.000 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:48.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:48.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:14:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:47 smithi171 ceph-mon[90031]: pgmap v472: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 5 peering, 80 active+clean; 328 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 870/15624 objects misplaced (5.568%); 7.4 MiB/s, 122 objects/s recovering 2024-08-29T15:14:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:47 smithi171 ceph-mon[90031]: osdmap e162: 8 total, 8 up, 7 in 2024-08-29T15:14:48.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:47 smithi027 ceph-mon[93921]: pgmap v472: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 5 peering, 80 active+clean; 328 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 870/15624 objects misplaced (5.568%); 7.4 MiB/s, 122 objects/s recovering 2024-08-29T15:14:48.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:47 smithi027 ceph-mon[93921]: osdmap e162: 8 total, 8 up, 7 in 2024-08-29T15:14:48.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:47 smithi027 ceph-mon[98970]: pgmap v472: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 5 peering, 80 active+clean; 328 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 870/15624 objects misplaced (5.568%); 7.4 MiB/s, 122 objects/s recovering 2024-08-29T15:14:48.056 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:47 smithi027 ceph-mon[98970]: osdmap e162: 8 total, 8 up, 7 in 2024-08-29T15:14:48.101 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:48.169 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:48.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:48.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:48.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:48.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:48.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:48.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:48.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:48.440 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:48.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:48.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:48.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:48.580 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:48.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:48.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:48.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:48.740 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:48.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:48.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:48.876 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:48.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:48.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:48.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:49.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:49.103 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:49.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:49.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:49.230 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:49.257 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:49.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:14:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:14:49] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.51.0" 2024-08-29T15:14:49.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:49.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:49.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:49.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:49.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:49.665 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:49.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:49.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:49.876 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:14:49.877 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:14:49.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:49.914 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:14:49.914 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:14:49.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:49.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:49.959 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:14:49.959 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:14:49.959 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:49 smithi027 ceph-mon[98970]: pgmap v475: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 84 active+clean; 318 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 870/15132 objects misplaced (5.749%); 7.4 MiB/s, 123 objects/s recovering 2024-08-29T15:14:49.960 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:49 smithi027 ceph-mon[93921]: pgmap v475: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 84 active+clean; 318 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 870/15132 objects misplaced (5.749%); 7.4 MiB/s, 123 objects/s recovering 2024-08-29T15:14:49.999 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:14:49.999 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:14:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:49 smithi171 ceph-mon[90031]: pgmap v475: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 84 active+clean; 318 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 295 op/s; 870/15132 objects misplaced (5.749%); 7.4 MiB/s, 123 objects/s recovering 2024-08-29T15:14:50.041 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:14:50.042 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:14:50.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:50.074 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:14:50.074 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:14:50.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:50.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:50.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:50.108 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:14:50.108 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:14:50.138 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:14:50.138 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:14:50.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:50.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:50.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:50.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:50.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:50.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:50.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:50.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:50.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:50.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:50.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:50.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:50.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:50.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:50.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:50.768 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:50.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:50.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:50.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:50.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:51.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:51.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3278676742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:51.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:51.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3278676742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:51.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3278676742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:14:51.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:51.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.174 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:51.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:51.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:51.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:51.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:51.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:51.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:51.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:51.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:51.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:51.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:51.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:51.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:51.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:51.823 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Removed 133749 objects 2024-08-29T15:14:51.823 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Clean up completed and total clean up time :168.337 2024-08-29T15:14:51.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:51.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:51.964 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:51.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:51.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:52.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:51 smithi171 ceph-mon[90031]: pgmap v476: 89 pgs: 89 active+clean; 163 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 8.9 MiB/s, 146 objects/s recovering 2024-08-29T15:14:52.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:51 smithi027 ceph-mon[98970]: pgmap v476: 89 pgs: 89 active+clean; 163 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 8.9 MiB/s, 146 objects/s recovering 2024-08-29T15:14:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:51 smithi027 ceph-mon[93921]: pgmap v476: 89 pgs: 89 active+clean; 163 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 8.9 MiB/s, 146 objects/s recovering 2024-08-29T15:14:52.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:52.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:52.212 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:52.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:52.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:52.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:52.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:52.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:52.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:52.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:52.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:52.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:52.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:52.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:52.703 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:52.703 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:52.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:52.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:52.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:52.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:52.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:52.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:52.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:53.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:53.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:53.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:53.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:53.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:53.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:53.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:53.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:53.476 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:53.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:53.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:53.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:53.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:53.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:53.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:53.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:53.861 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:53.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:53.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:53.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:53 smithi171 ceph-mon[90031]: pgmap v477: 89 pgs: 89 active+clean; 138 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 5.9 MiB/s, 94 objects/s recovering 2024-08-29T15:14:54.056 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:53 smithi027 ceph-mon[93921]: pgmap v477: 89 pgs: 89 active+clean; 138 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 5.9 MiB/s, 94 objects/s recovering 2024-08-29T15:14:54.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:53 smithi027 ceph-mon[98970]: pgmap v477: 89 pgs: 89 active+clean; 138 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 5.9 MiB/s, 94 objects/s recovering 2024-08-29T15:14:54.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:54.115 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:54.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:54.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:54.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:54.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:54.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:54.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:54.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:54.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:54.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:54.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:54.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:54.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:54.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:54.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:54.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:54.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:54.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:54.748 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:54.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:54.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:54.850 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:54.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:54.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:54.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:55.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:55.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:55.132 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:55.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:55.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:55.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:55.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:55.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:55.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:55.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:55.519 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:55.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:55.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:55.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:55.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:55.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:55.756 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-08-29T15:14:55.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-08-29T15:14:55.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:55.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:55.902 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:55.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:55.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:14:56.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:56.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:55 smithi171 ceph-mon[90031]: pgmap v478: 89 pgs: 89 active+clean; 57 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 4.9 MiB/s, 79 objects/s recovering 2024-08-29T15:14:56.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:55 smithi027 ceph-mon[98970]: pgmap v478: 89 pgs: 89 active+clean; 57 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 4.9 MiB/s, 79 objects/s recovering 2024-08-29T15:14:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:55 smithi027 ceph-mon[93921]: pgmap v478: 89 pgs: 89 active+clean; 57 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 4.9 MiB/s, 79 objects/s recovering 2024-08-29T15:14:56.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:56.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:56.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:56.276 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:56.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:56.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:56.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:56.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:56.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:56.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:56.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:56.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:56.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:56.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:56.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:56.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.772 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:56.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:56.873 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:56.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:56.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:57.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:57.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:57.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:57.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:57.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:57.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:57.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:57.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:57.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:57.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:57.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:14:57.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:57.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:14:57.579 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:14:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:14:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23186 5.6 5:623074f2:::benchmark_data_smithi027_135490_object23185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.949059+0000", 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.62780940300001, 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14490519299999999, 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23186 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949059+0000", 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949059+0000", 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949060+0000", 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3e-07 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949082+0000", 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1919000000000001e-05 2024-08-29T15:14:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949092+0000", 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0761000000000001e-05 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949099+0000", 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9330000000000004e-06 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949455+0000", 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035609599999999997 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949687+0000", 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023169900000000001 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.949829+0000", 2024-08-29T15:14:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000142006 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953284+0000", 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034552850000000002 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953844+0000", 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00055922800000000005 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092116+0000", 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.138272688 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093921+0000", 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018042819999999999 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093964+0000", 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3566000000000003e-05 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23191 5.6 5:65ba2179:::benchmark_data_smithi027_135490_object23190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.951583+0000", 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.62528558100001, 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14251501499999999, 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23191 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951583+0000", 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951583+0000", 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951584+0000", 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.595e-06 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951607+0000", 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2668999999999999e-05 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951621+0000", 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3484e-05 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951628+0000", 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9109999999999993e-06 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951686+0000", 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7859999999999998e-05 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951815+0000", 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012877 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.951960+0000", 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144976 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953673+0000", 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001712748 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954262+0000", 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058924800000000001 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094015+0000", 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13975259500000001 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094062+0000", 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7509999999999997e-05 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094098+0000", 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5648999999999997e-05 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23194 5.8 5:1e14095a:::benchmark_data_smithi027_135490_object23193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.953618+0000", 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.62325066, 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14412351700000001, 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23194 2024-08-29T15:14:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953618+0000", 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953618+0000", 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953619+0000", 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139e-06 2024-08-29T15:14:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953643+0000", 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4454e-05 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953654+0000", 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1015999999999999e-05 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953661+0000", 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8179999999999998e-06 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953927+0000", 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026623199999999998 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092057+0000", 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13812923499999999 2024-08-29T15:14:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092234+0000", 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000176925 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097469+0000", 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052357339999999997 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097509+0000", 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9536000000000002e-05 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097622+0000", 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011326400000000001 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097701+0000", 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9154999999999997e-05 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097741+0000", 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0009000000000003e-05 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:23186 5.6 e115/111 ondisk, result = 0)", 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.953984+0000", 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.62288411899999, 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13999176999999999, 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953984+0000", 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953984+0000", 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953985+0000", 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7500000000000001e-07 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953987+0000", 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6249999999999999e-06 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.953996+0000", 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3359999999999995e-06 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954001+0000", 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4539999999999998e-06 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954288+0000", 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028707100000000002 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954294+0000", 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3080000000000004e-06 2024-08-29T15:14:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093976+0000", 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13968160099999999 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:23191 5.6 e115/111 ondisk, result = 0)", 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.954027+0000", 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.62284124000001, 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.140091682, 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954027+0000", 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954027+0000", 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954028+0000", 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5400000000000002e-07 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954030+0000", 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6929999999999999e-06 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954038+0000", 2024-08-29T15:14:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5789999999999998e-06 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954043+0000", 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1580000000000004e-06 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094001+0000", 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13995839700000001 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094009+0000", 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5140000000000004e-06 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094119+0000", 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109887 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23196 5.b 5:db57f893:::benchmark_data_smithi027_135490_object23195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.954654+0000", 2024-08-29T15:14:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.62221414599998, 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14965039399999999, 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23196 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954654+0000", 2024-08-29T15:14:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954654+0000", 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954655+0000", 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.071e-06 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.954678+0000", 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2886000000000001e-05 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092083+0000", 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13740485399999999 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092101+0000", 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7850999999999999e-05 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092171+0000", 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9696999999999994e-05 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092322+0000", 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000151547 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.094055+0000", 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017333260000000001 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.097546+0000", 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003490904 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.103537+0000", 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059910780000000004 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.104211+0000", 2024-08-29T15:14:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00067352000000000004 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.104264+0000", 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2648999999999997e-05 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.104305+0000", 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1010999999999997e-05 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:23209 5.4 e115/111 5:24099454:::benchmark_data_smithi027_135490_object23208:head v 115'1489, mlcod=115'1487)", 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.957922+0000", 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.61894660000002, 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13573950700000001, 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957922+0000", 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957922+0000", 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957923+0000", 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.274e-06 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.957979+0000", 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5788000000000002e-05 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092353+0000", 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134373779 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092360+0000", 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2559999999999996e-06 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092409+0000", 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9481000000000003e-05 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.092430+0000", 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0945e-05 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093619+0000", 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001188417 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093661+0000", 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2567000000000001e-05 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.619 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68618 5.b 5:dedf1630:::benchmark_data_smithi027_135490_object68617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.636700+0000", 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 232.94016861200001, 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134410845, 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68618 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636700+0000", 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636700+0000", 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636701+0000", 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0699999999999996e-07 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636727+0000", 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5918000000000002e-05 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636738+0000", 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.13e-05 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.636745+0000", 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9779999999999999e-06 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.643278+0000", 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065328010000000004 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.643406+0000", 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012882799999999999 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.643546+0000", 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013906899999999999 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648290+0000", 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047440140000000004 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.648333+0000", 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3448999999999997e-05 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.770935+0000", 2024-08-29T15:14:57.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.122602275 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771025+0000", 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9437000000000004e-05 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.771111+0000", 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5869000000000001e-05 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68896 5.b 5:d4a8571b:::benchmark_data_smithi027_135490_object68895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 232.67465250999999, 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15188533300000001, 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68896 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.626 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902217+0000", 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2399999999999996e-07 2024-08-29T15:14:57.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902238+0000", 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1038000000000001e-05 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902505+0000", 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026729299999999999 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902514+0000", 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5869999999999993e-06 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908889+0000", 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063751140000000003 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909004+0000", 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011566299999999999 2024-08-29T15:14:57.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909129+0000", 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012426399999999999 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.911589+0000", 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024604990000000001 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912563+0000", 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000974156 2024-08-29T15:14:57.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053999+0000", 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14143561299999999 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054055+0000", 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6283999999999998e-05 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054101+0000", 2024-08-29T15:14:57.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5898e-05 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:221723 5.6 5:6241f24b:::benchmark_data_smithi027_135490_object87971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:13:45.161614+0000", 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 72.415254778999994, 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12971991799999999, 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.632 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 221723 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161614+0000", 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161614+0000", 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161614+0000", 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5999999999999999e-07 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161617+0000", 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8499999999999998e-06 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161626+0000", 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9029999999999993e-06 2024-08-29T15:14:57.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161631+0000", 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6459999999999998e-06 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161676+0000", 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4121e-05 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161776+0000", 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100121 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.161890+0000", 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114067 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:57.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.166641+0000", 2024-08-29T15:14:57.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047510579999999998 2024-08-29T15:14:57.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.168226+0000", 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0015854160000000001 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.291236+0000", 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12300995300000001 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.291286+0000", 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0077999999999997e-05 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.291333+0000", 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7244999999999999e-05 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:221726 5.b 5:d225b419:::benchmark_data_smithi027_135490_object87974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:13:45.162194+0000", 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 72.414673958999998, 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.129687733, 2024-08-29T15:14:57.639 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 221726 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162194+0000", 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162194+0000", 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162195+0000", 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0999999999999997e-07 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162200+0000", 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3069999999999996e-06 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162211+0000", 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1262e-05 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162218+0000", 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0720000000000002e-06 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162272+0000", 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4342000000000002e-05 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162374+0000", 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102039 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.162478+0000", 2024-08-29T15:14:57.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103519 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.167010+0000", 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045323209999999997 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.176360+0000", 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093496169999999993 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.291785+0000", 2024-08-29T15:14:57.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.115424809 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.291843+0000", 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8658e-05 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:45.291882+0000", 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8877000000000001e-05 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.645 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239385 5.b 5:d0c9da49:::benchmark_data_smithi027_135490_object105633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.720457+0000", 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 51.856411708000003, 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.228704257, 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.646 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239385 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720457+0000", 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720457+0000", 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720458+0000", 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2619999999999999e-06 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720462+0000", 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3030000000000001e-06 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720476+0000", 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3988999999999999e-05 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720484+0000", 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3470000000000001e-06 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720588+0000", 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104298 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720691+0000", 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010344 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.720795+0000", 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010419700000000001 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.725129+0000", 2024-08-29T15:14:57.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043339449999999996 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.726368+0000", 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012390459999999999 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.746755+0000", 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020386532999999998 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.746765+0000", 2024-08-29T15:14:57.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0156000000000001e-05 2024-08-29T15:14:57.652 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:14:57.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.696104+0000", 2024-08-29T15:14:57.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94933848399999998 2024-08-29T15:14:57.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.696115+0000", 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.153e-05 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.949069+0000", 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.252953937 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.949161+0000", 2024-08-29T15:14:57.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.179e-05 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239389 5.6 5:6048f328:::benchmark_data_smithi027_135490_object105637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.728304+0000", 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 51.848564205999999, 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3283723119999999, 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.655 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239389 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728304+0000", 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728304+0000", 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728306+0000", 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4759999999999999e-06 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728311+0000", 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5060000000000003e-06 2024-08-29T15:14:57.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728321+0000", 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.927e-06 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728326+0000", 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3499999999999996e-06 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728402+0000", 2024-08-29T15:14:57.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5881000000000001e-05 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728590+0000", 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000187663 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.728735+0000", 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014538499999999999 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.734734+0000", 2024-08-29T15:14:57.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059985009999999998 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.735624+0000", 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00089028000000000002 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.743072+0000", 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074482359999999996 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.743086+0000", 2024-08-29T15:14:57.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.345e-05 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.694608+0000", 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95152238099999997 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.694628+0000", 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9565999999999999e-05 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.056602+0000", 2024-08-29T15:14:57.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3619738639999999 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.056676+0000", 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4845999999999995e-05 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239401 5.d 5:b2f5332d:::benchmark_data_smithi027_135490_object105649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.758568+0000", 2024-08-29T15:14:57.662 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 51.818300555, 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3419801910000002, 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239401 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758568+0000", 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758568+0000", 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758569+0000", 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.669e-06 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758575+0000", 2024-08-29T15:14:57.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4970000000000001e-06 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758593+0000", 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.772e-05 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758602+0000", 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0389999999999993e-06 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758753+0000", 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015147700000000001 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758769+0000", 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6138e-05 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.696610+0000", 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93784030200000001 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.696620+0000", 2024-08-29T15:14:57.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0258000000000001e-05 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.058322+0000", 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.361702151 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.058513+0000", 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019125899999999999 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5,7", 2024-08-29T15:14:57.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.058685+0000", 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017140600000000001 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.066677+0000", 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079926140000000003 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.085868+0000", 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019191165 2024-08-29T15:14:57.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.085911+0000", 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2762000000000002e-05 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100384+0000", 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014472914999999999 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100438+0000", 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4231999999999998e-05 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100548+0000", 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010958700000000001 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239403 5.b 5:d625efa3:::benchmark_data_smithi027_135490_object105651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:14:57.670 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.758945+0000", 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 51.817923727999997, 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2291714229999999, 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239403 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758945+0000", 2024-08-29T15:14:57.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758945+0000", 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758946+0000", 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3900000000000003e-07 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758950+0000", 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.425e-06 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758965+0000", 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5237e-05 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758974+0000", 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3659999999999992e-06 2024-08-29T15:14:57.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.759042+0000", 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8343000000000006e-05 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.759053+0000", 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1055e-05 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.696143+0000", 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93708977999999998 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.696151+0000", 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2859999999999999e-06 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.949685+0000", 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.253534006 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.949831+0000", 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000145506 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5,7", 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.950701+0000", 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00087079799999999999 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.957730+0000", 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070285360000000002 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.979172+0000", 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021441688 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.979224+0000", 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2821999999999997e-05 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.987927+0000", 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0087025420000000006 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.987992+0000", 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5364999999999995e-05 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.988116+0000", 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012372899999999999 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239407 5.6 5:60530104:::benchmark_data_smithi027_135490_object105655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:14:57.678 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:07.933083+0000", 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.643785180000002, 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14829234199999999, 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239407 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933083+0000", 2024-08-29T15:14:57.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933083+0000", 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933084+0000", 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0800000000000003e-07 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933087+0000", 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3440000000000001e-06 2024-08-29T15:14:57.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933101+0000", 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3816e-05 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933109+0000", 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8259999999999993e-06 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.069314+0000", 2024-08-29T15:14:57.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13620543700000001 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.069445+0000", 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013047499999999999 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.069858+0000", 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041299000000000002 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.080945+0000", 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011087094 2024-08-29T15:14:57.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081080+0000", 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013479799999999999 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081094+0000", 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4171e-05 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081154+0000", 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9559999999999999e-05 2024-08-29T15:14:57.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081257+0000", 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103149 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081376+0000", 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118774 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:239405 5.e e141/140 5:7ac1943d:::benchmark_data_smithi027_135490_object105653:head v 141'15075, mlcod=0'0)", 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:07.935544+0000", 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.641323960999998, 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14549305100000001, 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.935544+0000", 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.935544+0000", 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.935546+0000", 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139e-06 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.935556+0000", 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0611000000000001e-05 2024-08-29T15:14:57.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.935566+0000", 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8409999999999995e-06 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.935576+0000", 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0293e-05 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.056757+0000", 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12118071900000001 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.056777+0000", 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9805000000000001e-05 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.080956+0000", 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024179394 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081037+0000", 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1248999999999994e-05 2024-08-29T15:14:57.688 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239408 5.d 5:b848a32f:::benchmark_data_smithi027_135490_object105656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:07.947241+0000", 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.629627526, 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.153451217, 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.689 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239408 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.947241+0000", 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.947241+0000", 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.947243+0000", 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0499999999999999e-06 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.947251+0000", 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0490000000000004e-06 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.947269+0000", 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7893999999999998e-05 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.947280+0000", 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1029000000000001e-05 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.066713+0000", 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11943311600000001 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.066840+0000", 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012737999999999999 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5,7", 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.066970+0000", 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129832 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.085950+0000", 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018979590000000001 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.085987+0000", 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7707000000000003e-05 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.086031+0000", 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3820999999999999e-05 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100556+0000", 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014524601 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100597+0000", 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1307000000000003e-05 2024-08-29T15:14:57.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100692+0000", 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4840999999999995e-05 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239411 5.6 5:61e1a24b:::benchmark_data_smithi027_135490_object105659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:07.951806+0000", 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.625062110000002, 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13105420500000001, 2024-08-29T15:14:57.695 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239411 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.951806+0000", 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.951806+0000", 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.951808+0000", 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6220000000000001e-06 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.951814+0000", 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9449999999999998e-06 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.951839+0000", 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4885999999999999e-05 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.951847+0000", 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.67e-06 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.075846+0000", 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.123998514 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.075975+0000", 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012945799999999999 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.076122+0000", 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014639899999999999 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081387+0000", 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052652899999999997 2024-08-29T15:14:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081427+0000", 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0036000000000001e-05 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081467+0000", 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0049999999999998e-05 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081503+0000", 2024-08-29T15:14:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5982999999999998e-05 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.081589+0000", 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5677e-05 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.082860+0000", 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012716750000000001 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239412 5.d 5:ba575c77:::benchmark_data_smithi027_135490_object105660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:07.956204+0000", 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.620664052000002, 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.144644139, 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239412 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:14:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.956204+0000", 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.956204+0000", 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.956207+0000", 2024-08-29T15:14:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5100000000000001e-06 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.956215+0000", 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0600000000000008e-06 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.956230+0000", 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5327000000000001e-05 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.956240+0000", 2024-08-29T15:14:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9709999999999999e-06 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.080445+0000", 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.124204498 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.080570+0000", 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012508899999999999 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5,7", 2024-08-29T15:14:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.080734+0000", 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016390799999999999 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100018+0000", 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019284058999999999 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100058+0000", 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0763000000000003e-05 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100134+0000", 2024-08-29T15:14:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5593000000000006e-05 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100700+0000", 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000566338 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100741+0000", 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0564e-05 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.100848+0000", 2024-08-29T15:14:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107459 2024-08-29T15:14:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:14:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:14:57.708 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:14:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:57.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:57.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:57.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:57.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:57.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:14:57.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:57 smithi027 ceph-mon[98970]: pgmap v479: 89 pgs: 89 active+clean; 5.1 MiB data, 1012 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:14:57.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:57 smithi027 ceph-mon[93921]: pgmap v479: 89 pgs: 89 active+clean; 5.1 MiB data, 1012 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:14:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:57.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:57.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:14:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:57 smithi171 ceph-mon[90031]: pgmap v479: 89 pgs: 89 active+clean; 5.1 MiB data, 1012 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:14:58.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:58.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:58.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:58.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:58.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:58.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:58.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:14:58.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:58.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:14:58.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:58.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:58.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:58.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:58.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:58.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:58.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:58.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:58.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:58.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:58.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:59.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:59.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:59.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:14:59.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:14:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:59.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:14:59.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:14:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:14:59] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-29T15:14:59.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:59.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:59.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:59.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:59.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:14:59.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:14:59.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:59.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:14:59.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:14:59.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:14:59.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:14:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:59 smithi027 ceph-mon[93921]: pgmap v480: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 3.7 MiB/s, 58 objects/s recovering 2024-08-29T15:14:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:14:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:59 smithi027 ceph-mon[98970]: pgmap v480: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 3.7 MiB/s, 58 objects/s recovering 2024-08-29T15:14:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:14:59.821 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:14:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:14:59.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:14:59.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:14:59.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:14:59.964 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:14:59.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:14:59.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:59 smithi171 ceph-mon[90031]: pgmap v480: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 3.7 MiB/s, 58 objects/s recovering 2024-08-29T15:15:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:14:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:00.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:00.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:00.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:00.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:00.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:00.342 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:00.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:00.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:00.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:00.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:00.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:00.594 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:00.683 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:15:00.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:00.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:00.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:00.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:00.841 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4249099512' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:00.841 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4249099512' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:00.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:00.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:00.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:00.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:00.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:00.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4249099512' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:01.073 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:01.075 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:01.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:01.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:01.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:01.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:01.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:01.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:01.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:01.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:01.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:01.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:01.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:01.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:01.604 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:01.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:01.687 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.0 to 1 (10000) 2024-08-29T15:15:01.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:01.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:01.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[93921]: pgmap v481: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 3.3 MiB/s, 52 objects/s recovering 2024-08-29T15:15:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4249099512' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:15:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[93921]: osdmap e163: 8 total, 8 up, 7 in 2024-08-29T15:15:01.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/83958539' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:15:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[98970]: pgmap v481: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 3.3 MiB/s, 52 objects/s recovering 2024-08-29T15:15:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4249099512' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:15:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[98970]: osdmap e163: 8 total, 8 up, 7 in 2024-08-29T15:15:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/83958539' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:15:01.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:01.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:01.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:01.973 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:02.001 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:02.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:01 smithi171 ceph-mon[90031]: pgmap v481: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 3.3 MiB/s, 52 objects/s recovering 2024-08-29T15:15:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:01 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4249099512' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:15:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:01 smithi171 ceph-mon[90031]: osdmap e163: 8 total, 8 up, 7 in 2024-08-29T15:15:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/83958539' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:15:02.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:02.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:02.228 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:02.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:02.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:02.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:02.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:02.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:02.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:02.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:02.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 1 1 2024-08-29T15:15:02.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:02.642 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:02.744 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:02.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:02.809 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/83958539' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:15:02.809 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:02 smithi027 ceph-mon[93921]: osdmap e164: 8 total, 8 up, 7 in 2024-08-29T15:15:02.809 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/83958539' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:15:02.809 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:02 smithi027 ceph-mon[98970]: osdmap e164: 8 total, 8 up, 7 in 2024-08-29T15:15:02.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:02.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:02.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:02.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:02.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/83958539' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:15:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:02 smithi171 ceph-mon[90031]: osdmap e164: 8 total, 8 up, 7 in 2024-08-29T15:15:03.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:03.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:03.071 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:03.076 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:15:03.077 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:15:03.095 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:15:03.096 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:15:03.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:15:03.112 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:15:03.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:03.129 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:15:03.129 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:15:03.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:03.146 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:15:03.146 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:15:03.163 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:15:03.163 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:15:03.179 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:15:03.180 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:15:03.196 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:15:03.197 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:15:03.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:03.266 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:03.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:03.368 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:03.394 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:03.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:03.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:03.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:03.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:15:03.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:03.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:03.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:03.747 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:03.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:03.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:03.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:03.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:03 smithi027 ceph-mon[98970]: pgmap v484: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 103 op/s 2024-08-29T15:15:03.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:03 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3802587011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:03.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:03 smithi027 ceph-mon[93921]: pgmap v484: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 103 op/s 2024-08-29T15:15:03.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:03 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3802587011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:03.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:03.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:03.998 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:03 smithi171 ceph-mon[90031]: pgmap v484: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 103 op/s 2024-08-29T15:15:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:03 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3802587011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:04.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:04.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:04.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:04.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:04.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:04.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:04.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:04.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:04.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:04.519 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:04.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:04.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:04.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:04.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:04.769 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:04.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:04.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:04.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:05.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:05.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:05.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:05.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:05.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:05.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:05.342 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:05.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:05.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:05.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:05.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:05.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:05.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:05.639 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:05.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:05.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:05.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:05.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:05.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:15:05.934 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:05 smithi027 ceph-mon[98970]: pgmap v485: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T15:15:05.935 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:05 smithi027 ceph-mon[93921]: pgmap v485: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T15:15:05.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:05.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:05.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:05 smithi171 ceph-mon[90031]: pgmap v485: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 0 op/s 2024-08-29T15:15:06.075 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:06.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:06.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:06.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:06.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:06.328 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:06.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:06.365 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:06.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:06.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:06.581 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:06.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:06.702 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_0' removed 2024-08-29T15:15:06.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:06.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:06.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:06.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:06.959 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:06.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:06.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:06.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4004898168' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:15:07.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4004898168' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:15:07.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4004898168' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:15:07.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:07.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:07.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:07.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:07.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:07.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:07.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:07.397 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:07.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:07.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:07.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:07.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:07.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:07.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:07.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:07.751 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:07.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:07.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:07.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:07.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:07 smithi027 ceph-mon[98970]: pgmap v486: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:07.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4004898168' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:15:07.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:07 smithi027 ceph-mon[98970]: osdmap e165: 8 total, 8 up, 7 in 2024-08-29T15:15:07.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:07 smithi027 ceph-mon[93921]: pgmap v486: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:07.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4004898168' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:15:07.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:07 smithi027 ceph-mon[93921]: osdmap e165: 8 total, 8 up, 7 in 2024-08-29T15:15:07.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:07.906 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:07.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:08.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:07 smithi171 ceph-mon[90031]: pgmap v486: 89 pgs: 89 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4004898168' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:15:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:07 smithi171 ceph-mon[90031]: osdmap e165: 8 total, 8 up, 7 in 2024-08-29T15:15:08.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:08.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:08.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:08.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:08.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:08.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:08.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:08.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:08.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:08.484 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-29T15:15:08.485 INFO:tasks.radosbench:Beginning radosbench... 2024-08-29T15:15:08.486 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-08-29T15:15:08.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-08-29T15:15:08.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:08.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:08.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:08.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:08.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.778 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:08.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:08.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:08.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:08.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:08.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:09.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:09.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:09.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:09.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:09.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:09.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:09.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:09.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:15:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:15:09] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-29T15:15:09.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:09.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:09.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:09.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:09.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:09.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:09.721 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:15:09.722 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:15:09.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:09.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:09.882 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:09.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:09.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:10.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:10.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:09 smithi027 ceph-mon[98970]: pgmap v488: 73 pgs: 73 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:10.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:09 smithi027 ceph-mon[93921]: pgmap v488: 73 pgs: 73 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:09 smithi171 ceph-mon[90031]: pgmap v488: 73 pgs: 73 active+clean; 4.6 MiB data, 1003 MiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:10.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:10.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:10.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:10.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:10.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:10.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:10.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.282 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:10.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:10.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:10.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:10.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:15:10.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:10.572 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:10.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:10.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:10.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:10.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:10.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:10.842 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:10.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:10.951 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:10.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:10.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:11.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:11.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:11.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:11.202 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:11.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:11.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:11.409 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:11.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:11.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:11.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:11.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:11.593 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:11.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:11.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:11.712 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.1 to 1 (10000) 2024-08-29T15:15:11.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:11.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:11.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:11.869 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:11 smithi027 ceph-mon[93921]: pgmap v489: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:11.870 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/290404637' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:15:11.870 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:15:11.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:11 smithi027 ceph-mon[98970]: pgmap v489: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:11.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/290404637' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:15:11.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:15:11.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:11.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:11.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:11 smithi171 ceph-mon[90031]: pgmap v489: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/290404637' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:15:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:15:12.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:12.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:12.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:12.179 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:12.205 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:12.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:12.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:12.329 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:12.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 2 1 2024-08-29T15:15:12.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:12.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:12.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:12.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:12.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:12.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:12.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:12.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:12.839 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:12.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:12.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:12.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:12.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:12.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:12 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:15:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:12 smithi171 ceph-mon[90031]: osdmap e166: 8 total, 8 up, 7 in 2024-08-29T15:15:13.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:12 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:15:13.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:12 smithi027 ceph-mon[98970]: osdmap e166: 8 total, 8 up, 7 in 2024-08-29T15:15:13.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:12 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:15:13.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:12 smithi027 ceph-mon[93921]: osdmap e166: 8 total, 8 up, 7 in 2024-08-29T15:15:13.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:13.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:13.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:13.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:13.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:13.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:13.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:13.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:13.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:13.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:13.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:13.572 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:13.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:13.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:13.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:13.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:13.731 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:15:13.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:13.849 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:13.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:13.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:13.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:13.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:13.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:13 smithi171 ceph-mon[90031]: pgmap v491: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1115478586' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:15:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:15:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[98970]: pgmap v491: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1115478586' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:15:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:15:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[93921]: pgmap v491: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1115478586' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:15:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:15:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:14.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:14.098 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:14.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:14.226 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:14.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:14.328 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:14.350 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:14.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:14.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:14.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:14.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:14.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.604 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:14.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:14.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:14.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:14.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:14.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:14.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:14.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:14.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:14.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:14.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:14.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:14 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:14 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:15:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:14 smithi171 ceph-mon[90031]: osdmap e167: 8 total, 8 up, 7 in 2024-08-29T15:15:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:14 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:14 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:15:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:14 smithi027 ceph-mon[93921]: osdmap e167: 8 total, 8 up, 7 in 2024-08-29T15:15:15.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:14 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:15.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:14 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:15:15.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:14 smithi027 ceph-mon[98970]: osdmap e167: 8 total, 8 up, 7 in 2024-08-29T15:15:15.085 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:15.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:15.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:15.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:15.239 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:15.340 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:15.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:15.466 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:15.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:15.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:15.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:15.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:15.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:15.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:15.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:15.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:15.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:15.912 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:15.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:15.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:15 smithi027 ceph-mon[93921]: pgmap v493: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:15.985 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:15 smithi027 ceph-mon[98970]: pgmap v493: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:15.991 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:15.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:15.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:15 smithi171 ceph-mon[90031]: pgmap v493: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:16.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:16.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:16.216 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:16.242 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:16.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:16.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:16.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:16.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:16.495 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:16.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:16.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:16.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:16.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:16.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:16.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:16.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:16.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:16.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:16.994 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:16.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:16.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:17.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:17.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:17.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:17.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:17.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:17.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:17.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:17.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:17.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:17.471 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:17.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:17.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:17.595 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:17.621 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:17.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:17.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:17.742 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_1' created 2024-08-29T15:15:17.751 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:17.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:17.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:17.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:17.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:17.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:17.950 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:15:17.950 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:15:17.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:18.001 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:18.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:17 smithi171 ceph-mon[90031]: pgmap v494: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4132973820' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-29T15:15:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:17 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-29T15:15:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:17 smithi027 ceph-mon[93921]: pgmap v494: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4132973820' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-29T15:15:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:17 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-29T15:15:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:17 smithi027 ceph-mon[98970]: pgmap v494: 73 pgs: 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4132973820' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-29T15:15:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:17 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-29T15:15:18.103 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:18.128 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:18.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:18.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:18.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:18.379 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:18.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:18.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:15:18.481 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:18.506 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:18.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:18.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:18.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:18.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:18.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:18.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:18.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:18.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:18.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:18.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:19.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:18 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-29T15:15:19.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:18 smithi027 ceph-mon[93921]: osdmap e168: 8 total, 8 up, 7 in 2024-08-29T15:15:19.004 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:18 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-29T15:15:19.004 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:18 smithi027 ceph-mon[98970]: osdmap e168: 8 total, 8 up, 7 in 2024-08-29T15:15:19.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:19.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:18 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-29T15:15:19.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:18 smithi171 ceph-mon[90031]: osdmap e168: 8 total, 8 up, 7 in 2024-08-29T15:15:19.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-08-29T15:15:19.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:19.140 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:19.241 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:19.267 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:19.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:15:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:15:19] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.51.0" 2024-08-29T15:15:19.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:19.394 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:19.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:19.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:15:19.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:19.520 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:19.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:19.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:19.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:19.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:19.771 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:19.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:19.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:19.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:19.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:19.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:19 smithi171 ceph-mon[90031]: pgmap v496: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:19 smithi171 ceph-mon[90031]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:15:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:19 smithi171 ceph-mon[90031]: osdmap e169: 8 total, 8 up, 7 in 2024-08-29T15:15:20.022 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:20.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:19 smithi027 ceph-mon[98970]: pgmap v496: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:20.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:19 smithi027 ceph-mon[98970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:15:20.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:19 smithi027 ceph-mon[98970]: osdmap e169: 8 total, 8 up, 7 in 2024-08-29T15:15:20.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:19 smithi027 ceph-mon[93921]: pgmap v496: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:20.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:19 smithi027 ceph-mon[93921]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:15:20.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:19 smithi027 ceph-mon[93921]: osdmap e169: 8 total, 8 up, 7 in 2024-08-29T15:15:20.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:20.146 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:20.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:20.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:20.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:20.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:20.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.399 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:20.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:20.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:20.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.523 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:20.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:20.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:20.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:20.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:20.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:20.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:20.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:20.891 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:20 smithi027 ceph-mon[98970]: osdmap e170: 8 total, 8 up, 7 in 2024-08-29T15:15:20.891 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:20 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3466247039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:20.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:20 smithi027 ceph-mon[93921]: osdmap e170: 8 total, 8 up, 7 in 2024-08-29T15:15:20.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:20 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3466247039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:20.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:20.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:21.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:20 smithi171 ceph-mon[90031]: osdmap e170: 8 total, 8 up, 7 in 2024-08-29T15:15:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:20 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3466247039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:21.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:21.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:21.068 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:15:21.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:15:21.070 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:15:21.070 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:15:21.070 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:15:21.070 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:15:21.070 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:15:21.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.160 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:21.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:21.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:21.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:21.388 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:21.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:21.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:21.537 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:21.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:21.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:21.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:21.772 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.2 to 1 (10000) 2024-08-29T15:15:21.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:21.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:21.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:21.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:22.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:21 smithi171 ceph-mon[90031]: pgmap v499: 89 pgs: 5 unknown, 84 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/151789246' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:15:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:21 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:15:22.031 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:22.034 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:22.042 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:22.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:22.047 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:21 smithi027 ceph-mon[98970]: pgmap v499: 89 pgs: 5 unknown, 84 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/151789246' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:15:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:21 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:15:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:21 smithi027 ceph-mon[93921]: pgmap v499: 89 pgs: 5 unknown, 84 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/151789246' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:15:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:21 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:15:22.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:22.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:22.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:22.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:22.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.423 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:22.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:22.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:22.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:22.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 3 1 2024-08-29T15:15:22.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:22.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:22.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:22.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:22.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:22.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:22.807 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:15:22.807 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:22 smithi027 ceph-mon[93921]: osdmap e171: 8 total, 8 up, 7 in 2024-08-29T15:15:22.807 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:15:22.807 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:22 smithi027 ceph-mon[98970]: osdmap e171: 8 total, 8 up, 7 in 2024-08-29T15:15:22.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:22.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:22.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:23.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:23.053 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:23.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:23.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:15:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:22 smithi171 ceph-mon[90031]: osdmap e171: 8 total, 8 up, 7 in 2024-08-29T15:15:23.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:23.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:23.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:23.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:23.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:23.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:23.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:23.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:23.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:23.772 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:23.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:15:23.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:23.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:23.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:23.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:23.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:24.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:24.053 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:23 smithi027 ceph-mon[93921]: pgmap v501: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:23 smithi027 ceph-mon[98970]: pgmap v501: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:24.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:24.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:23 smithi171 ceph-mon[90031]: pgmap v501: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:24.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:24.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:24.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:24.429 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:24.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:24.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:24.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:24.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:24.681 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:24.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:24.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:24.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:24.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:24.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:24.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:25.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:25.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:25.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:25.180 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:25.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:25.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:25.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:25.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:25.472 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:25.508 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:25.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:25.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:25.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:25.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:25.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:25.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:25.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:25.928 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:25 smithi027 ceph-mon[98970]: pgmap v502: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:25.928 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:25 smithi027 ceph-mon[93921]: pgmap v502: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:25.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:25.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:26.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:26.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:26.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:26.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:25 smithi171 ceph-mon[90031]: pgmap v502: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:26.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:26.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:26.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:26.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:26.440 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:26.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:26.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:26.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:26.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:26.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:26.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:26.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:26.819 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:26.921 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:26.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:26.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:27.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:27.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:27.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:27.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:27.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:27.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:27.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:27.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:27.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:27.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:27.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:27.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:27.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:27.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:27.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:27.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:27.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:27.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:27.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:27.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:27.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:27.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:28.006 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:27 smithi027 ceph-mon[93921]: pgmap v503: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:28.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:27 smithi027 ceph-mon[98970]: pgmap v503: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:28.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:28.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:28.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:28.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:28.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:28.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:28.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:27 smithi171 ceph-mon[90031]: pgmap v503: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:28.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:28.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:28.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:28.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:28.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:28.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:28.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:28.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:28.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:28.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:28.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:28.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:28.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:28.785 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:15:28.785 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:15:28.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:88170 5.0 5:0489ce30:::benchmark_data_smithi027_135490_object88169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:20.041904+0000", 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 248.73961076800001, 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.058206958000000003, 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.786 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88170 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.041904+0000", 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.041904+0000", 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.041905+0000", 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4100000000000001e-06 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.041951+0000", 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5380000000000003e-05 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.041968+0000", 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6877000000000001e-05 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.041976+0000", 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8620000000000003e-06 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042045+0000", 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9766999999999998e-05 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042199+0000", 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015374900000000001 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042375+0000", 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017628199999999999 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.045321+0000", 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029455499999999999 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.050865+0000", 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055445370000000004 2024-08-29T15:15:28.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.099674+0000", 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.048808919999999999 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.099960+0000", 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028573599999999998 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.100111+0000", 2024-08-29T15:15:28.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015088800000000001 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:88176 5.0 5:0c235365:::benchmark_data_smithi027_135490_object88175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:20.042547+0000", 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 248.73896756900001, 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.058057036999999999, 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.792 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88176 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042547+0000", 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042547+0000", 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042548+0000", 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.133e-06 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042579+0000", 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0889999999999997e-05 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042591+0000", 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.218e-05 2024-08-29T15:15:28.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.042599+0000", 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4440000000000002e-06 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.045443+0000", 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028438869999999998 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.045784+0000", 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034152500000000002 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.046269+0000", 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00048503799999999997 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.049602+0000", 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033328059999999998 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.051267+0000", 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0016652069999999999 2024-08-29T15:15:28.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.100278+0000", 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.049010625000000002 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.100478+0000", 2024-08-29T15:15:28.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019955999999999999 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:20.100604+0000", 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126742 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:166997 5.0 5:01b30b49:::benchmark_data_smithi027_135490_object33245:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:12:40.330903+0000", 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 168.450611796, 2024-08-29T15:15:28.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.066806192, 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166997 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.330903+0000", 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.330903+0000", 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.330903+0000", 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7800000000000002e-07 2024-08-29T15:15:28.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.330906+0000", 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2189999999999998e-06 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.330912+0000", 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1970000000000001e-06 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.330917+0000", 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9099999999999996e-06 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.343468+0000", 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012551659 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.343802+0000", 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033406899999999999 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.344214+0000", 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041118099999999999 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.349217+0000", 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050036159999999998 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.356739+0000", 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075217030000000002 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.397404+0000", 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.040664709 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.397580+0000", 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017662600000000001 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:40.397709+0000", 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128925 2024-08-29T15:15:28.804 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:181000 5.0 5:0823da65:::benchmark_data_smithi027_135490_object47248:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e128)", 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:12:57.110215+0000", 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 151.67130010899999, 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.056553184999999999, 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 181000 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110215+0000", 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110215+0000", 2024-08-29T15:15:28.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110217+0000", 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8339999999999999e-06 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110222+0000", 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5169999999999999e-06 2024-08-29T15:15:28.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110241+0000", 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8933e-05 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110250+0000", 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1759999999999999e-06 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110324+0000", 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3478000000000004e-05 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110473+0000", 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000149316 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.110589+0000", 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011582999999999999 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.117439+0000", 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068499299999999997 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.120487+0000", 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003048537 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.166189+0000", 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.045701277999999998 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.166606+0000", 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041708899999999999 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:12:57.166768+0000", 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016226700000000001 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:220339 5.0 5:04e0e11f:::benchmark_data_smithi027_135490_object86587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:15:28.811 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:13:43.509635+0000", 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 105.271879946, 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.066207295999999999, 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 220339 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.509635+0000", 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.509635+0000", 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.509637+0000", 2024-08-29T15:15:28.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:28.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.996e-06 2024-08-29T15:15:28.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.509643+0000", 2024-08-29T15:15:28.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4049999999999997e-06 2024-08-29T15:15:28.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.509701+0000", 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7367999999999997e-05 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.509710+0000", 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4900000000000006e-06 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520427+0000", 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010716656999999999 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520737+0000", 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000310375 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520851+0000", 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011372200000000001 2024-08-29T15:15:28.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.524851+0000", 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039997310000000003 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.536277+0000", 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011426433 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.575348+0000", 2024-08-29T15:15:28.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039070423999999999 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.575680+0000", 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033296999999999998 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.575842+0000", 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016172499999999999 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:220349 5.0 5:01efbbb8:::benchmark_data_smithi027_135490_object86597:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:13:43.520357+0000", 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 105.261157877, 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.056171353, 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 220349 2024-08-29T15:15:28.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520357+0000", 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520357+0000", 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520359+0000", 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5910000000000001e-06 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520364+0000", 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.95e-06 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520377+0000", 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3354e-05 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.520385+0000", 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9549999999999992e-06 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.525905+0000", 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055205289999999997 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.526196+0000", 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029038000000000002 2024-08-29T15:15:28.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.526537+0000", 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034086799999999998 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.540286+0000", 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013749152000000001 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.540357+0000", 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1727e-05 2024-08-29T15:15:28.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.576023+0000", 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.035665412000000001 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.576383+0000", 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035999900000000001 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:13:43.576528+0000", 2024-08-29T15:15:28.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014543600000000001 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237081 5.0 5:0dda9191:::benchmark_data_smithi027_135490_object103329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.797802+0000", 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 85.983713073000004, 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.113892862, 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.825 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237081 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797802+0000", 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797802+0000", 2024-08-29T15:15:28.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797807+0000", 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8189999999999999e-06 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797812+0000", 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9509999999999997e-06 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797860+0000", 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8785999999999997e-05 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797868+0000", 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6550000000000004e-06 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.828 INFO:teuthology.orchestra.run.smithi027.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-08-29T15:15:28.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797934+0000", 2024-08-29T15:15:28.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5526999999999998e-05 2024-08-29T15:15:28.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798064+0000", 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013022700000000001 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798183+0000", 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119543 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.813963+0000", 2024-08-29T15:15:28.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015779782999999999 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.814006+0000", 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2895999999999997e-05 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911577+0000", 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.097570961999999997 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911648+0000", 2024-08-29T15:15:28.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0757e-05 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911695+0000", 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6956000000000002e-05 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237085 5.0 5:02701778:::benchmark_data_smithi027_135490_object103333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.802468+0000", 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 85.979046590999999, 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.109798035, 2024-08-29T15:15:28.835 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237085 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802468+0000", 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802468+0000", 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802471+0000", 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5330000000000001e-06 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802480+0000", 2024-08-29T15:15:28.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8480000000000007e-06 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802503+0000", 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3691000000000001e-05 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802513+0000", 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9189999999999994e-06 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.803202+0000", 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00068899399999999996 2024-08-29T15:15:28.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.803556+0000", 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035358399999999999 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.803995+0000", 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00043872900000000001 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.814150+0000", 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010155229999999999 2024-08-29T15:15:28.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.817712+0000", 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035621939999999999 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911773+0000", 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.094060560000000001 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.912088+0000", 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031585599999999999 2024-08-29T15:15:28.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.912266+0000", 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017789699999999999 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237116 5.0 5:06a3e46a:::benchmark_data_smithi027_135490_object103364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.838860+0000", 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 85.942654564999998, 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.090343065, 2024-08-29T15:15:28.841 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237116 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838860+0000", 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838860+0000", 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838862+0000", 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.984e-06 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838870+0000", 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6639999999999998e-06 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838890+0000", 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9576999999999999e-05 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838898+0000", 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0779999999999996e-06 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838977+0000", 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9532999999999999e-05 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.839138+0000", 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016038099999999999 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.839305+0000", 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016747600000000001 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.843249+0000", 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039444700000000003 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.848136+0000", 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004886395 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.928934+0000", 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.080798306 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929076+0000", 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141834 2024-08-29T15:15:28.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929203+0000", 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127367 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237118 5.0 5:098193b5:::benchmark_data_smithi027_135490_object103366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.844332+0000", 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 85.937182802999999, 2024-08-29T15:15:28.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091090242000000002, 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237118 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844332+0000", 2024-08-29T15:15:28.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844332+0000", 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844334+0000", 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.669e-06 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844338+0000", 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6489999999999999e-06 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844352+0000", 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4025e-05 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844361+0000", 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5690000000000005e-06 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844418+0000", 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7178e-05 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844551+0000", 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133065 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844702+0000", 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015070400000000001 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.848497+0000", 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037951209999999998 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.849105+0000", 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00060838400000000001 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929264+0000", 2024-08-29T15:15:28.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.080158141000000002 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929399+0000", 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013499200000000001 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.935422+0000", 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060237449999999996 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.853 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:238120 5.0 5:0aaae9b3:::benchmark_data_smithi027_135490_object104368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:04.167351+0000", 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 84.614163958000006, 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.057527561999999997, 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.854 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 238120 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167351+0000", 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167351+0000", 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167353+0000", 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6649999999999999e-06 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167358+0000", 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.835e-06 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167372+0000", 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3164e-05 2024-08-29T15:15:28.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167380+0000", 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4130000000000006e-06 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167533+0000", 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015263700000000001 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167650+0000", 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117837 2024-08-29T15:15:28.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167787+0000", 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136666 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.172476+0000", 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046890609999999996 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.194978+0000", 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022501915000000001 2024-08-29T15:15:28.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.224758+0000", 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.029779415 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.224830+0000", 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2144000000000006e-05 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.224878+0000", 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8810000000000002e-05 2024-08-29T15:15:28.859 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:238123 5.0 5:01200163:::benchmark_data_smithi027_135490_object104371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:04.167404+0000", 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 84.614110737999994, 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.066944041999999995, 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.860 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 238123 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167404+0000", 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167404+0000", 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167405+0000", 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8999999999999997e-07 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167407+0000", 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.419e-06 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167414+0000", 2024-08-29T15:15:28.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1280000000000002e-06 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167420+0000", 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5720000000000002e-06 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.172509+0000", 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050888280000000001 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.176915+0000", 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044067070000000002 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.177054+0000", 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013846400000000001 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.186423+0000", 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093696539999999998 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.195002+0000", 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085784659999999999 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.234172+0000", 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039170476000000003 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.234298+0000", 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012538100000000001 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.234348+0000", 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0457000000000002e-05 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:238124 5.0 5:05450dfc:::benchmark_data_smithi027_135490_object104372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:04.167444+0000", 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 84.614070690000005, 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.067742327000000005, 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 238124 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167444+0000", 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167444+0000", 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167445+0000", 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0240000000000001e-06 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167450+0000", 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7439999999999998e-06 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167460+0000", 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0553999999999999e-05 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.167468+0000", 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7470000000000005e-06 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.181197+0000", 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013728304 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.181334+0000", 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137737 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.181963+0000", 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00062841299999999998 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.194906+0000", 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012943488 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.195028+0000", 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121406 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.234775+0000", 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039747748999999999 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.235034+0000", 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025823300000000002 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.235186+0000", 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015292800000000001 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.871 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:238135 5.0 5:081c09cb:::benchmark_data_smithi027_135490_object104383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:04.182100+0000", 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 84.599414569000004, 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053663611, 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.872 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 238135 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.182100+0000", 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.182100+0000", 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.182102+0000", 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5120000000000001e-06 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.182108+0000", 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6680000000000004e-06 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.182122+0000", 2024-08-29T15:15:28.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3839000000000001e-05 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.182132+0000", 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8530000000000004e-06 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.186459+0000", 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043265320000000001 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.186581+0000", 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012212599999999999 2024-08-29T15:15:28.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.186716+0000", 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013517999999999999 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.195061+0000", 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083448580000000001 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.195145+0000", 2024-08-29T15:15:28.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4407999999999996e-05 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.235387+0000", 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.040241682000000001 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.235616+0000", 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022949099999999999 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:04.235764+0000", 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014746200000000001 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:254191 5.0 5:0eccd30d:::benchmark_data_smithi027_135490_object120439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:32.934206+0000", 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 55.847308417000001, 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.056782553999999999, 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.878 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 254191 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934206+0000", 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934206+0000", 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934209+0000", 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1919999999999999e-06 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934216+0000", 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7700000000000001e-06 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934240+0000", 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3646e-05 2024-08-29T15:15:28.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934250+0000", 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0107e-05 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934420+0000", 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017022200000000001 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.934725+0000", 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00030502600000000002 2024-08-29T15:15:28.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.935069+0000", 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034384599999999997 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.940036+0000", 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049664690000000003 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.940143+0000", 2024-08-29T15:15:28.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010723800000000001 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.990618+0000", 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.050475258000000002 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.990811+0000", 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000193251 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:32.990989+0000", 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000177529 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:261220 5.0 5:08d9b25c:::benchmark_data_smithi027_135490_object127468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:40.689979+0000", 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 48.091536298999998, 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.057141554999999997, 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.884 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 261220 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.689979+0000", 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.689979+0000", 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.689979+0000", 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3699999999999999e-07 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.689984+0000", 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3549999999999998e-06 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.689994+0000", 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0390999999999999e-05 2024-08-29T15:15:28.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.690001+0000", 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6080000000000001e-06 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.694392+0000", 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043912129999999997 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.694513+0000", 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120876 2024-08-29T15:15:28.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.694628+0000", 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115204 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.703725+0000", 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090968119999999993 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.708158+0000", 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044333860000000001 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.746892+0000", 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.038733413000000001 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.747072+0000", 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018050600000000001 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.747120+0000", 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7954000000000002e-05 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:261223 5.0 5:0add0b69:::benchmark_data_smithi027_135490_object127471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:40.693741+0000", 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 48.087774232000001, 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.063121047999999999, 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.890 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 261223 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.693741+0000", 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.693741+0000", 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.693743+0000", 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.243e-06 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.693750+0000", 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0049999999999998e-06 2024-08-29T15:15:28.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.693766+0000", 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.63e-05 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.693775+0000", 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1390000000000001e-06 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.698937+0000", 2024-08-29T15:15:28.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005161873 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.699138+0000", 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000200571 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.699251+0000", 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011324600000000001 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.703747+0000", 2024-08-29T15:15:28.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044956359999999999 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.708263+0000", 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045165329999999997 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.756417+0000", 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.048154089999999997 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.756803+0000", 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038618099999999998 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.756862+0000", 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8230999999999999e-05 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:261253 5.0 5:055ac25f:::benchmark_data_smithi027_135490_object127501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:15:28.896 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:40.741952+0000", 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 48.039563186999999, 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.054355982999999997, 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 261253 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.741952+0000", 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.741952+0000", 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.741954+0000", 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9640000000000002e-06 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.741960+0000", 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6749999999999996e-06 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.741979+0000", 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8957000000000002e-05 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.741988+0000", 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.551e-06 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742331+0000", 2024-08-29T15:15:28.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034319999999999999 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742417+0000", 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5719999999999999e-05 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742488+0000", 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1136999999999993e-05 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.751433+0000", 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089447669999999993 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.795863+0000", 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.044430776999999998 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.796163+0000", 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029937300000000002 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.796247+0000", 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3776000000000006e-05 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.796308+0000", 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1086000000000004e-05 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:261256 5.0 5:0404550d:::benchmark_data_smithi027_135490_object127504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:15:28.903 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:40.742589+0000", 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 48.038925544999998, 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053921218, 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 261256 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742589+0000", 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742589+0000", 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742590+0000", 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4499999999999995e-07 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742594+0000", 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1930000000000003e-06 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742636+0000", 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1869999999999997e-05 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.742644+0000", 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0860000000000002e-06 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.747162+0000", 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045172040000000004 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.747263+0000", 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101135 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.747367+0000", 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010384 2024-08-29T15:15:28.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.756373+0000", 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090060009999999996 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.795923+0000", 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039550252000000001 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.796380+0000", 2024-08-29T15:15:28.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045707499999999998 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.796460+0000", 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0462000000000003e-05 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.796511+0000", 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0154999999999997e-05 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:261272 5.0 5:07549f0e:::benchmark_data_smithi027_135490_object127520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e154)", 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:40.766128+0000", 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 48.015386604, 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053819156999999999, 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 261272 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.910 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.766128+0000", 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.766128+0000", 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:15:28.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.766130+0000", 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.04e-06 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.766137+0000", 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9020000000000002e-06 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.766160+0000", 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2558e-05 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:15:28.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.766171+0000", 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.146e-05 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.798928+0000", 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.032756408000000001 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.799051+0000", 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012329600000000001 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:15:28.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.799175+0000", 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012382199999999999 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.813285+0000", 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014109768 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.813378+0000", 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3360000000000001e-05 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:28.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.819548+0000", 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061697330000000002 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.819790+0000", 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024214899999999999 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:40.819947+0000", 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015766099999999999 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.915 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.916 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:15:28.916 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:15:28.916 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:15:28.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:28.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:29.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:29.037 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/818020006' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:15:29.037 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/818020006' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]} : dispatch 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:29.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:29.039 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:15:29] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.51.0" 2024-08-29T15:15:29.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:29.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:29.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/818020006' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:15:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-29T15:15:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]} : dispatch 2024-08-29T15:15:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]} : dispatch 2024-08-29T15:15:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:29.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:29.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:29.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:29.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:29.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:29.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:29.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:29.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:29.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:29.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:29.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:29.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:29.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:29.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[98970]: pgmap v504: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/818020006' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[98970]: osdmap e172: 8 total, 8 up, 7 in 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[93921]: pgmap v504: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/818020006' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:15:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-29T15:15:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2024-08-29T15:15:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2024-08-29T15:15:29.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:29 smithi027 ceph-mon[93921]: osdmap e172: 8 total, 8 up, 7 in 2024-08-29T15:15:29.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:29.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:29.927 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:29.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:30.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:30.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:30.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:30.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:29 smithi171 ceph-mon[90031]: pgmap v504: 89 pgs: 89 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/818020006' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:15:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-29T15:15:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2024-08-29T15:15:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2024-08-29T15:15:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:29 smithi171 ceph-mon[90031]: osdmap e172: 8 total, 8 up, 7 in 2024-08-29T15:15:30.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:30.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:30.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:30.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:30.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:30.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:30.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:30.551 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:30.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:30.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:30.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:30.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:30.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:30.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:30.804 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.3 to 1 (10000) 2024-08-29T15:15:30.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:30.923 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:30.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:31.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:31.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:31.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:30 smithi027 ceph-mon[93921]: osdmap e173: 8 total, 8 up, 7 in 2024-08-29T15:15:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4097425699' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:15:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:15:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:30 smithi027 ceph-mon[98970]: osdmap e173: 8 total, 8 up, 7 in 2024-08-29T15:15:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4097425699' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:15:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:15:31.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:31.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:31.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:31.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:30 smithi171 ceph-mon[90031]: osdmap e173: 8 total, 8 up, 7 in 2024-08-29T15:15:31.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4097425699' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:15:31.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:15:31.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:31.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:31.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:31.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:31.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:31.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:31.520 DEBUG:teuthology.orchestra.run.smithi027:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-08-29T15:15:31.523 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-29T15:15:31.523 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-29T15:15:31.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:31.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:31.575 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:hints = 1 2024-08-29T15:15:31.576 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-29T15:15:31.576 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object prefix: benchmark_data_smithi027_169445 2024-08-29T15:15:31.576 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:15:31.576 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 0 0 0 0 0 0 - 0 2024-08-29T15:15:31.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:31.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:31.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:31.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:31.907 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[98970]: pgmap v507: 89 pgs: 1 activating, 2 peering, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:31.907 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:15:31.907 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[98970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:15:31.907 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:15:31.907 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[98970]: osdmap e174: 8 total, 8 up, 7 in 2024-08-29T15:15:31.907 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[93921]: pgmap v507: 89 pgs: 1 activating, 2 peering, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:31.908 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:15:31.908 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[93921]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:15:31.908 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:15:31.908 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:31 smithi027 ceph-mon[93921]: osdmap e174: 8 total, 8 up, 7 in 2024-08-29T15:15:31.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:31.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:31.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:32.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:32.075 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:32.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:32.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:32.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:32.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:31 smithi171 ceph-mon[90031]: pgmap v507: 89 pgs: 1 activating, 2 peering, 86 active+clean; 4.6 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail 2024-08-29T15:15:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:31 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:15:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:31 smithi171 ceph-mon[90031]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:15:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:15:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:31 smithi171 ceph-mon[90031]: osdmap e174: 8 total, 8 up, 7 in 2024-08-29T15:15:32.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 4 1 2024-08-29T15:15:32.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:32.338 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:32.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:32.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:32.470 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:15:32.470 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:15:32.516 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:15:32.516 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:15:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:32.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:32.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:32.554 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:15:32.555 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:15:32.576 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 1 16 3015 2999 187.427 187.438 0.0174474 0.00530938 2024-08-29T15:15:32.593 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:15:32.594 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:15:32.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:32.629 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:15:32.629 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:15:32.665 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:15:32.666 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:15:32.676 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:32.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:32.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:32.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:15:32.701 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:15:32.735 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:15:32.736 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:15:32.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:32.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:32.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:32.829 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:15:32.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:32.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:32.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:32.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:15:33.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3295696464' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2651397781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:15:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3295696464' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2651397781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:32 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:15:33.097 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:33.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:33.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:33.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:33.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3295696464' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:32 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:15:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2651397781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:32 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:15:33.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:33.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:33.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:33.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:33.503 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:33.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:33.576 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 2 16 6066 6050 189.049 190.688 0.00311703 0.00527846 2024-08-29T15:15:33.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:33.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:33.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:33.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:33.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:33.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:33.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:33.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:33.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:33.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:34.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:34.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:34.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:34.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[93921]: pgmap v509: 89 pgs: 1 activating, 2 peering, 86 active+clean; 7.1 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail; 426 KiB/s wr, 6 op/s 2024-08-29T15:15:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:15:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:15:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[93921]: osdmap e175: 8 total, 8 up, 7 in 2024-08-29T15:15:34.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[98970]: pgmap v509: 89 pgs: 1 activating, 2 peering, 86 active+clean; 7.1 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail; 426 KiB/s wr, 6 op/s 2024-08-29T15:15:34.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:15:34.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:15:34.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:33 smithi027 ceph-mon[98970]: osdmap e175: 8 total, 8 up, 7 in 2024-08-29T15:15:34.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:34.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:34.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:33 smithi171 ceph-mon[90031]: pgmap v509: 89 pgs: 1 activating, 2 peering, 86 active+clean; 7.1 MiB data, 1.0 GiB used, 625 GiB / 626 GiB avail; 426 KiB/s wr, 6 op/s 2024-08-29T15:15:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:33 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:15:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:33 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:15:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:33 smithi171 ceph-mon[90031]: osdmap e175: 8 total, 8 up, 7 in 2024-08-29T15:15:34.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:34.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:34.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:34.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:34.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:34.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:34.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:34.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:34.577 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 3 16 8991 8975 186.947 182.812 0.00809971 0.00533886 2024-08-29T15:15:34.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:34.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:34.613 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:34.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:34.745 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:34.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:34.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:34.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:34.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:34.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:34.986 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:15:34.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:35.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:34 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:15:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:34 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:15:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:34 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:15:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:34 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:15:35.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:35.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:35.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:35.162 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:34 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:15:35.162 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:34 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:15:35.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:35.286 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:35.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:35.419 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:35.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:35.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:35.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:35.577 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 4 16 11646 11630 181.692 165.938 0.0059121 0.00549645 2024-08-29T15:15:35.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:35.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:35.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:35.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:35.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:35.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:35.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:35.983 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:35.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:36.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:15:36.062 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:35 smithi027 ceph-mon[93921]: pgmap v511: 89 pgs: 1 activating, 88 active+clean; 90 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 15 MiB/s wr, 241 op/s 2024-08-29T15:15:36.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:35 smithi027 ceph-mon[98970]: pgmap v511: 89 pgs: 1 activating, 88 active+clean; 90 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 15 MiB/s wr, 241 op/s 2024-08-29T15:15:36.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:36.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:36.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:36.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:36.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:36.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:36.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:35 smithi171 ceph-mon[90031]: pgmap v511: 89 pgs: 1 activating, 88 active+clean; 90 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 15 MiB/s wr, 241 op/s 2024-08-29T15:15:36.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:36.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:36.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:36.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:36.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:15:36.555 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:36.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:36.577 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 5 16 14345 14329 179.083 168.688 0.00476673 0.00557614 2024-08-29T15:15:36.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:36.687 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:36.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:36.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:36.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:36.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:36.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:36.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:36.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:36.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:37.077 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:37.107 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:37.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:37.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:37.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:37.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:37.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:37.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:37.375 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:37.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:37.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:37.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:37.500 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:15:37.500 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:15:37.518 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:37.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:37.577 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 6 16 16752 16736 174.307 150.438 0.00526282 0.00573081 2024-08-29T15:15:37.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:37.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:37.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:37.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:37.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:37.778 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:37.799 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:37.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:37.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:37.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:37.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:37 smithi027 ceph-mon[93921]: pgmap v512: 89 pgs: 89 active+clean; 472 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:15:37.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:37.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:37.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:37.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:38.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:38.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:38.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:38.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:38.185 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:37 smithi171 ceph-mon[90031]: pgmap v512: 89 pgs: 89 active+clean; 472 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:15:38.202 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:37 smithi027 ceph-mon[98970]: pgmap v512: 89 pgs: 89 active+clean; 472 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:15:38.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:38.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:38.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:38.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:15:38.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:38.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:38.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:38.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:38.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:38.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:38.577 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 7 16 18983 18967 169.324 139.438 0.00714924 0.00589867 2024-08-29T15:15:38.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:38.685 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:38.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:38.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:38.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:38.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:38.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:38.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:38.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:38.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:39.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:39.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:15:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:15:39] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.51.0" 2024-08-29T15:15:39.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:39.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:39.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:39.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:39.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:39.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:39.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:39.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:39.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:39.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:39.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:39.577 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 8 16 20900 20884 163.134 119.812 0.00524586 0.00612382 2024-08-29T15:15:39.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:39.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:39.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:39.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:39.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:39.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:39.897 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.4 to 1 (10000) 2024-08-29T15:15:39.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:39.930 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:39.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:40.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:40.054 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:40.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:40.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:39 smithi171 ceph-mon[90031]: pgmap v513: 89 pgs: 89 active+clean; 635 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-29T15:15:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:39 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1063488058' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:15:40.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:40.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:39 smithi027 ceph-mon[98970]: pgmap v513: 89 pgs: 89 active+clean; 635 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-29T15:15:40.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:39 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1063488058' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:15:40.300 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:39 smithi027 ceph-mon[93921]: pgmap v513: 89 pgs: 89 active+clean; 635 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-29T15:15:40.300 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:39 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1063488058' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:15:40.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:40.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:40.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:40.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:40.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:40.578 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 9 16 22647 22631 157.125 109.188 0.00608773 0.00635601 2024-08-29T15:15:40.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:40.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:40.730 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:40.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:40.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:40.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:40.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:40.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:40.993 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:40.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:40.996 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:15:41.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1063488058' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:15:41.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:40 smithi027 ceph-mon[98970]: osdmap e176: 8 total, 8 up, 7 in 2024-08-29T15:15:41.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1063488058' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:15:41.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:40 smithi027 ceph-mon[93921]: osdmap e176: 8 total, 8 up, 7 in 2024-08-29T15:15:41.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:41.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:41.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:41.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:41.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:41.223 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1063488058' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:15:41.223 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:40 smithi171 ceph-mon[90031]: osdmap e176: 8 total, 8 up, 7 in 2024-08-29T15:15:41.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:41.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:41.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:41.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:41.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:41.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:41.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:41.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:41.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:41.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:41.578 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 10 16 24075 24059 150.338 89.25 0.00936192 0.00664592 2024-08-29T15:15:41.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:41.673 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:41.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:41.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:41.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:41.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:41.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:41.921 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:41.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:42.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:42.053 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:42.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:42.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:42.162 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:41 smithi027 ceph-mon[98970]: pgmap v515: 89 pgs: 89 active+clean; 1.1 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-08-29T15:15:42.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:41 smithi027 ceph-mon[93921]: pgmap v515: 89 pgs: 89 active+clean; 1.1 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-08-29T15:15:42.164 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:41 smithi171 ceph-mon[90031]: pgmap v515: 89 pgs: 89 active+clean; 1.1 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-08-29T15:15:42.196 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:42.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:42.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:42.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:42.472 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:15:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=sqlstore.transactions t=2024-08-29T15:15:42.162505397Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-29T15:15:42.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:42.578 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 11 16 25659 25643 145.67 99 0.00444962 0.00685854 2024-08-29T15:15:42.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:42.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:42.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:42.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:42.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:42.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:42.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:42.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:42.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:42.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:43.093 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:43.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:43.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:43.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:43.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:43.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:43.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:43.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:43.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:43.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:43.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:43.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:43.578 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 12 16 27418 27402 142.692 109.938 0.00387204 0.0070016 2024-08-29T15:15:43.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:43.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:43.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:43.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:43.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:43.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:43.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:43.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:43.936 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:43.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:43.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:43.998 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:43 smithi171 ceph-mon[90031]: pgmap v516: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-08-29T15:15:43.998 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:43.999 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:44.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:44.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:44.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:44.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:44.136 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:15:44.137 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:15:44.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:44.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:44.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:44.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:43 smithi027 ceph-mon[98970]: pgmap v516: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-08-29T15:15:44.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:44.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:44.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:43 smithi027 ceph-mon[93921]: pgmap v516: 89 pgs: 89 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-08-29T15:15:44.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:44.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:15:44.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:44.352 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:44.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:44.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:44.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:44.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:44.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:44.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:44.578 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 13 16 29278 29262 140.657 116.25 0.00387132 0.00710249 2024-08-29T15:15:44.586 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:44.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:44.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:44.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:44.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:44.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:44.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:44.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 7 1 2024-08-29T15:15:44.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:44.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:44.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:44.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:45.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:45.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:45.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:45.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:45.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:45.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:45.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:45.157 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:45.157 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:45.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:15:45.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:45.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:45.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:45.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:45.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:45.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:45.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:45.518 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:15:45.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:45.578 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 14 16 31293 31277 139.605 125.938 0.0114747 0.00715753 2024-08-29T15:15:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:45.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:45.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:45.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:45.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:45.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:45.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:45.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:45.877 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:45.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:45.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:46.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:46.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:45 smithi027 ceph-mon[93921]: pgmap v517: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-08-29T15:15:46.050 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:45 smithi027 ceph-mon[98970]: pgmap v517: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-08-29T15:15:46.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.141 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:46.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:46.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:45 smithi171 ceph-mon[90031]: pgmap v517: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 620 GiB / 626 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-08-29T15:15:46.285 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:46.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:46.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:46.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:46.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:46.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:46.579 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 15 16 33107 33091 137.856 113.375 0.00547336 0.00724776 2024-08-29T15:15:46.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:46.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:46.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:46.775 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:15:46.775 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:15:46.784 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:46.791 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:15:46.791 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:15:46.803 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:46.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:46.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:46.840 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:15:46.841 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:15:46.885 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:15:46.886 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:15:46.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:46.918 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:15:46.918 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:15:46.958 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:15:46.959 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:15:46.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:46.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:46.996 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:15:46.996 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:15:47.036 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:15:47.037 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:15:47.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:47.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:47.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:47.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:47.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:47.210 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1825923211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:47.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:47.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:47.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1825923211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1825923211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:47.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:47.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:47.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:47.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:47.534 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:47.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:47.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:47.579 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 16 16 34851 34835 136.052 109 0.0103564 0.00734463 2024-08-29T15:15:47.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:47.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:47.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:47.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:47.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:47.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:47.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:47.863 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:15:47.863 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:15:47.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:15:47.863 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:23215 5.e 5:7c274c1d:::benchmark_data_smithi027_135490_object23214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 306.89890828599999, 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13346666099999999, 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 23215 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.864 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960452+0000", 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960487+0000", 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4594000000000001e-05 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960497+0000", 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0348e-05 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960504+0000", 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6660000000000002e-06 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960556+0000", 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2392999999999998e-05 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960676+0000", 2024-08-29T15:15:47.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119975 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960812+0000", 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013557900000000001 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.961708+0000", 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00089608400000000001 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.962192+0000", 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00048372900000000002 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093816+0000", 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13162422100000001 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093876+0000", 2024-08-29T15:15:47.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9784000000000002e-05 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093920+0000", 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4129999999999999e-05 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68906 5.e 5:73dcaf78:::benchmark_data_smithi027_135490_object68905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:15:47.870 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 282.950882745, 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14042518300000001, 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68906 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:15:47.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908477+0000", 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908551+0000", 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3900999999999996e-05 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908563+0000", 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1518999999999999e-05 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908570+0000", 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0559999999999999e-06 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908679+0000", 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010953700000000001 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908808+0000", 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012906199999999999 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908966+0000", 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000158018 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913092+0000", 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041260869999999996 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.916369+0000", 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032766510000000002 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048843+0000", 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13247402 2024-08-29T15:15:47.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048878+0000", 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5175000000000001e-05 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048904+0000", 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5653e-05 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68910 5.e 5:7bc74f0d:::benchmark_data_smithi027_135490_object68909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 282.94663581999998, 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13951414700000001, 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.877 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68910 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912724+0000", 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912763+0000", 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9082999999999998e-05 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912777+0000", 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4174e-05 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912784+0000", 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0419999999999997e-06 2024-08-29T15:15:47.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913119+0000", 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00033500199999999999 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913247+0000", 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012835799999999999 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913411+0000", 2024-08-29T15:15:47.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016349600000000001 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.916432+0000", 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030208549999999998 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.917924+0000", 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014924789999999999 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052043+0000", 2024-08-29T15:15:47.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13411882899999999 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052180+0000", 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013707500000000001 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052240+0000", 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9765999999999997e-05 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71204 5.e 5:72845dde:::benchmark_data_smithi027_135490_object71203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 281.02270682099999, 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10090418, 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71204 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.884 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836653+0000", 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836686+0000", 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3639000000000001e-05 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836700+0000", 2024-08-29T15:15:47.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3372e-05 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836715+0000", 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5500000000000001e-05 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836792+0000", 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7065999999999997e-05 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836904+0000", 2024-08-29T15:15:47.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111976 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.837024+0000", 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119389 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891058+0000", 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.054034182 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891105+0000", 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6613999999999998e-05 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937443+0000", 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046338468000000001 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937523+0000", 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0029000000000002e-05 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937559+0000", 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5862999999999999e-05 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71206 5.e 5:74b85a3d:::benchmark_data_smithi027_135490_object71205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 281.02080884200001, 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.099155491999999998, 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.890 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71206 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:15:47.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838551+0000", 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838623+0000", 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2087000000000003e-05 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.892 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838635+0000", 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.164e-05 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838642+0000", 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7109999999999998e-06 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839510+0000", 2024-08-29T15:15:47.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00086809300000000001 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839722+0000", 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021236700000000001 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839853+0000", 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013045999999999999 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891096+0000", 2024-08-29T15:15:47.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051243155999999998 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891152+0000", 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5998000000000002e-05 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937606+0000", 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046454595000000001 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937672+0000", 2024-08-29T15:15:47.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5773999999999997e-05 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937708+0000", 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5984e-05 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71207 5.e 5:712e06d6:::benchmark_data_smithi027_135490_object71206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 281.01978661800001, 2024-08-29T15:15:47.896 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.101516334, 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71207 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839574+0000", 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839633+0000", 2024-08-29T15:15:47.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8607999999999999e-05 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839644+0000", 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.187e-05 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839652+0000", 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1910000000000003e-06 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844029+0000", 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004377148 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844134+0000", 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010534100000000001 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844266+0000", 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000132223 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891145+0000", 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046878523999999998 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891156+0000", 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1148000000000001e-05 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.940954+0000", 2024-08-29T15:15:47.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049798430999999997 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.941041+0000", 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7043e-05 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.941091+0000", 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9765999999999998e-05 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.902 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82159 5.1 5:8a273d61:::benchmark_data_smithi027_135490_object82158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 272.54362508200001, 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098041845000000002, 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.903 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82159 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315735+0000", 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315774+0000", 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8905000000000001e-05 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315787+0000", 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.288e-05 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315794+0000", 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9639999999999997e-06 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322730+0000", 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0069358759999999997 2024-08-29T15:15:47.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322849+0000", 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119078 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323562+0000", 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00071293799999999998 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329649+0000", 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060869790000000002 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329680+0000", 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1455999999999998e-05 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413675+0000", 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083995138999999996 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413734+0000", 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9247999999999999e-05 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413778+0000", 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.384e-05 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.909 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82163 5.1 5:8a4e1f99:::benchmark_data_smithi027_135490_object82162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 272.54268726100003, 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100377148, 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82163 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.910 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316673+0000", 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:47.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316702+0000", 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8294e-05 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316719+0000", 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7774e-05 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316726+0000", 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9430000000000003e-06 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326240+0000", 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0095137480000000007 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326418+0000", 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017811300000000001 2024-08-29T15:15:47.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326530+0000", 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111513 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335223+0000", 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086934460000000005 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413816+0000", 2024-08-29T15:15:47.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078592598999999999 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416754+0000", 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029382990000000001 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416879+0000", 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012490299999999999 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417051+0000", 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000172492 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82171 5.9 e118/111 5:952386eb:::benchmark_data_smithi027_135490_object82170:head v 118'5035, mlcod=118'5031)", 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 272.53594068699999, 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093309638, 2024-08-29T15:15:47.916 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323420+0000", 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323447+0000", 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.656e-05 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323458+0000", 2024-08-29T15:15:47.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1242e-05 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323464+0000", 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5130000000000002e-06 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323833+0000", 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00036861199999999999 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323850+0000", 2024-08-29T15:15:47.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6948000000000001e-05 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416578+0000", 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.092727863999999993 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416730+0000", 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015279099999999999 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:47.922 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:220847 5.e 5:70ceb708:::benchmark_data_smithi027_135490_object87095:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:15:47.922 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:15:47.922 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 123.78603044099999, 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095839788999999995, 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 220847 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073333+0000", 2024-08-29T15:15:47.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2689999999999998e-06 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073338+0000", 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4009999999999996e-06 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073342+0000", 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0799999999999999e-06 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.077819+0000", 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044769440000000001 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.077934+0000", 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000115091 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.078049+0000", 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000114052 2024-08-29T15:15:47.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.082327+0000", 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042779640000000004 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.086356+0000", 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040299300000000001 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169109+0000", 2024-08-29T15:15:47.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082753023999999994 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169144+0000", 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5011000000000001e-05 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169171+0000", 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6423000000000001e-05 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239381 5.e 5:711fbf86:::benchmark_data_smithi027_135490_object105629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 102.145290339, 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.21797998, 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239381 2024-08-29T15:15:47.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714074+0000", 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.54e-06 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714081+0000", 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8430000000000004e-06 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714086+0000", 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6070000000000001e-06 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714321+0000", 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000235135 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714648+0000", 2024-08-29T15:15:47.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00032731600000000002 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.715035+0000", 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000386602 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728628+0000", 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013593190999999999 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728671+0000", 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2849000000000003e-05 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755904+0000", 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027233362000000001 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755912+0000", 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9650000000000008e-06 2024-08-29T15:15:47.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693075+0000", 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.937162461 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693084+0000", 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3929999999999997e-06 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.931934+0000", 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2388502020000001 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932051+0000", 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117007 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239384 5.e 5:760d1bed:::benchmark_data_smithi027_135490_object105632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:15:47.936 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 102.130885112, 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2039694540000001, 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239384 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728475+0000", 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728482+0000", 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5410000000000003e-06 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728496+0000", 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4047e-05 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728504+0000", 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3010000000000001e-06 2024-08-29T15:15:47.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728714+0000", 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021033 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728818+0000", 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000103436 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728983+0000", 2024-08-29T15:15:47.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016501800000000001 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755958+0000", 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026975717 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755964+0000", 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2480000000000004e-06 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693203+0000", 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.937239665 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693213+0000", 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7189999999999996e-06 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932372+0000", 2024-08-29T15:15:47.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2391595259999999 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932446+0000", 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3459000000000003e-05 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239387 5.e 5:742a0183:::benchmark_data_smithi027_135490_object105635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 102.130849024, 2024-08-29T15:15:47.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.204054685, 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239387 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728512+0000", 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728515+0000", 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5450000000000002e-06 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728521+0000", 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.703e-06 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728526+0000", 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4220000000000002e-06 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.732840+0000", 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043146590000000002 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.733088+0000", 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00024788399999999997 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.733205+0000", 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011640400000000001 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755994+0000", 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022789411999999998 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.756004+0000", 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7180000000000008e-06 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693242+0000", 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93723825599999999 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693251+0000", 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9320000000000002e-06 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932479+0000", 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.239228132 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932567+0000", 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8008000000000002e-05 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239382 5.f 5:fda737cc:::benchmark_data_smithi027_135490_object105630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 102.111210418, 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2015833570000001, 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:15:47.950 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239382 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:15:47.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748149+0000", 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748157+0000", 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.109e-06 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748168+0000", 2024-08-29T15:15:47.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1785e-05 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748177+0000", 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6270000000000006e-06 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748229+0000", 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2408999999999999e-05 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748239+0000", 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9629999999999993e-06 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693049+0000", 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.944809695 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693062+0000", 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.29e-05 2024-08-29T15:15:47.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.947700+0000", 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.254637813 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949734+0000", 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020346959999999999 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239402 5.f 5:f2c183c5:::benchmark_data_smithi027_135490_object105650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 102.100418413, 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2791852029999999, 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239402 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.956 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758942+0000", 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758947+0000", 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5459999999999999e-06 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758958+0000", 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0305e-05 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758964+0000", 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0689999999999998e-06 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758998+0000", 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4258999999999998e-05 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.759005+0000", 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5520000000000001e-06 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693436+0000", 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93443171199999997 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693448+0000", 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1666e-05 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949915+0000", 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2564665989999999 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.950510+0000", 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00059576300000000002 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5,6", 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.950770+0000", 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00025979900000000001 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980049+0000", 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029278985 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980078+0000", 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9043e-05 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.981086+0000", 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0010076250000000001 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038008+0000", 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.056922683000000002 2024-08-29T15:15:47.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038051+0000", 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2484000000000002e-05 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038128+0000", 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7443999999999999e-05 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.963 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239405 5.e 5:7ac1943d:::benchmark_data_smithi027_135490_object105653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 99.927094435000001, 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.149048244, 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239405 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932265+0000", 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932278+0000", 2024-08-29T15:15:47.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2337999999999999e-05 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932292+0000", 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3723000000000001e-05 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932301+0000", 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8660000000000001e-06 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932591+0000", 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00028943000000000003 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932743+0000", 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015197400000000001 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,2,5", 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.933306+0000", 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00056278000000000003 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.941711+0000", 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0084057430000000002 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.941847+0000", 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136174 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.955062+0000", 2024-08-29T15:15:47.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013214889 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081140+0000", 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12607742999999999 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081195+0000", 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4777000000000001e-05 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081315+0000", 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012078000000000001 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239409 5.f 5:f1f503e2:::benchmark_data_smithi027_135490_object105657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 99.914432585, 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093393523000000006, 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.970 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239409 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:15:47.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944927+0000", 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944936+0000", 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3550000000000005e-06 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944950+0000", 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3862e-05 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944956+0000", 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3049999999999998e-06 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.976857+0000", 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031901438999999997 2024-08-29T15:15:47.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.976996+0000", 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000138603 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5,6", 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.977150+0000", 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000154559 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980087+0000", 2024-08-29T15:15:47.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029361729999999998 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980644+0000", 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00055748400000000002 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.981163+0000", 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00051858500000000001 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038192+0000", 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.057029253000000002 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038240+0000", 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7635e-05 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038322+0000", 2024-08-29T15:15:47.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2867999999999993e-05 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:242790 5.e 5:7ba30800:::benchmark_data_smithi027_135490_object109038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 95.003232816999997, 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100980449, 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:47.977 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242790 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856128+0000", 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856134+0000", 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.669e-06 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856145+0000", 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0298e-05 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856150+0000", 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2220000000000001e-06 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856187+0000", 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7452e-05 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856270+0000", 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2522999999999996e-05 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,2,5", 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856349+0000", 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9153000000000006e-05 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.859142+0000", 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027929640000000002 2024-08-29T15:15:47.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.859413+0000", 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00027162699999999998 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954345+0000", 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.094931353999999996 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956931+0000", 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025865459999999999 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956981+0000", 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9325e-05 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.957109+0000", 2024-08-29T15:15:47.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012847100000000001 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:242790 5.e e143/140 ondisk, result = 0)", 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 94.998780021000002, 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093766336000000006, 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860581+0000", 2024-08-29T15:15:47.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860586+0000", 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9599999999999999e-06 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954254+0000", 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093668543000000007 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954271+0000", 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6373999999999999e-05 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954325+0000", 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4342999999999998e-05 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954337+0000", 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1558e-05 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954348+0000", 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1318000000000001e-05 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:242791 5.7 e143/140 5:e93ed9d7:::benchmark_data_smithi027_135490_object109039:head v 143'15344, mlcod=143'15341)", 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 94.993373382000001, 2024-08-29T15:15:47.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.120497535, 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:47.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865987+0000", 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865994+0000", 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8990000000000004e-06 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954486+0000", 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088492092999999994 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954496+0000", 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5550000000000008e-06 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954810+0000", 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031430999999999999 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954830+0000", 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0162000000000001e-05 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.986452+0000", 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031621761999999998 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.986486+0000", 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3630000000000002e-05 2024-08-29T15:15:47.992 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.993 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.993 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.993 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:47.993 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:47.993 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:15:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:47.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:47.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:47.995 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:15:48.096 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:48.126 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:48.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:48.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:48.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:48.247 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:47 smithi171 ceph-mon[90031]: pgmap v518: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-29T15:15:48.247 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1447230367' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:15:48.247 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:47 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:15:48.266 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:48.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:48.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:47 smithi027 ceph-mon[93921]: pgmap v518: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-29T15:15:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1447230367' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:15:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:47 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:15:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:47 smithi027 ceph-mon[98970]: pgmap v518: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-29T15:15:48.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1447230367' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:15:48.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:47 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:15:48.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:48.410 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:48.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:48.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:48.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:48.555 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:48.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:48.579 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 17 16 36754 36738 135.045 118.938 0.00504079 0.00739928 2024-08-29T15:15:48.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:48.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:48.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:48.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:15:48.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:48.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:48.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:48.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:48.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:48.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:48.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:49.034 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:15:49.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:49.130 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:49.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:49.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:49.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:49 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:49 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:15:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:49 smithi171 ceph-mon[90031]: osdmap e177: 8 total, 8 up, 7 in 2024-08-29T15:15:49.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:49.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:49.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:15:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:15:49] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.51.0" 2024-08-29T15:15:49.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:49 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:49.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:49 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:15:49.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:49 smithi027 ceph-mon[93921]: osdmap e177: 8 total, 8 up, 7 in 2024-08-29T15:15:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:49 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:15:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:49 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:15:49.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:49 smithi027 ceph-mon[98970]: osdmap e177: 8 total, 8 up, 7 in 2024-08-29T15:15:49.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:49.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:49.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:49.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:49.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:49.537 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:49.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:49.579 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 18 16 38623 38607 134.031 116.812 0.00725811 0.00745496 2024-08-29T15:15:49.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:49.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:49.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:49.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:49.799 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:49.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:49.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:49.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:49.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:15:50.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:50.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:50.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:50.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:50.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:50.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:50 smithi171 ceph-mon[90031]: pgmap v520: 89 pgs: 89 active+clean; 1.8 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:15:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:50 smithi027 ceph-mon[93921]: pgmap v520: 89 pgs: 89 active+clean; 1.8 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:15:50.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:50 smithi027 ceph-mon[98970]: pgmap v520: 89 pgs: 89 active+clean; 1.8 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:15:50.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:50.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:50.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:50.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:50.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:50.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:50.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:50.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:50.583 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 19 16 40340 40324 132.597 107.312 0.0115529 0.00753476 2024-08-29T15:15:50.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:50.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:50.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:50.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:50.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:50.812 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:50.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:50.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:50.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:50.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:50.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:51.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:51.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:51.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:51.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:51.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:51.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:51.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:51.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:51.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:51.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:51.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:51.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:51.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:51.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:51.583 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:15:51.583157+0000 min lat: 0.00129485 max lat: 0.174854 avg lat: 0.00764425 2024-08-29T15:15:51.583 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:15:51.583 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 20 16 41857 41841 130.708 94.8125 0.00804193 0.00764425 2024-08-29T15:15:51.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:51.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:51.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:51.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:51.652 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:15:51.652 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:41958 6.7 6:ec296062:::benchmark_data_smithi027_169445_object41957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:15:51.640783+0000", 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.010429414, 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010454, 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:15:51.653 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 41958 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.640783+0000", 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.640783+0000", 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.640782+0000", 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.640867+0000", 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5403999999999997e-05 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.640878+0000", 2024-08-29T15:15:51.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.041e-05 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.640989+0000", 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000110812 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:51.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641035+0000", 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6573000000000003e-05 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641175+0000", 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013984199999999999 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641333+0000", 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015797200000000001 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.646643+0000", 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005310431 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.648573+0000", 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019300260000000001 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.658 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:41960 6.7 6:e4acfb9b:::benchmark_data_smithi027_169445_object41959:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:15:51.641906+0000", 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0093064619999999997, 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0093848809999999994, 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 41960 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:51.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641906+0000", 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641906+0000", 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641905+0000", 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641937+0000", 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1238999999999997e-05 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641978+0000", 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1289999999999999e-05 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641985+0000", 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2239999999999998e-06 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642913+0000", 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00092824599999999998 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643012+0000", 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8515000000000001e-05 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643126+0000", 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000114004 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.648583+0000", 2024-08-29T15:15:51.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0054568350000000002 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.648621+0000", 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8087999999999999e-05 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:41961 6.d 6:b73ca857:::benchmark_data_smithi027_169445_object41960:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:15:51.641993+0000", 2024-08-29T15:15:51.664 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0092197370000000004, 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0093519569999999993, 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 41961 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641993+0000", 2024-08-29T15:15:51.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641993+0000", 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.641992+0000", 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642032+0000", 2024-08-29T15:15:51.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9536999999999998e-05 2024-08-29T15:15:51.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:51.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642041+0000", 2024-08-29T15:15:51.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6889999999999993e-06 2024-08-29T15:15:51.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:51.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642046+0000", 2024-08-29T15:15:51.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7219999999999996e-06 2024-08-29T15:15:51.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:51.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642184+0000", 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013805800000000001 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642302+0000", 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117225 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642439+0000", 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013741900000000001 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.644082+0000", 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016433559999999999 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.644796+0000", 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00071399799999999998 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:41963 6.7 6:e7d71fd1:::benchmark_data_smithi027_169445_object41962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:15:51.642059+0000", 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0091540159999999992, 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0093352360000000002, 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 41963 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.671 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642059+0000", 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642059+0000", 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:51.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642058+0000", 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642086+0000", 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7861999999999999e-05 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:51.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642094+0000", 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0260000000000007e-06 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.642099+0000", 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.006e-06 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.645050+0000", 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002950991 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.645163+0000", 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011303600000000001 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.645285+0000", 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012247000000000001 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.648587+0000", 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033018520000000001 2024-08-29T15:15:51.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.649999+0000", 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014115410000000001 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:41966 6.7 6:e1974c62:::benchmark_data_smithi027_169445_object41965:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:15:51.643732+0000", 2024-08-29T15:15:51.676 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0074811449999999998, 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007705847, 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 41966 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:15:51.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643732+0000", 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643732+0000", 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643730+0000", 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643790+0000", 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9443e-05 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643806+0000", 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6753999999999999e-05 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.643814+0000", 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8299999999999996e-06 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.646667+0000", 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028532760000000001 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.646808+0000", 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014078800000000001 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.646962+0000", 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015385900000000001 2024-08-29T15:15:51.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.648776+0000", 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018135580000000001 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:15:51.650069+0000", 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012937129999999999 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.681 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:15:51.682 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:15:51.682 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 5 2024-08-29T15:15:51.682 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:15:51.724 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:51.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:51.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:51.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:51.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:51.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:51.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:51.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:51.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:52.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:15:52.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:52.095 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:52.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:52.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:52.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:52.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:52.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:52.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:52.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:52 smithi027 ceph-mon[93921]: pgmap v521: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-29T15:15:52.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:52 smithi027 ceph-mon[98970]: pgmap v521: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-29T15:15:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:52 smithi171 ceph-mon[90031]: pgmap v521: 89 pgs: 89 active+clean; 2.2 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-29T15:15:52.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:52.394 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:52.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:52.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:52.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:15:52.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:52.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:52.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:52.583 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 21 16 43353 43337 128.936 93.5 0.00629694 0.00775 2024-08-29T15:15:52.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:52.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:52.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:52.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:52.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:52.798 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:15:52.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:52.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:52.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:52.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:52.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:52.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:53.018 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.7 to 1 (10000) 2024-08-29T15:15:53.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:53.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:53.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:53.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:53.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:53.210 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3015128706' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:15:53.210 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:53 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:15:53.250 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:53.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:53.283 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3015128706' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:15:53.283 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:53 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:15:53.283 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3015128706' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:15:53.283 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:53 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:15:53.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:53.404 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:53.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:53.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:53.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:53.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:53.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:53.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:53.583 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 22 16 45047 45031 127.888 105.875 0.00366309 0.00781343 2024-08-29T15:15:53.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:53.688 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:53.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:53.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:53.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:53.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:53.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:53.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:53.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:53.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:54.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:54.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:54.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:54.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:54.238 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:54.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:54.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:54.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[93921]: pgmap v522: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:15:54.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:15:54.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[93921]: osdmap e178: 8 total, 8 up, 7 in 2024-08-29T15:15:54.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:54.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:54.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[98970]: pgmap v522: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:15:54.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:15:54.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[98970]: osdmap e178: 8 total, 8 up, 7 in 2024-08-29T15:15:54.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:54.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:54.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:54.376 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:54 smithi171 ceph-mon[90031]: pgmap v522: 89 pgs: 89 active+clean; 2.4 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:15:54.376 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:54 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:15:54.377 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:54 smithi171 ceph-mon[90031]: osdmap e178: 8 total, 8 up, 7 in 2024-08-29T15:15:54.377 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:54.377 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:54.383 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:54.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:54.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:54.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:54.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:54.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:54.584 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 23 16 46514 46498 126.309 91.6875 0.00625445 0.00791153 2024-08-29T15:15:54.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:54.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:54.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:54.665 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:15:54.665 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:15:54.665 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:15:54.665 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:15:54.665 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:15:54.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 5 1 2024-08-29T15:15:54.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:54.831 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:54.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:54.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:54.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:54.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:54.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:55.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:55.082 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:55.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:55.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:55.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:55.091 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:15:55.092 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:15:55.138 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:15:55.139 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:15:55.147 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:15:55.173 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:15:55.174 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:15:55.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:55.216 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:15:55.217 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:15:55.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:55.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:55.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:55.251 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:15:55.251 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:15:55.251 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:55.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:15:55.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:15:55.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:55.252 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:15:55.252 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:15:55.285 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:15:55.285 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:15:55.319 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:15:55.319 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:15:55.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:55.352 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:15:55.353 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:15:55.365 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:55.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:55.412 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:15:55.412 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:15:55.412 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:15:55.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:15:55.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:15:55.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:55.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:55.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:15:55.584 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 24 16 47958 47942 124.806 90.25 0.0128621 0.00800718 2024-08-29T15:15:55.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:15:55.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:55.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:55.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:15:55.742 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:15:55.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:55.800 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:55.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:55.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:55.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:55.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:55.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:55.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:56.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:56.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:56.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:56.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:56.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:56.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:56.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:56.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:56.255 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:56 smithi171 ceph-mon[90031]: pgmap v524: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-29T15:15:56.255 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4077028897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:56 smithi027 ceph-mon[98970]: pgmap v524: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-29T15:15:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4077028897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:56 smithi027 ceph-mon[93921]: pgmap v524: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-29T15:15:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4077028897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:15:56.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:56.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:56.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:56.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:56.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:56.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:56.584 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 25 16 49573 49557 123.851 100.938 0.00742824 0.00806907 2024-08-29T15:15:56.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:56.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:56.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:56.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:56.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:56.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:56.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:56.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:56.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:56.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:56.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:57.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:57.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:57.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:57.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:57.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:57.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:57.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:57.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:57.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:57.421 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:57.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:57.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:57.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:57.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:57.584 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 26 16 51227 51211 123.064 103.375 0.00620336 0.00812006 2024-08-29T15:15:57.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:57.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:57.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:57.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:57.725 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:57.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:57.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:57.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:57.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:57.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:57.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:57.861 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:15:57.861 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:15:57.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:58.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:58.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:58.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:58.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:58 smithi171 ceph-mon[90031]: pgmap v525: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:15:58.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:58.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:58 smithi027 ceph-mon[98970]: pgmap v525: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:15:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:58 smithi027 ceph-mon[93921]: pgmap v525: 89 pgs: 89 active+clean; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:15:58.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:15:58.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.399 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:58.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:15:58.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:15:58.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:58.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:58.584 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 27 16 52590 52574 121.661 85.1875 0.00963761 0.00821366 2024-08-29T15:15:58.620 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:15:58.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:58.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:58.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:58.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:58.902 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:15:58.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:15:58.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:58.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:58.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:15:59.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:59.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:59.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:59.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:59.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:59.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:59.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:59.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:59.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:15:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:15:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:15:59] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-29T15:15:59.330 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:59.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:59.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:15:59.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:59.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:59.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:59.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:15:59.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 28 16 53942 53926 120.333 84.5 0.0141891 0.00830466 2024-08-29T15:15:59.601 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:59.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:59.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:15:59.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:15:59.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:15:59.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:59.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:15:59.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:15:59.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:15:59.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:15:59.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:15:59.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:00.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:59 smithi171 ceph-mon[90031]: pgmap v526: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-29T15:16:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:15:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:59 smithi027 ceph-mon[98970]: pgmap v526: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-29T15:16:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:15:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:59 smithi027 ceph-mon[93921]: pgmap v526: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-08-29T15:16:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:15:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:00.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:00.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:00.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:00.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:00.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:00.371 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:00.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:00.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:00.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:00.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:00.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:00.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 29 16 55135 55119 118.755 74.5625 0.013379 0.0084154 2024-08-29T15:16:00.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:00.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:00.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:00.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:00.771 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:16:00.771 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:16:00.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:00.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:00.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:00.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:00.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:01.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:01.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:01.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:01.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:01.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:01.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:01.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:01.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:01.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:01.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:01.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:01.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:01.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:01.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:01.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:01.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:01.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 30 16 56597 56581 117.843 91.375 0.0102169 0.00847939 2024-08-29T15:16:01.604 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:01.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:01.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:01.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:01.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:01.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:01.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:01.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:01.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:01.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:01.972 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:01 smithi027 ceph-mon[93921]: pgmap v527: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-29T15:16:01.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:01 smithi027 ceph-mon[98970]: pgmap v527: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-29T15:16:01.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:02.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:01 smithi171 ceph-mon[90031]: pgmap v527: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-29T15:16:02.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:02.134 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:02.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:02.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:02.288 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:02.390 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:02.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:02.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:02.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:02.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:02.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 31 16 57715 57699 116.295 69.875 0.0101875 0.00859269 2024-08-29T15:16:02.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:02.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:02.708 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:16:02.708 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.5 to 1 (10000) 2024-08-29T15:16:02.778 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:02.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:02.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2885524966' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:16:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:16:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3810646612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2885524966' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:16:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:16:02.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3810646612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:02.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:02 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:02.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:02.951 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:02.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:02.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2885524966' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:16:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:02 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:16:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3810646612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:02 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:03.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:03.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:03.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:03.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:16:03.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:03.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:03.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:03.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:03.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:03.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:03.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:03.469 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:16:03.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:03.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:03.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:03.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:03.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 32 16 59062 59046 115.292 84.1875 0.00894923 0.00866817 2024-08-29T15:16:03.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:03.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:03.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:03.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:03.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:03.783 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:03.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:03.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:03 smithi171 ceph-mon[90031]: pgmap v528: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:16:03.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:03 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:03.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:03 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:16:03.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:03 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:03.878 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:03 smithi171 ceph-mon[90031]: osdmap e179: 8 total, 8 up, 7 in 2024-08-29T15:16:03.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:03.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:03.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:03.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:04.020 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:04.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:04.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[93921]: pgmap v528: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:16:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:16:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[93921]: osdmap e179: 8 total, 8 up, 7 in 2024-08-29T15:16:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[98970]: pgmap v528: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:16:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:16:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:03 smithi027 ceph-mon[98970]: osdmap e179: 8 total, 8 up, 7 in 2024-08-29T15:16:04.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:04.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:04.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:04.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:04.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:04.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:04.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:04.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:04.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:04.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:04.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 33 16 60352 60336 114.241 80.625 0.00872666 0.00874712 2024-08-29T15:16:04.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:04.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:04.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:04.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:04.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:04.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:04.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:04.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:04.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:05.012 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:05.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:05.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:05.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:05.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:05.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:05.266 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:05.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:05.368 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:05.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:05.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:05.469 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:16:05.469 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:16:05.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:05.533 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:05.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:05.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 34 16 61660 61644 113.286 81.75 0.0145891 0.00882201 2024-08-29T15:16:05.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:05.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:05.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:05.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:16:05.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:05.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:05.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:05.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:05.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:05.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:05 smithi171 ceph-mon[90031]: pgmap v530: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-08-29T15:16:06.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:05 smithi027 ceph-mon[93921]: pgmap v530: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-08-29T15:16:06.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:05 smithi027 ceph-mon[98970]: pgmap v530: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-08-29T15:16:06.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:06.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:06.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:16:06.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:06.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:06.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:06.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:06.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:06.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:06.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:06.420 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:16:06.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:06.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:06.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:06.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 35 16 62903 62887 112.269 77.6875 0.0131978 0.0089017 2024-08-29T15:16:06.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:06.628 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:06.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:06.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:06.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:06.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:06.875 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:06.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:06.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:07.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:07.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:07.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:07.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:07.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:07.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:07.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:07.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:07.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:07.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:07.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:07.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:07.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:07.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:07.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:07.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 36 16 64170 64154 111.35 79.1875 0.00416038 0.00897222 2024-08-29T15:16:07.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:07.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:07.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:07.717 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:07.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:07.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:07.859 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:07.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:07.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:07.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:08.003 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:08.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:07 smithi171 ceph-mon[90031]: pgmap v531: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-29T15:16:08.033 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:07 smithi027 ceph-mon[98970]: pgmap v531: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-29T15:16:08.040 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:07 smithi027 ceph-mon[93921]: pgmap v531: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-29T15:16:08.071 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-29T15:16:08.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:16:08.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:08.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:08.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:08.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:08.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:08.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:08.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:08.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:08.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:08.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:08.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:08.589 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 37 16 65405 65389 110.426 77.1875 0.0101041 0.00904831 2024-08-29T15:16:08.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:08.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:08.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:08.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:08.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:08.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:08.824 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:08.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:08.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:08.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:08.902 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:08.907 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:16:08.908 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:16:08.948 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:16:08.948 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:16:08.982 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:16:08.983 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:16:08.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:09.021 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:16:09.022 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:16:09.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:09.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:09.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:09.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:16:09.055 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:16:09.088 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:16:09.089 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:16:09.120 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:16:09.121 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:16:09.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:09.153 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:16:09.154 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:16:09.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:09.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:09.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:09.235 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:16:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:16:09] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.51.0" 2024-08-29T15:16:09.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:09.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:09.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:09.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:09.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:09.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:09.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:09.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:09.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:09.585 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 38 16 66566 66550 109.43 72.5625 0.0263286 0.00913199 2024-08-29T15:16:09.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:09.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:09.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:09.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:09.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:09.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:09 smithi027 ceph-mon[93921]: pgmap v532: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:16:09.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1083764400' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:09.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:09 smithi027 ceph-mon[98970]: pgmap v532: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:16:09.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1083764400' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:09.834 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:09 smithi171 ceph-mon[90031]: pgmap v532: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:16:09.834 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1083764400' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:24063 5.2 5:40275125:::benchmark_data_smithi027_135490_object24062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:41.414843+0000", 2024-08-29T15:16:09.835 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 328.41598597199999, 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080352217000000004, 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 24063 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414843+0000", 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414843+0000", 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414842+0000", 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:09.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414971+0000", 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012967999999999999 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414985+0000", 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3345e-05 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.414992+0000", 2024-08-29T15:16:09.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3220000000000004e-06 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415036+0000", 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3732000000000001e-05 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415160+0000", 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012448200000000001 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.415278+0000", 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011834500000000001 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.416259+0000", 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00098051600000000007 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.417399+0000", 2024-08-29T15:16:09.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0011396900000000001 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.495096+0000", 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.077697567999999995 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.495153+0000", 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7089000000000002e-05 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.495195+0000", 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2038e-05 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:24449 5.2 5:4aea18f6:::benchmark_data_smithi027_135490_object24448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:41.632501+0000", 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 328.19832829799998, 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072524163000000003, 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.842 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 24449 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.632501+0000", 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.632501+0000", 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.632500+0000", 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.632578+0000", 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8059000000000006e-05 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.632587+0000", 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2080000000000006e-06 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.632594+0000", 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2250000000000003e-06 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.634483+0000", 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018890280000000001 2024-08-29T15:16:09.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.634593+0000", 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010967300000000001 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.634743+0000", 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000149689 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.637297+0000", 2024-08-29T15:16:09.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025546620000000001 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.637339+0000", 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1343e-05 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.704972+0000", 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067633121000000004 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.705002+0000", 2024-08-29T15:16:09.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9785999999999999e-05 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.705025+0000", 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3309e-05 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62784 5.2 5:4e4afa91:::benchmark_data_smithi027_135490_object62783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.698155+0000", 2024-08-29T15:16:09.848 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.13267450400002, 2024-08-29T15:16:09.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.230311092, 2024-08-29T15:16:09.849 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.849 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.849 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.849 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.849 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62784 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698155+0000", 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698155+0000", 2024-08-29T15:16:09.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698153+0000", 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698193+0000", 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9925999999999997e-05 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698206+0000", 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3353000000000001e-05 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698213+0000", 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7100000000000001e-06 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698253+0000", 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9659000000000001e-05 2024-08-29T15:16:09.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698338+0000", 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5224999999999998e-05 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.698423+0000", 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5331e-05 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.700528+0000", 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021043030000000001 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.700910+0000", 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00038244899999999998 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928405+0000", 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.22749487900000001 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928441+0000", 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5852000000000002e-05 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928466+0000", 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4910999999999999e-05 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.855 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62804 5.2 5:4b7a5767:::benchmark_data_smithi027_135490_object62803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.715007+0000", 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.11582246699999, 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.213522565, 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62804 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.715007+0000", 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.715007+0000", 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.715005+0000", 2024-08-29T15:16:09.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731773+0000", 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016767741999999999 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731790+0000", 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7112000000000002e-05 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731798+0000", 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4700000000000002e-06 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731853+0000", 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4864000000000002e-05 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731995+0000", 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014182299999999999 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.732139+0000", 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014374099999999999 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.736479+0000", 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043399129999999999 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.736505+0000", 2024-08-29T15:16:09.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5956999999999999e-05 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928488+0000", 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19198367899999999 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928512+0000", 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3311000000000001e-05 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928529+0000", 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7592999999999999e-05 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62806 5.2 5:454a10f1:::benchmark_data_smithi027_135490_object62805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.731814+0000", 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.09901528500001, 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19683216000000001, 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.862 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62806 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731814+0000", 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731814+0000", 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.731812+0000", 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.732029+0000", 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021693600000000001 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.732042+0000", 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2748999999999999e-05 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.732049+0000", 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2799999999999998e-06 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.734123+0000", 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020732429999999998 2024-08-29T15:16:09.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.734248+0000", 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012545699999999999 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.734380+0000", 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000132275 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.736590+0000", 2024-08-29T15:16:09.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022095679999999999 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.739212+0000", 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026218629999999999 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928568+0000", 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.18935637799999999 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928611+0000", 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2954999999999998e-05 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928646+0000", 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4925999999999997e-05 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.868 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62844 5.2 5:4dcb818a:::benchmark_data_smithi027_135490_object62843:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.763213+0000", 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.06761643499999, 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.165525802, 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62844 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763213+0000", 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763213+0000", 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763211+0000", 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763286+0000", 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4808999999999995e-05 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763302+0000", 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6452000000000001e-05 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763311+0000", 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2269999999999993e-06 2024-08-29T15:16:09.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763362+0000", 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1558999999999999e-05 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763511+0000", 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014912899999999999 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.763663+0000", 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000151576 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.766093+0000", 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002430382 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.767980+0000", 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018865329999999999 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928688+0000", 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16070811199999999 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928721+0000", 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3553999999999998e-05 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928739+0000", 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7119999999999999e-05 2024-08-29T15:16:09.874 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62856 5.2 5:42223c58:::benchmark_data_smithi027_135490_object62855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.769536+0000", 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.06129298299999, 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.159319341, 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.875 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62856 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769536+0000", 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769536+0000", 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769534+0000", 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769618+0000", 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3838999999999995e-05 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769632+0000", 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3752e-05 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769639+0000", 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4019999999999996e-06 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769682+0000", 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2305000000000002e-05 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769824+0000", 2024-08-29T15:16:09.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014233900000000001 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.769972+0000", 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014754699999999999 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.772011+0000", 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020397459999999998 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.772333+0000", 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00032216999999999999 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928777+0000", 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.156443788 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928825+0000", 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7267e-05 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928856+0000", 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1000999999999997e-05 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62872 5.2 5:4a8ad332:::benchmark_data_smithi027_135490_object62871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.778961+0000", 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.05186767100002, 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14996379700000001, 2024-08-29T15:16:09.881 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62872 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.778961+0000", 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.778961+0000", 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.778960+0000", 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.778992+0000", 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2085999999999997e-05 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.779006+0000", 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3896999999999999e-05 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.779014+0000", 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0160000000000008e-06 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.779069+0000", 2024-08-29T15:16:09.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4920000000000003e-05 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.779188+0000", 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119178 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.779322+0000", 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000134418 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.781441+0000", 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021190720000000001 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.782826+0000", 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013847950000000001 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928885+0000", 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14605908000000001 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928908+0000", 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2719000000000001e-05 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928925+0000", 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7441000000000001e-05 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.887 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62882 5.2 5:479e4993:::benchmark_data_smithi027_135490_object62881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.788681+0000", 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.04214832399998, 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.140366307, 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62882 2024-08-29T15:16:09.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788681+0000", 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788681+0000", 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788680+0000", 2024-08-29T15:16:09.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788731+0000", 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1956000000000002e-05 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788743+0000", 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1986e-05 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788749+0000", 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9800000000000003e-06 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788791+0000", 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1610000000000003e-05 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.788914+0000", 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012312799999999999 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.789074+0000", 2024-08-29T15:16:09.892 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:09.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015959000000000001 2024-08-29T15:16:09.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.893 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.893 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.791370+0000", 2024-08-29T15:16:09.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022960459999999999 2024-08-29T15:16:09.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.792407+0000", 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001036737 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.928963+0000", 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13655690600000001 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929013+0000", 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9734999999999997e-05 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929047+0000", 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3951000000000001e-05 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62888 5.2 5:4224ff04:::benchmark_data_smithi027_135490_object62887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.793582+0000", 2024-08-29T15:16:09.895 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.03724686499999, 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13554179899999999, 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62888 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.896 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793582+0000", 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793582+0000", 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793581+0000", 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.897 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793636+0000", 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5386000000000001e-05 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793647+0000", 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1165999999999999e-05 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793653+0000", 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.22e-06 2024-08-29T15:16:09.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793698+0000", 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5148000000000002e-05 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793848+0000", 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015043700000000001 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.793997+0000", 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014838899999999999 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.795783+0000", 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001786445 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.796151+0000", 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000368224 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929081+0000", 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.132929775 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929105+0000", 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4383999999999999e-05 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929124+0000", 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8708000000000001e-05 2024-08-29T15:16:09.901 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:62892 5.2 5:4c3987ab:::benchmark_data_smithi027_135490_object62891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:00.797290+0000", 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 309.03353916700001, 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13194727000000001, 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.902 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 62892 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797290+0000", 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797290+0000", 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797289+0000", 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797344+0000", 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4802000000000001e-05 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797356+0000", 2024-08-29T15:16:09.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2116999999999999e-05 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797361+0000", 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5999999999999997e-06 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797392+0000", 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0397999999999999e-05 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797485+0000", 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3103000000000006e-05 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.797577+0000", 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2466000000000005e-05 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.799660+0000", 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00208267 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.799969+0000", 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000308998 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929164+0000", 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12919460599999999 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929205+0000", 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1681999999999998e-05 2024-08-29T15:16:09.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:00.929237+0000", 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1999000000000003e-05 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:70255 5.2 5:4bd7978b:::benchmark_data_smithi027_135490_object70254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.039561+0000", 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 303.79126784499999, 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085469735000000005, 2024-08-29T15:16:09.908 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 70255 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039561+0000", 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039561+0000", 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039560+0000", 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039605+0000", 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5773999999999999e-05 2024-08-29T15:16:09.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039620+0000", 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4389000000000001e-05 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039626+0000", 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7109999999999998e-06 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039677+0000", 2024-08-29T15:16:09.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0847999999999998e-05 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039837+0000", 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000160032 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.039972+0000", 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013424699999999999 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.042595+0000", 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002623785 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.042889+0000", 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00029345400000000001 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.124963+0000", 2024-08-29T15:16:09.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082074625999999998 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.125003+0000", 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9941999999999998e-05 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.125031+0000", 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7687000000000001e-05 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.914 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:83007 5.2 5:4ffc4bd8:::benchmark_data_smithi027_135490_object83006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:16.026481+0000", 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 293.80434813900001, 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078788394999999997, 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 83007 2024-08-29T15:16:09.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026481+0000", 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026481+0000", 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026479+0000", 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026512+0000", 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3324999999999997e-05 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026532+0000", 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0217e-05 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026541+0000", 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6100000000000006e-06 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026605+0000", 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4426999999999998e-05 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026747+0000", 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000142261 2024-08-29T15:16:09.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.026899+0000", 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000151788 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.029727+0000", 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028281769999999999 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.031032+0000", 2024-08-29T15:16:09.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013044840000000001 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.105181+0000", 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.074149425000000005 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.105230+0000", 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8449000000000003e-05 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:16.105269+0000", 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9684999999999997e-05 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:141389 5.2 5:4ff9f510:::benchmark_data_smithi027_135490_object7637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:11.402637+0000", 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 238.428191935, 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080798574999999997, 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.921 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141389 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.402637+0000", 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.402637+0000", 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.402637+0000", 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.402640+0000", 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8750000000000002e-06 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.402653+0000", 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3006999999999999e-05 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.402659+0000", 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2669999999999997e-06 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.412316+0000", 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0096569350000000002 2024-08-29T15:16:09.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.412437+0000", 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000121382 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.412563+0000", 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000126208 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.425428+0000", 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012864881999999999 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.425724+0000", 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000296149 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.483332+0000", 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.057607630999999999 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.483386+0000", 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.448e-05 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.483436+0000", 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9400000000000001e-05 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:141400 5.2 5:49967f11:::benchmark_data_smithi027_135490_object7648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2024-08-29T15:16:09.927 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:12:11.415658+0000", 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 238.41517149200001, 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.089456215000000006, 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141400 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.415658+0000", 2024-08-29T15:16:09.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.415658+0000", 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.415657+0000", 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.415661+0000", 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7589999999999999e-06 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.415669+0000", 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0080000000000002e-06 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.415674+0000", 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8219999999999997e-06 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.420101+0000", 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044273560000000003 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.420392+0000", 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00029070399999999997 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.420728+0000", 2024-08-29T15:16:09.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00033636400000000002 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.425734+0000", 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050063499999999997 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.426260+0000", 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00052544099999999997 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.505022+0000", 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078762434000000006 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.505074+0000", 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1873999999999998e-05 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:12:11.505114+0000", 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9684000000000002e-05 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.933 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239386 5.2 5:4b9b9b8e:::benchmark_data_smithi027_135490_object105634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.721669+0000", 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 124.10916050100001, 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2226445109999999, 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.934 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239386 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721669+0000", 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721669+0000", 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721667+0000", 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721675+0000", 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1810000000000001e-06 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721694+0000", 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9380000000000001e-05 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721702+0000", 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8679999999999999e-06 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721761+0000", 2024-08-29T15:16:09.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8610999999999998e-05 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721866+0000", 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010515500000000001 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721970+0000", 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000103955 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728347+0000", 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0063774390000000004 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.744581+0000", 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016233282000000002 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751292+0000", 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067116160000000001 2024-08-29T15:16:09.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751309+0000", 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7039000000000001e-05 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.694440+0000", 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94313092200000004 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:16:09.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.694453+0000", 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2892e-05 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944216+0000", 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2497627339999999 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944313+0000", 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7345999999999999e-05 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.942 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239390 5.2 5:42e53391:::benchmark_data_smithi027_135490_object105638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.728345+0000", 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 124.102484094, 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2162731120000001, 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239390 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.943 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728345+0000", 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728345+0000", 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728343+0000", 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728351+0000", 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5900000000000002e-06 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728364+0000", 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3303e-05 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728371+0000", 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2899999999999997e-06 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728407+0000", 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5392000000000003e-05 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728543+0000", 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000135897 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728677+0000", 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000134105 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.745130+0000", 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016453035000000001 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.745168+0000", 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8362999999999998e-05 2024-08-29T15:16:09.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751342+0000", 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0061735719999999996 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751350+0000", 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3820000000000004e-06 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.695482+0000", 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94413139099999999 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.695495+0000", 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3203e-05 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944542+0000", 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.249046919 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944618+0000", 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6487000000000001e-05 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239393 5.2 5:4763d321:::benchmark_data_smithi027_135490_object105641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.729080+0000", 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 124.101749399, 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2203413840000001, 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:16:09.950 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239393 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.729080+0000", 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.729080+0000", 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.729079+0000", 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.729084+0000", 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8280000000000002e-06 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.729095+0000", 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1279e-05 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.729102+0000", 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.939e-06 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.736215+0000", 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0071130680000000002 2024-08-29T15:16:09.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.736332+0000", 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011737 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.736445+0000", 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011225799999999999 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751419+0000", 2024-08-29T15:16:09.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014974276999999999 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751429+0000", 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5659999999999996e-06 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.695522+0000", 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94409371600000003 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.695533+0000", 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0633e-05 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944650+0000", 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2491169950000001 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944732+0000", 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2583000000000005e-05 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949372+0000", 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046398780000000001 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949421+0000", 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8785999999999997e-05 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.957 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:242778 5.3 e143/140 5:ca05abd4:::benchmark_data_smithi027_135490_object109026:head v 143'15118, mlcod=143'15115)", 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.878652+0000", 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 116.952176782, 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.075221544000000001, 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878652+0000", 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878652+0000", 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878652+0000", 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878659+0000", 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1829999999999997e-06 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878671+0000", 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1902000000000001e-05 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878677+0000", 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5350000000000001e-06 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878721+0000", 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4013e-05 2024-08-29T15:16:09.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.878736+0000", 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4771e-05 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.953797+0000", 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.075060974000000003 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.953874+0000", 2024-08-29T15:16:09.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6942999999999998e-05 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:242787 5.3 e143/140 5:c3dbb427:::benchmark_data_smithi027_135490_object109035:head v 143'15119, mlcod=143'15115)", 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.881727+0000", 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 116.949102603, 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.075239196999999994, 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.881727+0000", 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.881727+0000", 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.881726+0000", 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:16:09.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.881732+0000", 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0349999999999998e-06 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.949971+0000", 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.068239387999999998 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:09.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.949989+0000", 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7629999999999999e-05 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.950231+0000", 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000242413 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.950259+0000", 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7847999999999998e-05 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956868+0000", 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0066087489999999997 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956966+0000", 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7907999999999998e-05 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:09.966 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:09.967 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:16:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:09.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:09.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:10.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:10.119 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:10.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:10.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:10.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:10.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:10.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:10.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:10.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:10.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:10.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:10.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:10.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:10.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 39 16 67866 67850 108.707 81.25 0.0108524 0.0091934 2024-08-29T15:16:10.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:10.658 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:10.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:10.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:16:10.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:10.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:10.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:10.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:10.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:10.946 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:10.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:11.031 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:16:11.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:11.081 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:11.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:11.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:11.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:11.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:11.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:11.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:11.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:11.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:11.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:11.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:11.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:11.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:11.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:16:11.586059+0000 min lat: 0.00129485 max lat: 0.174854 avg lat: 0.00927834 2024-08-29T15:16:11.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:16:11.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 40 16 68966 68950 107.708 68.75 0.0304105 0.00927834 2024-08-29T15:16:11.599 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:11.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:11.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:11.716 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:11 smithi171 ceph-mon[90031]: pgmap v533: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:16:11.727 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:11.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:11.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:11 smithi027 ceph-mon[98970]: pgmap v533: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:16:11.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:11 smithi027 ceph-mon[93921]: pgmap v533: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:16:11.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:11.872 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:11.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:11.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:11.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:12.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:12.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:12.116 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:12.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:12.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:12.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:12.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:12.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:12.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:12.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:12.454 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:12.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:12.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:12.587 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 41 16 70061 70045 106.75 68.4375 0.0126769 0.00936128 2024-08-29T15:16:12.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:12.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:12.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:12.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:12.737 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:12.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:12.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:12.743 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:16:12.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:12.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:12.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:12.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:13.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:13.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1080283528' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:16:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:12 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:16:13.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1080283528' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:16:13.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:12 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:16:13.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1080283528' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:16:13.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:12 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:16:13.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:13.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:13.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:16:13.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:13.300 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:13.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:13.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:13.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:13.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:71087 6.3 6:ca04cccf:::benchmark_data_smithi027_169445_object71086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:13.555253+0000", 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0055803729999999996, 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056250930000000003, 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:13.563 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71087 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555253+0000", 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555253+0000", 2024-08-29T15:16:13.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555251+0000", 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555287+0000", 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6532000000000002e-05 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555300+0000", 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3261999999999999e-05 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555308+0000", 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3880000000000002e-06 2024-08-29T15:16:13.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555352+0000", 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4110000000000003e-05 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555487+0000", 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013487500000000001 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:16:13.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.555637+0000", 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015032199999999999 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.558601+0000", 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029643009999999999 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559036+0000", 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00043450999999999999 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.568 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:71091 6.3 6:c5ebbc3d:::benchmark_data_smithi027_169445_object71090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e180)", 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:13.558881+0000", 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.001952114, 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020959630000000002, 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:16:13.569 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71091 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.558881+0000", 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.558881+0000", 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.558879+0000", 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.558934+0000", 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4311000000000002e-05 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.558948+0000", 2024-08-29T15:16:13.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3705e-05 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.558955+0000", 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6550000000000004e-06 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559065+0000", 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010991499999999999 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559207+0000", 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014201700000000001 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559353+0000", 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014560699999999999 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.573 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44912.0:71089 6.9 e180/168 6:9e763b85:::benchmark_data_smithi027_169445_object71088:head v 180'4483, mlcod=180'4481)", 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:13.559510+0000", 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.001322967, 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001542147, 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559510+0000", 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559510+0000", 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559509+0000", 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559619+0000", 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000109882 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559632+0000", 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3006e-05 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559638+0000", 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5439999999999998e-06 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559689+0000", 2024-08-29T15:16:13.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0874000000000001e-05 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.559716+0000", 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7426000000000001e-05 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44912.0:71094 6.c e180/168 6:32041668:::benchmark_data_smithi027_169445_object71093:head v 180'4379, mlcod=180'4377)", 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:13.560270+0000", 2024-08-29T15:16:13.577 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00056253399999999995, 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00081627999999999996, 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560270+0000", 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560270+0000", 2024-08-29T15:16:13.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560270+0000", 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560352+0000", 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2624e-05 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.579 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560363+0000", 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.049e-05 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560369+0000", 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6980000000000003e-06 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560411+0000", 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2873000000000001e-05 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560430+0000", 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8981e-05 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44912.0:71093 6.1 e180/168 6:8d9bb6c4:::benchmark_data_smithi027_169445_object71092:head v 180'4493, mlcod=180'4491)", 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:13.560287+0000", 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00054612900000000002, 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00083002300000000005, 2024-08-29T15:16:13.581 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560287+0000", 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560287+0000", 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560286+0000", 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560364+0000", 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7378000000000004e-05 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560374+0000", 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0582e-05 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560380+0000", 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8690000000000001e-06 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560432+0000", 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1832000000000001e-05 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:13.560449+0000", 2024-08-29T15:16:13.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7516999999999998e-05 2024-08-29T15:16:13.585 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.585 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:16:13.585 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.585 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:16:13.585 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:16:13.585 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 5 2024-08-29T15:16:13.585 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:16:13.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:13.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:13.587 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 42 16 71127 71111 105.795 66.625 0.00624766 0.00944658 2024-08-29T15:16:13.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:13.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:13.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:13.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:13.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:13.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:13.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[93921]: pgmap v534: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-29T15:16:13.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:13.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:16:13.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[93921]: osdmap e180: 8 total, 8 up, 7 in 2024-08-29T15:16:13.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:13.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[98970]: pgmap v534: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-29T15:16:13.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:13.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:16:13.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[98970]: osdmap e180: 8 total, 8 up, 7 in 2024-08-29T15:16:13.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:13.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:13.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:13.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:13 smithi171 ceph-mon[90031]: pgmap v534: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-29T15:16:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:13 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:16:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:13 smithi171 ceph-mon[90031]: osdmap e180: 8 total, 8 up, 7 in 2024-08-29T15:16:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:14.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:14.141 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:14.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:14.226 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:14.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:14.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:14.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:14.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:14.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:14.432 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:14.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:14.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:14.558 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:14.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:14.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 43 16 72179 72163 104.863 65.75 0.0160472 0.00953014 2024-08-29T15:16:14.626 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:16:14.628 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:16:14.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:14.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:14.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:14.807 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:14.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:14.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:14.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:14.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:14.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:14.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:15.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:15.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:15.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:15.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:16:15.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:15.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:15.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:15.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:15.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:15.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:15.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:15.457 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:16:15.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:15.520 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:15.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:15.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 44 16 73340 73324 104.129 72.5625 0.00775785 0.00959706 2024-08-29T15:16:15.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:15.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:15.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:15.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:16:15.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:15.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:15.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:15.854 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:15 smithi171 ceph-mon[90031]: pgmap v536: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-29T15:16:15.854 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:15 smithi171 ceph-mon[90031]: from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:16:15.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:15.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:15.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:16.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:16.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:16.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:16.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:16.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:15 smithi027 ceph-mon[93921]: pgmap v536: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-29T15:16:16.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:15 smithi027 ceph-mon[93921]: from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:16:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:15 smithi027 ceph-mon[98970]: pgmap v536: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-29T15:16:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:15 smithi027 ceph-mon[98970]: from='client.44972 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:16:16.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:16.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:16.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:16.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:16.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:16.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:16.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:16.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:16.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:16.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:16.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:16.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:16.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 45 16 74572 74556 103.526 77 0.00986448 0.00965367 2024-08-29T15:16:16.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:16.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:16.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:16.735 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:16.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:16.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:16.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:16.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:16.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:16.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:16.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:17.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:17.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:17.039 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":536,"stamp":"2024-08-29T15:16:14.465574+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4474279909,"num_objects":68227,"num_object_clones":0,"num_object_copies":204681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68227,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":69322,"num_write_kb":4379777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":2367640,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69381,"ondisk_log_size":69381,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14580576,"kb_used_data":13098592,"kb_used_omap":32,"kb_used_meta":1481887,"kb_avail":641635488,"statfs":{"total":671965249536,"available":657034739712,"internally_reserved":0,"allocated":13412958208,"data_stored":13406699487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33376,"internal_metadata":1517452704},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,5,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":821952512,"num_objects":12542,"num_object_clones":0,"num_object_copies":37626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12542,"num_write_kb":802688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003022"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":229,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767764+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:16:02.767764+0000","last_peered":"2024-08-29T15:16:02.767764+0000","last_clean":"2024-08-29T15:16:02.767764+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:16:02.767764+0000","last_undegraded":"2024-08-29T15:16:02.767764+0000","last_fullsized":"2024-08-29T15:16:02.767764+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023566+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:15:53.023566+0000","last_peered":"2024-08-29T15:15:53.023566+0000","last_clean":"2024-08-29T15:15:53.023566+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:15:53.023566+0000","last_undegraded":"2024-08-29T15:15:53.023566+0000","last_fullsized":"2024-08-29T15:15:53.023566+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":42,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.015856+0000","last_change":"2024-08-29T15:14:24.769430+0000","last_active":"2024-08-29T15:15:53.015856+0000","last_peered":"2024-08-29T15:15:53.015856+0000","last_clean":"2024-08-29T15:15:53.015856+0000","last_became_active":"2024-08-29T15:14:24.769068+0000","last_became_peered":"2024-08-29T15:14:24.769068+0000","last_unstale":"2024-08-29T15:15:53.015856+0000","last_undegraded":"2024-08-29T15:15:53.015856+0000","last_fullsized":"2024-08-29T15:15:53.015856+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:03:30.509384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745873+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:16:12.745873+0000","last_peered":"2024-08-29T15:16:12.745873+0000","last_clean":"2024-08-29T15:16:12.745873+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:16:12.745873+0000","last_undegraded":"2024-08-29T15:16:12.745873+0000","last_fullsized":"2024-08-29T15:16:12.745873+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745206+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:16:12.745206+0000","last_peered":"2024-08-29T15:16:12.745206+0000","last_clean":"2024-08-29T15:16:12.745206+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:16:12.745206+0000","last_undegraded":"2024-08-29T15:16:12.745206+0000","last_fullsized":"2024-08-29T15:16:12.745206+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023284+0000","last_change":"2024-08-29T15:14:41.708389+0000","last_active":"2024-08-29T15:15:53.023284+0000","last_peered":"2024-08-29T15:15:53.023284+0000","last_clean":"2024-08-29T15:15:53.023284+0000","last_became_active":"2024-08-29T15:14:41.708074+0000","last_became_peered":"2024-08-29T15:14:41.708074+0000","last_unstale":"2024-08-29T15:15:53.023284+0000","last_undegraded":"2024-08-29T15:15:53.023284+0000","last_fullsized":"2024-08-29T15:15:53.023284+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:36:56.089658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":26,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.750104+0000","last_change":"2024-08-29T15:15:30.439343+0000","last_active":"2024-08-29T15:16:12.750104+0000","last_peered":"2024-08-29T15:16:12.750104+0000","last_clean":"2024-08-29T15:16:12.750104+0000","last_became_active":"2024-08-29T15:15:30.438934+0000","last_became_peered":"2024-08-29T15:15:30.438934+0000","last_unstale":"2024-08-29T15:16:12.750104+0000","last_undegraded":"2024-08-29T15:16:12.750104+0000","last_fullsized":"2024-08-29T15:16:12.750104+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:44:45.377680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":87,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745392+0000","last_change":"2024-08-29T15:14:41.720900+0000","last_active":"2024-08-29T15:16:12.745392+0000","last_peered":"2024-08-29T15:16:12.745392+0000","last_clean":"2024-08-29T15:16:12.745392+0000","last_became_active":"2024-08-29T15:14:41.715217+0000","last_became_peered":"2024-08-29T15:14:41.715217+0000","last_unstale":"2024-08-29T15:16:12.745392+0000","last_undegraded":"2024-08-29T15:16:12.745392+0000","last_fullsized":"2024-08-29T15:16:12.745392+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:21:13.058909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"179'4215","reported_seq":4235,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.816951+0000","last_change":"2024-08-29T15:15:18.754051+0000","last_active":"2024-08-29T15:16:09.816951+0000","last_peered":"2024-08-29T15:16:09.816951+0000","last_clean":"2024-08-29T15:16:09.816951+0000","last_became_active":"2024-08-29T15:15:18.753647+0000","last_became_peered":"2024-08-29T15:15:18.753647+0000","last_unstale":"2024-08-29T15:16:09.816951+0000","last_undegraded":"2024-08-29T15:16:09.816951+0000","last_fullsized":"2024-08-29T15:16:09.816951+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4215,"log_dups_size":0,"ondisk_log_size":4215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:34:29.574588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276234240,"num_objects":4215,"num_object_clones":0,"num_object_copies":12645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4215,"num_write_kb":269760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023837+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:15:53.023837+0000","last_peered":"2024-08-29T15:15:53.023837+0000","last_clean":"2024-08-29T15:15:53.023837+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:15:53.023837+0000","last_undegraded":"2024-08-29T15:15:53.023837+0000","last_fullsized":"2024-08-29T15:15:53.023837+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.015547+0000","last_change":"2024-08-29T15:14:41.707030+0000","last_active":"2024-08-29T15:15:53.015547+0000","last_peered":"2024-08-29T15:15:53.015547+0000","last_clean":"2024-08-29T15:15:53.015547+0000","last_became_active":"2024-08-29T15:14:41.706699+0000","last_became_peered":"2024-08-29T15:14:41.706699+0000","last_unstale":"2024-08-29T15:15:53.015547+0000","last_undegraded":"2024-08-29T15:15:53.015547+0000","last_fullsized":"2024-08-29T15:15:53.015547+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:14:00.323667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"179'4242","reported_seq":4275,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:11.654296+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:16:11.654296+0000","last_peered":"2024-08-29T15:16:11.654296+0000","last_clean":"2024-08-29T15:16:11.654296+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:16:11.654296+0000","last_undegraded":"2024-08-29T15:16:11.654296+0000","last_fullsized":"2024-08-29T15:16:11.654296+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4242,"log_dups_size":0,"ondisk_log_size":4242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278003712,"num_objects":4242,"num_object_clones":0,"num_object_copies":12726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4242,"num_write_kb":271488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023714+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:15:53.023714+0000","last_peered":"2024-08-29T15:15:53.023714+0000","last_clean":"2024-08-29T15:15:53.023714+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:15:53.023714+0000","last_undegraded":"2024-08-29T15:15:53.023714+0000","last_fullsized":"2024-08-29T15:15:53.023714+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017244+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:15:53.017244+0000","last_peered":"2024-08-29T15:15:53.017244+0000","last_clean":"2024-08-29T15:15:53.017244+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:15:53.017244+0000","last_undegraded":"2024-08-29T15:15:53.017244+0000","last_fullsized":"2024-08-29T15:15:53.017244+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"179'4206","reported_seq":4226,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.493729+0000","last_change":"2024-08-29T15:15:18.757178+0000","last_active":"2024-08-29T15:16:09.493729+0000","last_peered":"2024-08-29T15:16:09.493729+0000","last_clean":"2024-08-29T15:16:09.493729+0000","last_became_active":"2024-08-29T15:15:18.756864+0000","last_became_peered":"2024-08-29T15:15:18.756864+0000","last_unstale":"2024-08-29T15:16:09.493729+0000","last_undegraded":"2024-08-29T15:16:09.493729+0000","last_fullsized":"2024-08-29T15:16:09.493729+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4206,"log_dups_size":0,"ondisk_log_size":4206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:43.382827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275644416,"num_objects":4206,"num_object_clones":0,"num_object_copies":12618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4206,"num_write_kb":269184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":169,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023764+0000","last_change":"2024-08-29T15:14:41.712300+0000","last_active":"2024-08-29T15:15:53.023764+0000","last_peered":"2024-08-29T15:15:53.023764+0000","last_clean":"2024-08-29T15:15:53.023764+0000","last_became_active":"2024-08-29T15:14:41.708025+0000","last_became_peered":"2024-08-29T15:14:41.708025+0000","last_unstale":"2024-08-29T15:15:53.023764+0000","last_undegraded":"2024-08-29T15:15:53.023764+0000","last_fullsized":"2024-08-29T15:15:53.023764+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:27:44.258784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":111,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768839+0000","last_change":"2024-08-29T15:14:41.710873+0000","last_active":"2024-08-29T15:16:02.768839+0000","last_peered":"2024-08-29T15:16:02.768839+0000","last_clean":"2024-08-29T15:16:02.768839+0000","last_became_active":"2024-08-29T15:14:41.710534+0000","last_became_peered":"2024-08-29T15:14:41.710534+0000","last_unstale":"2024-08-29T15:16:02.768839+0000","last_undegraded":"2024-08-29T15:16:02.768839+0000","last_fullsized":"2024-08-29T15:16:02.768839+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:50.786506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"180'4504","reported_seq":4539,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:13.840449+0000","last_change":"2024-08-29T15:15:18.753804+0000","last_active":"2024-08-29T15:16:13.840449+0000","last_peered":"2024-08-29T15:16:13.840449+0000","last_clean":"2024-08-29T15:16:13.840449+0000","last_became_active":"2024-08-29T15:15:18.753257+0000","last_became_peered":"2024-08-29T15:15:18.753257+0000","last_unstale":"2024-08-29T15:16:13.840449+0000","last_undegraded":"2024-08-29T15:16:13.840449+0000","last_fullsized":"2024-08-29T15:16:13.840449+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4504,"log_dups_size":0,"ondisk_log_size":4504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:59:57.157511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295174144,"num_objects":4504,"num_object_clones":0,"num_object_copies":13512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4504,"num_write_kb":288256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":51,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.016629+0000","last_change":"2024-08-29T15:14:41.712203+0000","last_active":"2024-08-29T15:15:53.016629+0000","last_peered":"2024-08-29T15:15:53.016629+0000","last_clean":"2024-08-29T15:15:53.016629+0000","last_became_active":"2024-08-29T15:14:41.711831+0000","last_became_peered":"2024-08-29T15:14:41.711831+0000","last_unstale":"2024-08-29T15:15:53.016629+0000","last_undegraded":"2024-08-29T15:15:53.016629+0000","last_fullsized":"2024-08-29T15:15:53.016629+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:54:29.075989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745039+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:16:12.745039+0000","last_peered":"2024-08-29T15:16:12.745039+0000","last_clean":"2024-08-29T15:16:12.745039+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:16:12.745039+0000","last_undegraded":"2024-08-29T15:16:12.745039+0000","last_fullsized":"2024-08-29T15:16:12.745039+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"179'4161","reported_seq":4181,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.649179+0000","last_change":"2024-08-29T15:15:18.752004+0000","last_active":"2024-08-29T15:16:09.649179+0000","last_peered":"2024-08-29T15:16:09.649179+0000","last_clean":"2024-08-29T15:16:09.649179+0000","last_became_active":"2024-08-29T15:15:18.751547+0000","last_became_peered":"2024-08-29T15:15:18.751547+0000","last_unstale":"2024-08-29T15:16:09.649179+0000","last_undegraded":"2024-08-29T15:16:09.649179+0000","last_fullsized":"2024-08-29T15:16:09.649179+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4161,"log_dups_size":0,"ondisk_log_size":4161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:43:33.859489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272695296,"num_objects":4161,"num_object_clones":0,"num_object_copies":12483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4161,"num_write_kb":266304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":214,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767530+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:16:02.767530+0000","last_peered":"2024-08-29T15:16:02.767530+0000","last_clean":"2024-08-29T15:16:02.767530+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:16:02.767530+0000","last_undegraded":"2024-08-29T15:16:02.767530+0000","last_fullsized":"2024-08-29T15:16:02.767530+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:54.656855+0000","last_change":"2024-08-29T15:14:41.712153+0000","last_active":"2024-08-29T15:15:54.656855+0000","last_peered":"2024-08-29T15:15:54.656855+0000","last_clean":"2024-08-29T15:15:54.656855+0000","last_became_active":"2024-08-29T15:14:41.711815+0000","last_became_peered":"2024-08-29T15:14:41.711815+0000","last_unstale":"2024-08-29T15:15:54.656855+0000","last_undegraded":"2024-08-29T15:15:54.656855+0000","last_fullsized":"2024-08-29T15:15:54.656855+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:28:05.338836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"180'4476","reported_seq":4509,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:14.098960+0000","last_change":"2024-08-29T15:15:18.751470+0000","last_active":"2024-08-29T15:16:14.098960+0000","last_peered":"2024-08-29T15:16:14.098960+0000","last_clean":"2024-08-29T15:16:14.098960+0000","last_became_active":"2024-08-29T15:15:18.751188+0000","last_became_peered":"2024-08-29T15:15:18.751188+0000","last_unstale":"2024-08-29T15:16:14.098960+0000","last_undegraded":"2024-08-29T15:16:14.098960+0000","last_fullsized":"2024-08-29T15:16:14.098960+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4476,"log_dups_size":0,"ondisk_log_size":4476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:03:58.623718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293339136,"num_objects":4476,"num_object_clones":0,"num_object_copies":13428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4476,"num_write_kb":286464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017503+0000","last_change":"2024-08-29T15:14:41.714417+0000","last_active":"2024-08-29T15:15:53.017503+0000","last_peered":"2024-08-29T15:15:53.017503+0000","last_clean":"2024-08-29T15:15:53.017503+0000","last_became_active":"2024-08-29T15:14:41.712773+0000","last_became_peered":"2024-08-29T15:14:41.712773+0000","last_unstale":"2024-08-29T15:15:53.017503+0000","last_undegraded":"2024-08-29T15:15:53.017503+0000","last_fullsized":"2024-08-29T15:15:53.017503+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:29:37.353192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":160,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:54.656891+0000","last_change":"2024-08-29T15:14:06.704490+0000","last_active":"2024-08-29T15:15:54.656891+0000","last_peered":"2024-08-29T15:15:54.656891+0000","last_clean":"2024-08-29T15:15:54.656891+0000","last_became_active":"2024-08-29T15:14:06.703917+0000","last_became_peered":"2024-08-29T15:14:06.703917+0000","last_unstale":"2024-08-29T15:15:54.656891+0000","last_undegraded":"2024-08-29T15:15:54.656891+0000","last_fullsized":"2024-08-29T15:15:54.656891+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:55:25.179472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"179'4235","reported_seq":4268,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:11.626277+0000","last_change":"2024-08-29T15:15:18.751951+0000","last_active":"2024-08-29T15:16:11.626277+0000","last_peered":"2024-08-29T15:16:11.626277+0000","last_clean":"2024-08-29T15:16:11.626277+0000","last_became_active":"2024-08-29T15:15:18.751645+0000","last_became_peered":"2024-08-29T15:15:18.751645+0000","last_unstale":"2024-08-29T15:16:11.626277+0000","last_undegraded":"2024-08-29T15:16:11.626277+0000","last_fullsized":"2024-08-29T15:16:11.626277+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4235,"log_dups_size":0,"ondisk_log_size":4235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:40:24.969000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277544960,"num_objects":4235,"num_object_clones":0,"num_object_copies":12705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4235,"num_write_kb":271040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":143,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017284+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:15:53.017284+0000","last_peered":"2024-08-29T15:15:53.017284+0000","last_clean":"2024-08-29T15:15:53.017284+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:15:53.017284+0000","last_undegraded":"2024-08-29T15:15:53.017284+0000","last_fullsized":"2024-08-29T15:15:53.017284+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":151,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:54.656919+0000","last_change":"2024-08-29T15:14:41.713336+0000","last_active":"2024-08-29T15:15:54.656919+0000","last_peered":"2024-08-29T15:15:54.656919+0000","last_clean":"2024-08-29T15:15:54.656919+0000","last_became_active":"2024-08-29T15:14:41.712786+0000","last_became_peered":"2024-08-29T15:14:41.712786+0000","last_unstale":"2024-08-29T15:15:54.656919+0000","last_undegraded":"2024-08-29T15:15:54.656919+0000","last_fullsized":"2024-08-29T15:15:54.656919+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:17:48.490496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"179'4244","reported_seq":4264,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.806311+0000","last_change":"2024-08-29T15:15:18.755857+0000","last_active":"2024-08-29T15:16:09.806311+0000","last_peered":"2024-08-29T15:16:09.806311+0000","last_clean":"2024-08-29T15:16:09.806311+0000","last_became_active":"2024-08-29T15:15:18.755586+0000","last_became_peered":"2024-08-29T15:15:18.755586+0000","last_unstale":"2024-08-29T15:16:09.806311+0000","last_undegraded":"2024-08-29T15:16:09.806311+0000","last_fullsized":"2024-08-29T15:16:09.806311+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4244,"log_dups_size":0,"ondisk_log_size":4244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:52:17.577528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278134784,"num_objects":4244,"num_object_clones":0,"num_object_copies":12732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4244,"num_write_kb":271616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":217,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767995+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:16:02.767995+0000","last_peered":"2024-08-29T15:16:02.767995+0000","last_clean":"2024-08-29T15:16:02.767995+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:16:02.767995+0000","last_undegraded":"2024-08-29T15:16:02.767995+0000","last_fullsized":"2024-08-29T15:16:02.767995+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":195,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.769735+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:16:02.769735+0000","last_peered":"2024-08-29T15:16:02.769735+0000","last_clean":"2024-08-29T15:16:02.769735+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:16:02.769735+0000","last_undegraded":"2024-08-29T15:16:02.769735+0000","last_fullsized":"2024-08-29T15:16:02.769735+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"179'3991","reported_seq":4010,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.821764+0000","last_change":"2024-08-29T15:15:18.752994+0000","last_active":"2024-08-29T15:16:09.821764+0000","last_peered":"2024-08-29T15:16:09.821764+0000","last_clean":"2024-08-29T15:16:09.821764+0000","last_became_active":"2024-08-29T15:15:18.752519+0000","last_became_peered":"2024-08-29T15:15:18.752519+0000","last_unstale":"2024-08-29T15:16:09.821764+0000","last_undegraded":"2024-08-29T15:16:09.821764+0000","last_fullsized":"2024-08-29T15:16:09.821764+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":3991,"log_dups_size":0,"ondisk_log_size":3991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:13:25.171934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261554176,"num_objects":3991,"num_object_clones":0,"num_object_copies":11973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3991,"num_write_kb":255424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":498,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:54.585644+0000","last_change":"2024-08-29T15:14:44.744053+0000","last_active":"2024-08-29T15:15:54.585644+0000","last_peered":"2024-08-29T15:15:54.585644+0000","last_clean":"2024-08-29T15:15:54.585644+0000","last_became_active":"2024-08-29T15:14:44.743581+0000","last_became_peered":"2024-08-29T15:14:44.743581+0000","last_unstale":"2024-08-29T15:15:54.585644+0000","last_undegraded":"2024-08-29T15:15:54.585644+0000","last_fullsized":"2024-08-29T15:15:54.585644+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:02:28.470398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":642,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.015837+0000","last_change":"2024-08-29T15:14:27.711565+0000","last_active":"2024-08-29T15:15:53.015837+0000","last_peered":"2024-08-29T15:15:53.015837+0000","last_clean":"2024-08-29T15:15:53.015837+0000","last_became_active":"2024-08-29T15:14:27.711044+0000","last_became_peered":"2024-08-29T15:14:27.711044+0000","last_unstale":"2024-08-29T15:15:53.015837+0000","last_undegraded":"2024-08-29T15:15:53.015837+0000","last_fullsized":"2024-08-29T15:15:53.015837+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:54:00.170761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768775+0000","last_change":"2024-08-29T15:14:41.714021+0000","last_active":"2024-08-29T15:16:02.768775+0000","last_peered":"2024-08-29T15:16:02.768775+0000","last_clean":"2024-08-29T15:16:02.768775+0000","last_became_active":"2024-08-29T15:14:41.713427+0000","last_became_peered":"2024-08-29T15:14:41.713427+0000","last_unstale":"2024-08-29T15:16:02.768775+0000","last_undegraded":"2024-08-29T15:16:02.768775+0000","last_fullsized":"2024-08-29T15:16:02.768775+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:43:27.654536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017497+0000","last_change":"2024-08-29T15:14:41.709125+0000","last_active":"2024-08-29T15:15:53.017497+0000","last_peered":"2024-08-29T15:15:53.017497+0000","last_clean":"2024-08-29T15:15:53.017497+0000","last_became_active":"2024-08-29T15:14:41.707940+0000","last_became_peered":"2024-08-29T15:14:41.707940+0000","last_unstale":"2024-08-29T15:15:53.017497+0000","last_undegraded":"2024-08-29T15:15:53.017497+0000","last_fullsized":"2024-08-29T15:15:53.017497+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:05:13.324130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"179'4265","reported_seq":4298,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:10.494535+0000","last_change":"2024-08-29T15:15:18.756055+0000","last_active":"2024-08-29T15:16:10.494535+0000","last_peered":"2024-08-29T15:16:10.494535+0000","last_clean":"2024-08-29T15:16:10.494535+0000","last_became_active":"2024-08-29T15:15:18.755737+0000","last_became_peered":"2024-08-29T15:15:18.755737+0000","last_unstale":"2024-08-29T15:16:10.494535+0000","last_undegraded":"2024-08-29T15:16:10.494535+0000","last_fullsized":"2024-08-29T15:16:10.494535+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4265,"log_dups_size":0,"ondisk_log_size":4265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:34.363816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279511040,"num_objects":4265,"num_object_clones":0,"num_object_copies":12795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4265,"num_write_kb":272960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":871,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023176+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:15:53.023176+0000","last_peered":"2024-08-29T15:15:53.023176+0000","last_clean":"2024-08-29T15:15:53.023176+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:15:53.023176+0000","last_undegraded":"2024-08-29T15:15:53.023176+0000","last_fullsized":"2024-08-29T15:15:53.023176+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745329+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:16:12.745329+0000","last_peered":"2024-08-29T15:16:12.745329+0000","last_clean":"2024-08-29T15:16:12.745329+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:16:12.745329+0000","last_undegraded":"2024-08-29T15:16:12.745329+0000","last_fullsized":"2024-08-29T15:16:12.745329+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":219,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745867+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:16:12.745867+0000","last_peered":"2024-08-29T15:16:12.745867+0000","last_clean":"2024-08-29T15:16:12.745867+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:16:12.745867+0000","last_undegraded":"2024-08-29T15:16:12.745867+0000","last_fullsized":"2024-08-29T15:16:12.745867+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"179'4148","reported_seq":4167,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.574637+0000","last_change":"2024-08-29T15:15:18.757881+0000","last_active":"2024-08-29T15:16:09.574637+0000","last_peered":"2024-08-29T15:16:09.574637+0000","last_clean":"2024-08-29T15:16:09.574637+0000","last_became_active":"2024-08-29T15:15:18.757483+0000","last_became_peered":"2024-08-29T15:15:18.757483+0000","last_unstale":"2024-08-29T15:16:09.574637+0000","last_undegraded":"2024-08-29T15:16:09.574637+0000","last_fullsized":"2024-08-29T15:16:09.574637+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4148,"log_dups_size":0,"ondisk_log_size":4148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:39:19.468364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271843328,"num_objects":4148,"num_object_clones":0,"num_object_copies":12444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4148,"num_write_kb":265472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":683,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.742331+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:16:12.742331+0000","last_peered":"2024-08-29T15:16:12.742331+0000","last_clean":"2024-08-29T15:16:12.742331+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:16:12.742331+0000","last_undegraded":"2024-08-29T15:16:12.742331+0000","last_fullsized":"2024-08-29T15:16:12.742331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":204,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768725+0000","last_change":"2024-08-29T15:14:41.711008+0000","last_active":"2024-08-29T15:16:02.768725+0000","last_peered":"2024-08-29T15:16:02.768725+0000","last_clean":"2024-08-29T15:16:02.768725+0000","last_became_active":"2024-08-29T15:14:41.709892+0000","last_became_peered":"2024-08-29T15:14:41.709892+0000","last_unstale":"2024-08-29T15:16:02.768725+0000","last_undegraded":"2024-08-29T15:16:02.768725+0000","last_fullsized":"2024-08-29T15:16:02.768725+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:16.428341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":243,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.769627+0000","last_change":"2024-08-29T15:14:24.771674+0000","last_active":"2024-08-29T15:16:02.769627+0000","last_peered":"2024-08-29T15:16:02.769627+0000","last_clean":"2024-08-29T15:16:02.769627+0000","last_became_active":"2024-08-29T15:14:24.771246+0000","last_became_peered":"2024-08-29T15:14:24.771246+0000","last_unstale":"2024-08-29T15:16:02.769627+0000","last_undegraded":"2024-08-29T15:16:02.769627+0000","last_fullsized":"2024-08-29T15:16:02.769627+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:19:31.634147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"179'4339","reported_seq":4372,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:11.650677+0000","last_change":"2024-08-29T15:15:18.752654+0000","last_active":"2024-08-29T15:16:11.650677+0000","last_peered":"2024-08-29T15:16:11.650677+0000","last_clean":"2024-08-29T15:16:11.650677+0000","last_became_active":"2024-08-29T15:15:18.752238+0000","last_became_peered":"2024-08-29T15:15:18.752238+0000","last_unstale":"2024-08-29T15:16:11.650677+0000","last_undegraded":"2024-08-29T15:16:11.650677+0000","last_fullsized":"2024-08-29T15:16:11.650677+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4339,"log_dups_size":0,"ondisk_log_size":4339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:59:30.256649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284360704,"num_objects":4339,"num_object_clones":0,"num_object_copies":13017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4339,"num_write_kb":277696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":87,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.016222+0000","last_change":"2024-08-29T15:14:41.724584+0000","last_active":"2024-08-29T15:15:53.016222+0000","last_peered":"2024-08-29T15:15:53.016222+0000","last_clean":"2024-08-29T15:15:53.016222+0000","last_became_active":"2024-08-29T15:14:41.711085+0000","last_became_peered":"2024-08-29T15:14:41.711085+0000","last_unstale":"2024-08-29T15:15:53.016222+0000","last_undegraded":"2024-08-29T15:15:53.016222+0000","last_fullsized":"2024-08-29T15:15:53.016222+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:00:29.527981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":152,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017672+0000","last_change":"2024-08-29T15:14:41.714189+0000","last_active":"2024-08-29T15:15:53.017672+0000","last_peered":"2024-08-29T15:15:53.017672+0000","last_clean":"2024-08-29T15:15:53.017672+0000","last_became_active":"2024-08-29T15:14:41.713833+0000","last_became_peered":"2024-08-29T15:14:41.713833+0000","last_unstale":"2024-08-29T15:15:53.017672+0000","last_undegraded":"2024-08-29T15:15:53.017672+0000","last_fullsized":"2024-08-29T15:15:53.017672+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:51.314860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":17,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.016331+0000","last_change":"2024-08-29T15:15:30.024329+0000","last_active":"2024-08-29T15:15:53.016331+0000","last_peered":"2024-08-29T15:15:53.016331+0000","last_clean":"2024-08-29T15:15:53.016331+0000","last_became_active":"2024-08-29T15:15:30.024056+0000","last_became_peered":"2024-08-29T15:15:30.024056+0000","last_unstale":"2024-08-29T15:15:53.016331+0000","last_undegraded":"2024-08-29T15:15:53.016331+0000","last_fullsized":"2024-08-29T15:15:53.016331+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:25:39.008078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"180'4519","reported_seq":4554,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:13.845767+0000","last_change":"2024-08-29T15:15:18.753466+0000","last_active":"2024-08-29T15:16:13.845767+0000","last_peered":"2024-08-29T15:16:13.845767+0000","last_clean":"2024-08-29T15:16:13.845767+0000","last_became_active":"2024-08-29T15:15:18.753190+0000","last_became_peered":"2024-08-29T15:15:18.753190+0000","last_unstale":"2024-08-29T15:16:13.845767+0000","last_undegraded":"2024-08-29T15:16:13.845767+0000","last_fullsized":"2024-08-29T15:16:13.845767+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4519,"log_dups_size":0,"ondisk_log_size":4519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:39:23.536331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296157184,"num_objects":4519,"num_object_clones":0,"num_object_copies":13557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4519,"num_write_kb":289216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":42,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.015929+0000","last_change":"2024-08-29T15:14:24.771445+0000","last_active":"2024-08-29T15:15:53.015929+0000","last_peered":"2024-08-29T15:15:53.015929+0000","last_clean":"2024-08-29T15:15:53.015929+0000","last_became_active":"2024-08-29T15:14:24.770188+0000","last_became_peered":"2024-08-29T15:14:24.770188+0000","last_unstale":"2024-08-29T15:15:53.015929+0000","last_undegraded":"2024-08-29T15:15:53.015929+0000","last_fullsized":"2024-08-29T15:15:53.015929+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:39:43.261706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":647,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.769886+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:16:02.769886+0000","last_peered":"2024-08-29T15:16:02.769886+0000","last_clean":"2024-08-29T15:16:02.769886+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:16:02.769886+0000","last_undegraded":"2024-08-29T15:16:02.769886+0000","last_fullsized":"2024-08-29T15:16:02.769886+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":199,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.769126+0000","last_change":"2024-08-29T15:14:24.768088+0000","last_active":"2024-08-29T15:16:02.769126+0000","last_peered":"2024-08-29T15:16:02.769126+0000","last_clean":"2024-08-29T15:16:02.769126+0000","last_became_active":"2024-08-29T15:14:24.767617+0000","last_became_peered":"2024-08-29T15:14:24.767617+0000","last_unstale":"2024-08-29T15:16:02.769126+0000","last_undegraded":"2024-08-29T15:16:02.769126+0000","last_fullsized":"2024-08-29T15:16:02.769126+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:01:44.687737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"179'4281","reported_seq":4314,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:10.505163+0000","last_change":"2024-08-29T15:15:18.753416+0000","last_active":"2024-08-29T15:16:10.505163+0000","last_peered":"2024-08-29T15:16:10.505163+0000","last_clean":"2024-08-29T15:16:10.505163+0000","last_became_active":"2024-08-29T15:15:18.753105+0000","last_became_peered":"2024-08-29T15:15:18.753105+0000","last_unstale":"2024-08-29T15:16:10.505163+0000","last_undegraded":"2024-08-29T15:16:10.505163+0000","last_fullsized":"2024-08-29T15:16:10.505163+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4281,"log_dups_size":0,"ondisk_log_size":4281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:35.527281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280559616,"num_objects":4281,"num_object_clones":0,"num_object_copies":12843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4281,"num_write_kb":273984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":211,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.769733+0000","last_change":"2024-08-29T15:14:24.771408+0000","last_active":"2024-08-29T15:16:02.769733+0000","last_peered":"2024-08-29T15:16:02.769733+0000","last_clean":"2024-08-29T15:16:02.769733+0000","last_became_active":"2024-08-29T15:14:24.770270+0000","last_became_peered":"2024-08-29T15:14:24.770270+0000","last_unstale":"2024-08-29T15:16:02.769733+0000","last_undegraded":"2024-08-29T15:16:02.769733+0000","last_fullsized":"2024-08-29T15:16:02.769733+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:09:10.706502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":568,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023737+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:15:53.023737+0000","last_peered":"2024-08-29T15:15:53.023737+0000","last_clean":"2024-08-29T15:15:53.023737+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:15:53.023737+0000","last_undegraded":"2024-08-29T15:15:53.023737+0000","last_fullsized":"2024-08-29T15:15:53.023737+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":211,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744958+0000","last_change":"2024-08-29T15:14:24.771845+0000","last_active":"2024-08-29T15:16:12.744958+0000","last_peered":"2024-08-29T15:16:12.744958+0000","last_clean":"2024-08-29T15:16:12.744958+0000","last_became_active":"2024-08-29T15:14:24.771443+0000","last_became_peered":"2024-08-29T15:14:24.771443+0000","last_unstale":"2024-08-29T15:16:12.744958+0000","last_undegraded":"2024-08-29T15:16:12.744958+0000","last_fullsized":"2024-08-29T15:16:12.744958+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:33:30.018083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"179'4182","reported_seq":4202,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.622351+0000","last_change":"2024-08-29T15:15:18.752272+0000","last_active":"2024-08-29T15:16:09.622351+0000","last_peered":"2024-08-29T15:16:09.622351+0000","last_clean":"2024-08-29T15:16:09.622351+0000","last_became_active":"2024-08-29T15:15:18.751972+0000","last_became_peered":"2024-08-29T15:15:18.751972+0000","last_unstale":"2024-08-29T15:16:09.622351+0000","last_undegraded":"2024-08-29T15:16:09.622351+0000","last_fullsized":"2024-08-29T15:16:09.622351+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4182,"log_dups_size":0,"ondisk_log_size":4182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:40:03.741164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274071552,"num_objects":4182,"num_object_clones":0,"num_object_copies":12546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4182,"num_write_kb":267648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":214,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767887+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:16:02.767887+0000","last_peered":"2024-08-29T15:16:02.767887+0000","last_clean":"2024-08-29T15:16:02.767887+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:16:02.767887+0000","last_undegraded":"2024-08-29T15:16:02.767887+0000","last_fullsized":"2024-08-29T15:16:02.767887+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":769,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.770186+0000","last_change":"2024-08-29T15:14:44.732925+0000","last_active":"2024-08-29T15:16:02.770186+0000","last_peered":"2024-08-29T15:16:02.770186+0000","last_clean":"2024-08-29T15:16:02.770186+0000","last_became_active":"2024-08-29T15:14:44.732304+0000","last_became_peered":"2024-08-29T15:14:44.732304+0000","last_unstale":"2024-08-29T15:16:02.770186+0000","last_undegraded":"2024-08-29T15:16:02.770186+0000","last_fullsized":"2024-08-29T15:16:02.770186+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:58:17.177306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":211,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767726+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:16:02.767726+0000","last_peered":"2024-08-29T15:16:02.767726+0000","last_clean":"2024-08-29T15:16:02.767726+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:16:02.767726+0000","last_undegraded":"2024-08-29T15:16:02.767726+0000","last_fullsized":"2024-08-29T15:16:02.767726+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"179'4191","reported_seq":4211,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:09.574500+0000","last_change":"2024-08-29T15:15:18.757134+0000","last_active":"2024-08-29T15:16:09.574500+0000","last_peered":"2024-08-29T15:16:09.574500+0000","last_clean":"2024-08-29T15:16:09.574500+0000","last_became_active":"2024-08-29T15:15:18.756774+0000","last_became_peered":"2024-08-29T15:15:18.756774+0000","last_unstale":"2024-08-29T15:16:09.574500+0000","last_undegraded":"2024-08-29T15:16:09.574500+0000","last_fullsized":"2024-08-29T15:16:09.574500+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4191,"log_dups_size":0,"ondisk_log_size":4191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:24:07.106346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274661376,"num_objects":4191,"num_object_clones":0,"num_object_copies":12573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4191,"num_write_kb":268224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":212,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745750+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:16:12.745750+0000","last_peered":"2024-08-29T15:16:12.745750+0000","last_clean":"2024-08-29T15:16:12.745750+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:16:12.745750+0000","last_undegraded":"2024-08-29T15:16:12.745750+0000","last_fullsized":"2024-08-29T15:16:12.745750+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":340,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023883+0000","last_change":"2024-08-29T15:14:41.726971+0000","last_active":"2024-08-29T15:15:53.023883+0000","last_peered":"2024-08-29T15:15:53.023883+0000","last_clean":"2024-08-29T15:15:53.023883+0000","last_became_active":"2024-08-29T15:14:41.709484+0000","last_became_peered":"2024-08-29T15:14:41.709484+0000","last_unstale":"2024-08-29T15:15:53.023883+0000","last_undegraded":"2024-08-29T15:15:53.023883+0000","last_fullsized":"2024-08-29T15:15:53.023883+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:51:28.628784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":154,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017146+0000","last_change":"2024-08-29T15:14:41.709697+0000","last_active":"2024-08-29T15:15:53.017146+0000","last_peered":"2024-08-29T15:15:53.017146+0000","last_clean":"2024-08-29T15:15:53.017146+0000","last_became_active":"2024-08-29T15:14:41.707901+0000","last_became_peered":"2024-08-29T15:14:41.707901+0000","last_unstale":"2024-08-29T15:15:53.017146+0000","last_undegraded":"2024-08-29T15:15:53.017146+0000","last_fullsized":"2024-08-29T15:15:53.017146+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:32:42.182970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":159,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017523+0000","last_change":"2024-08-29T15:14:41.709215+0000","last_active":"2024-08-29T15:15:53.017523+0000","last_peered":"2024-08-29T15:15:53.017523+0000","last_clean":"2024-08-29T15:15:53.017523+0000","last_became_active":"2024-08-29T15:14:41.707784+0000","last_became_peered":"2024-08-29T15:14:41.707784+0000","last_unstale":"2024-08-29T15:15:53.017523+0000","last_undegraded":"2024-08-29T15:15:53.017523+0000","last_fullsized":"2024-08-29T15:15:53.017523+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:21:24.116890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":169,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017620+0000","last_change":"2024-08-29T15:14:41.719049+0000","last_active":"2024-08-29T15:15:53.017620+0000","last_peered":"2024-08-29T15:15:53.017620+0000","last_clean":"2024-08-29T15:15:53.017620+0000","last_became_active":"2024-08-29T15:14:41.705898+0000","last_became_peered":"2024-08-29T15:14:41.705898+0000","last_unstale":"2024-08-29T15:15:53.017620+0000","last_undegraded":"2024-08-29T15:15:53.017620+0000","last_fullsized":"2024-08-29T15:15:53.017620+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:40:45.948387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":202,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768970+0000","last_change":"2024-08-29T15:14:41.711774+0000","last_active":"2024-08-29T15:16:02.768970+0000","last_peered":"2024-08-29T15:16:02.768970+0000","last_clean":"2024-08-29T15:16:02.768970+0000","last_became_active":"2024-08-29T15:14:41.711453+0000","last_became_peered":"2024-08-29T15:14:41.711453+0000","last_unstale":"2024-08-29T15:16:02.768970+0000","last_undegraded":"2024-08-29T15:16:02.768970+0000","last_fullsized":"2024-08-29T15:16:02.768970+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:10:12.878022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.023637+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:15:53.023637+0000","last_peered":"2024-08-29T15:15:53.023637+0000","last_clean":"2024-08-29T15:15:53.023637+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:15:53.023637+0000","last_undegraded":"2024-08-29T15:15:53.023637+0000","last_fullsized":"2024-08-29T15:15:53.023637+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":42,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.015583+0000","last_change":"2024-08-29T15:14:24.769581+0000","last_active":"2024-08-29T15:15:53.015583+0000","last_peered":"2024-08-29T15:15:53.015583+0000","last_clean":"2024-08-29T15:15:53.015583+0000","last_became_active":"2024-08-29T15:14:24.769060+0000","last_became_peered":"2024-08-29T15:14:24.769060+0000","last_unstale":"2024-08-29T15:15:53.015583+0000","last_undegraded":"2024-08-29T15:15:53.015583+0000","last_fullsized":"2024-08-29T15:15:53.015583+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:27:52.647907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768120+0000","last_change":"2024-08-29T15:14:41.710653+0000","last_active":"2024-08-29T15:16:02.768120+0000","last_peered":"2024-08-29T15:16:02.768120+0000","last_clean":"2024-08-29T15:16:02.768120+0000","last_became_active":"2024-08-29T15:14:41.708547+0000","last_became_peered":"2024-08-29T15:14:41.708547+0000","last_unstale":"2024-08-29T15:16:02.768120+0000","last_undegraded":"2024-08-29T15:16:02.768120+0000","last_fullsized":"2024-08-29T15:16:02.768120+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:18:57.194953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":222,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745462+0000","last_change":"2024-08-29T15:14:24.770144+0000","last_active":"2024-08-29T15:16:12.745462+0000","last_peered":"2024-08-29T15:16:12.745462+0000","last_clean":"2024-08-29T15:16:12.745462+0000","last_became_active":"2024-08-29T15:14:24.769805+0000","last_became_peered":"2024-08-29T15:14:24.769805+0000","last_unstale":"2024-08-29T15:16:12.745462+0000","last_undegraded":"2024-08-29T15:16:12.745462+0000","last_fullsized":"2024-08-29T15:16:12.745462+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:59:31.140323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":223,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767574+0000","last_change":"2024-08-29T15:14:41.714831+0000","last_active":"2024-08-29T15:16:02.767574+0000","last_peered":"2024-08-29T15:16:02.767574+0000","last_clean":"2024-08-29T15:16:02.767574+0000","last_became_active":"2024-08-29T15:14:41.714418+0000","last_became_peered":"2024-08-29T15:14:41.714418+0000","last_unstale":"2024-08-29T15:16:02.767574+0000","last_undegraded":"2024-08-29T15:16:02.767574+0000","last_fullsized":"2024-08-29T15:16:02.767574+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:29.132215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":157,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.016013+0000","last_change":"2024-08-29T15:14:41.710252+0000","last_active":"2024-08-29T15:15:53.016013+0000","last_peered":"2024-08-29T15:15:53.016013+0000","last_clean":"2024-08-29T15:15:53.016013+0000","last_became_active":"2024-08-29T15:14:41.709936+0000","last_became_peered":"2024-08-29T15:14:41.709936+0000","last_unstale":"2024-08-29T15:15:53.016013+0000","last_undegraded":"2024-08-29T15:15:53.016013+0000","last_fullsized":"2024-08-29T15:15:53.016013+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:01:26.996814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":232,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767952+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:16:02.767952+0000","last_peered":"2024-08-29T15:16:02.767952+0000","last_clean":"2024-08-29T15:16:02.767952+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:16:02.767952+0000","last_undegraded":"2024-08-29T15:16:02.767952+0000","last_fullsized":"2024-08-29T15:16:02.767952+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745298+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:16:12.745298+0000","last_peered":"2024-08-29T15:16:12.745298+0000","last_clean":"2024-08-29T15:16:12.745298+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:16:12.745298+0000","last_undegraded":"2024-08-29T15:16:12.745298+0000","last_fullsized":"2024-08-29T15:16:12.745298+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":196,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768755+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:16:02.768755+0000","last_peered":"2024-08-29T15:16:02.768755+0000","last_clean":"2024-08-29T15:16:02.768755+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:16:02.768755+0000","last_undegraded":"2024-08-29T15:16:02.768755+0000","last_fullsized":"2024-08-29T15:16:02.768755+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":214,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767901+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:16:02.767901+0000","last_peered":"2024-08-29T15:16:02.767901+0000","last_clean":"2024-08-29T15:16:02.767901+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:16:02.767901+0000","last_undegraded":"2024-08-29T15:16:02.767901+0000","last_fullsized":"2024-08-29T15:16:02.767901+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":158,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017600+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:15:53.017600+0000","last_peered":"2024-08-29T15:15:53.017600+0000","last_clean":"2024-08-29T15:15:53.017600+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:15:53.017600+0000","last_undegraded":"2024-08-29T15:15:53.017600+0000","last_fullsized":"2024-08-29T15:15:53.017600+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":244,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.750035+0000","last_change":"2024-08-29T15:15:30.439845+0000","last_active":"2024-08-29T15:16:12.750035+0000","last_peered":"2024-08-29T15:16:12.750035+0000","last_clean":"2024-08-29T15:16:12.750035+0000","last_became_active":"2024-08-29T15:15:30.439476+0000","last_became_peered":"2024-08-29T15:15:30.439476+0000","last_unstale":"2024-08-29T15:16:12.750035+0000","last_undegraded":"2024-08-29T15:16:12.750035+0000","last_fullsized":"2024-08-29T15:16:12.750035+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:16:45.599107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":200,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744854+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:16:12.744854+0000","last_peered":"2024-08-29T15:16:12.744854+0000","last_clean":"2024-08-29T15:16:12.744854+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:16:12.744854+0000","last_undegraded":"2024-08-29T15:16:12.744854+0000","last_fullsized":"2024-08-29T15:16:12.744854+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":156,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017451+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:15:53.017451+0000","last_peered":"2024-08-29T15:15:53.017451+0000","last_clean":"2024-08-29T15:15:53.017451+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:15:53.017451+0000","last_undegraded":"2024-08-29T15:15:53.017451+0000","last_fullsized":"2024-08-29T15:15:53.017451+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":46,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.016682+0000","last_change":"2024-08-29T15:14:24.778194+0000","last_active":"2024-08-29T15:15:53.016682+0000","last_peered":"2024-08-29T15:15:53.016682+0000","last_clean":"2024-08-29T15:15:53.016682+0000","last_became_active":"2024-08-29T15:14:24.771664+0000","last_became_peered":"2024-08-29T15:14:24.771664+0000","last_unstale":"2024-08-29T15:15:53.016682+0000","last_undegraded":"2024-08-29T15:15:53.016682+0000","last_fullsized":"2024-08-29T15:15:53.016682+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:38:27.921998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":156,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017426+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:15:53.017426+0000","last_peered":"2024-08-29T15:15:53.017426+0000","last_clean":"2024-08-29T15:15:53.017426+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:15:53.017426+0000","last_undegraded":"2024-08-29T15:15:53.017426+0000","last_fullsized":"2024-08-29T15:15:53.017426+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":59,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.015783+0000","last_change":"2024-08-29T15:14:41.715411+0000","last_active":"2024-08-29T15:15:53.015783+0000","last_peered":"2024-08-29T15:15:53.015783+0000","last_clean":"2024-08-29T15:15:53.015783+0000","last_became_active":"2024-08-29T15:14:41.706868+0000","last_became_peered":"2024-08-29T15:14:41.706868+0000","last_unstale":"2024-08-29T15:15:53.015783+0000","last_undegraded":"2024-08-29T15:15:53.015783+0000","last_fullsized":"2024-08-29T15:15:53.015783+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:47:08.936672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":235,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768426+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:16:02.768426+0000","last_peered":"2024-08-29T15:16:02.768426+0000","last_clean":"2024-08-29T15:16:02.768426+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:16:02.768426+0000","last_undegraded":"2024-08-29T15:16:02.768426+0000","last_fullsized":"2024-08-29T15:16:02.768426+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":230,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768711+0000","last_change":"2024-08-29T15:14:41.706396+0000","last_active":"2024-08-29T15:16:02.768711+0000","last_peered":"2024-08-29T15:16:02.768711+0000","last_clean":"2024-08-29T15:16:02.768711+0000","last_became_active":"2024-08-29T15:14:41.706027+0000","last_became_peered":"2024-08-29T15:14:41.706027+0000","last_unstale":"2024-08-29T15:16:02.768711+0000","last_undegraded":"2024-08-29T15:16:02.768711+0000","last_fullsized":"2024-08-29T15:16:02.768711+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:22:11.025293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":150,"reported_epoch":178,"state":"active+clean","last_fresh":"2024-08-29T15:15:53.017141+0000","last_change":"2024-08-29T15:14:41.715832+0000","last_active":"2024-08-29T15:15:53.017141+0000","last_peered":"2024-08-29T15:15:53.017141+0000","last_clean":"2024-08-29T15:15:53.017141+0000","last_became_active":"2024-08-29T15:14:41.715329+0000","last_became_peered":"2024-08-29T15:14:41.715329+0000","last_unstale":"2024-08-29T15:15:53.017141+0000","last_undegraded":"2024-08-29T15:15:53.017141+0000","last_fullsized":"2024-08-29T15:15:53.017141+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:04.911775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":238,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745342+0000","last_change":"2024-08-29T15:14:24.781220+0000","last_active":"2024-08-29T15:16:12.745342+0000","last_peered":"2024-08-29T15:16:12.745342+0000","last_clean":"2024-08-29T15:16:12.745342+0000","last_became_active":"2024-08-29T15:14:24.771475+0000","last_became_peered":"2024-08-29T15:14:24.771475+0000","last_unstale":"2024-08-29T15:16:12.745342+0000","last_undegraded":"2024-08-29T15:16:12.745342+0000","last_fullsized":"2024-08-29T15:16:12.745342+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:46:20.615745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4469489664,"num_objects":68199,"num_object_clones":0,"num_object_copies":204597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":68199,"num_write_kb":4364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13519618048,"data_stored":13519618048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68199,"ondisk_log_size":68199,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":6130,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234258,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2271068,"kb_used_data":2105428,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":91474084,"statfs":{"total":95995035648,"available":93669462016,"internally_reserved":0,"allocated":2155958272,"data_stored":2155049361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":169605579},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568495894,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757531,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1569668,"kb_used_data":1326076,"kb_used_omap":1,"kb_used_meta":243582,"kb_avail":92175484,"statfs":{"total":95995035648,"available":94387695616,"internally_reserved":0,"allocated":1357901824,"data_stored":1357012313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":249428422},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019168,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1569936,"kb_used_data":1326344,"kb_used_omap":1,"kb_used_meta":243582,"kb_avail":92175216,"statfs":{"total":95995035648,"available":94387421184,"internally_reserved":0,"allocated":1358176256,"data_stored":1357278746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":249428422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435353,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2860612,"kb_used_data":2720700,"kb_used_omap":4,"kb_used_meta":139899,"kb_avail":90884540,"statfs":{"total":95995035648,"available":93065768960,"internally_reserved":0,"allocated":2785996800,"data_stored":2785107121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":143256841},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598880,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3099412,"kb_used_data":2846984,"kb_used_omap":7,"kb_used_meta":252408,"kb_avail":90645740,"statfs":{"total":95995035648,"available":92821237760,"internally_reserved":0,"allocated":2915311616,"data_stored":2914421897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":258466127},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762406,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1790020,"kb_used_data":1627836,"kb_used_omap":7,"kb_used_meta":162168,"kb_avail":91955132,"statfs":{"total":95995035648,"available":94162055168,"internally_reserved":0,"allocated":1666904064,"data_stored":1666014788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":166060386},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925933,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1419860,"kb_used_data":1145224,"kb_used_omap":7,"kb_used_meta":274616,"kb_avail":92325292,"statfs":{"total":95995035648,"available":94541099008,"internally_reserved":0,"allocated":1172709376,"data_stored":1171815261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":281206927},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1183318016,"data_stored":1183318016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1672544256,"data_stored":1672544256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2952200192,"data_stored":2952200192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2814050304,"data_stored":2814050304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1354301440,"data_stored":1354301440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1360986112,"data_stored":1360986112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2182217728,"data_stored":2182217728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:16:17.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:16:17.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:17.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:17.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:17.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:17.255 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:16:17.255 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:16:17.255 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:16:17.255 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:16:17.255 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:16:17.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:17.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:17.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:17.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:17.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:17.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:17.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:17.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:17.587 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 46 16 75660 75644 102.754 68 0.0241474 0.00972666 2024-08-29T15:16:17.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:17.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:17.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:17.705 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:17.707 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:17.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:17.751 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:16:17.751 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:16:17.789 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:16:17.789 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:16:17.790 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:17.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:17.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:17.828 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:16:17.828 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:16:17.866 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:16:17.867 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:16:17.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:17.895 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:17 smithi171 ceph-mon[90031]: pgmap v537: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-29T15:16:17.895 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3571035103' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:17.903 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:16:17.904 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:16:17.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:17 smithi027 ceph-mon[93921]: pgmap v537: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-29T15:16:17.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3571035103' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:17.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:17 smithi027 ceph-mon[98970]: pgmap v537: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-29T15:16:17.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3571035103' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:17.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:16:17.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:17.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:17.941 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:16:17.941 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:16:17.974 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:16:17.975 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:16:18.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:16:18.010 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:16:18.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:18.086 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:18.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:18.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:18.213 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:16:18.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:18.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:18.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:18.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:18.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:18.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:18.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:18.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:18.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:18.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:18.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:18.586 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 47 16 76787 76771 102.066 70.4375 0.0135716 0.00979104 2024-08-29T15:16:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:18.625 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:18.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:18.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:18.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:18.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:18.858 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:18.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:18.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:18.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:19.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:19.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:19.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:19.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:19.139 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:16:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:16:19] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.51.0" 2024-08-29T15:16:19.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:19.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:19.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:19.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:19.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:19.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:19.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:19.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:19.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:19.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:19.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:19.587 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 48 16 77989 77973 101.505 75.125 0.00505627 0.00984585 2024-08-29T15:16:19.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:19.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:19.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:19.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:19.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:19 smithi027 ceph-mon[98970]: pgmap v538: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-29T15:16:19.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:19 smithi027 ceph-mon[93921]: pgmap v538: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-29T15:16:19.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:19.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:19.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:19.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:20.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:19 smithi171 ceph-mon[90031]: pgmap v538: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-29T15:16:20.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:20.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:20.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:20.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:20.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:20.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:20.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:20.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:20.552 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:20.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:20.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 49 16 79156 79140 100.92 72.9375 0.0161222 0.00990338 2024-08-29T15:16:20.653 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:20.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:20.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:20.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:20.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:20.881 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:16:20.881 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:16:20.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:16:20.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:20.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:20.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:20.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:21.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:21.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:21.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:21.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:21.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:21.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:21.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:21.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:21.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:21.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:21.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:21.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:16:21.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:21.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 50 16 80146 80130 100.139 61.875 0.0255077 0.00998003 2024-08-29T15:16:21.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:21.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:21.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:21.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:21.784 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:16:21.785 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:16:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:21.810 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:21.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:21.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:21.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:21.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:21.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:21 smithi027 ceph-mon[98970]: pgmap v539: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-29T15:16:21.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:21 smithi027 ceph-mon[93921]: pgmap v539: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-29T15:16:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:21 smithi171 ceph-mon[90031]: pgmap v539: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-29T15:16:22.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:22.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:22.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:22.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:22.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:22.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:22.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:22.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:22.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:22.365 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:22.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:22.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:22.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:22.506 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:22.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:22.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:22.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 51 16 81164 81148 99.4231 63.625 0.005099 0.0100515 2024-08-29T15:16:22.608 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:22.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:22.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:22.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:22.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:22.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:22.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:22.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:22.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:22.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:23.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:22 smithi171 ceph-mon[90031]: from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:16:23.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:23.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:23.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:22 smithi027 ceph-mon[98970]: from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:16:23.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:22 smithi027 ceph-mon[93921]: from='client.35049 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:16:23.145 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:23.152 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:23.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:23.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:23.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:23.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:23.474 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:23.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:23.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 52 16 82243 82227 98.8079 67.4375 0.00797273 0.0101151 2024-08-29T15:16:23.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:23.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:23.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:23.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:23.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:23.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:23.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:23.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:23.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:23 smithi171 ceph-mon[90031]: pgmap v540: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:16:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:23 smithi027 ceph-mon[98970]: pgmap v540: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:16:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:23 smithi027 ceph-mon[93921]: pgmap v540: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:16:24.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:24.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:24.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:24.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:24.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:16:24.264 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:24.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:24.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:16:24.366 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:24.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:24.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:24.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:24.521 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:24.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:24.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 53 16 83318 83302 98.2113 67.1875 0.0128459 0.010176 2024-08-29T15:16:24.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:24.630 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":539,"stamp":"2024-08-29T15:16:20.467274+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5033629669,"num_objects":76762,"num_object_clones":0,"num_object_copies":230286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76762,"num_whiteouts":0,"num_read":6379,"num_read_kb":13834,"num_write":77857,"num_write_kb":4926017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":2367640,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77916,"ondisk_log_size":77916,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16102676,"kb_used_data":14584860,"kb_used_omap":32,"kb_used_meta":1517727,"kb_avail":640113388,"statfs":{"total":671965249536,"available":655476109312,"internally_reserved":0,"allocated":14934896640,"data_stored":14928617909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33376,"internal_metadata":1554152864},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,6,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":755236864,"num_objects":11524,"num_object_clones":0,"num_object_copies":34572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11524,"num_write_kb":737536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003188"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":231,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.748973+0000","last_change":"2024-08-29T15:08:58.882385+0000","last_active":"2024-08-29T15:16:12.748973+0000","last_peered":"2024-08-29T15:16:12.748973+0000","last_clean":"2024-08-29T15:16:12.748973+0000","last_became_active":"2024-08-29T15:08:58.881953+0000","last_became_peered":"2024-08-29T15:08:58.881953+0000","last_unstale":"2024-08-29T15:16:12.748973+0000","last_undegraded":"2024-08-29T15:16:12.748973+0000","last_fullsized":"2024-08-29T15:16:12.748973+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:03:34.299099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":146,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.769578+0000","last_change":"2024-08-29T15:08:25.185593+0000","last_active":"2024-08-29T15:16:02.769578+0000","last_peered":"2024-08-29T15:16:02.769578+0000","last_clean":"2024-08-29T15:16:02.769578+0000","last_became_active":"2024-08-29T15:08:25.185176+0000","last_became_peered":"2024-08-29T15:08:25.185176+0000","last_unstale":"2024-08-29T15:16:02.769578+0000","last_undegraded":"2024-08-29T15:16:02.769578+0000","last_fullsized":"2024-08-29T15:16:02.769578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:20.124948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":43,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767901+0000","last_change":"2024-08-29T15:14:24.769430+0000","last_active":"2024-08-29T15:16:02.767901+0000","last_peered":"2024-08-29T15:16:02.767901+0000","last_clean":"2024-08-29T15:16:02.767901+0000","last_became_active":"2024-08-29T15:14:24.769068+0000","last_became_peered":"2024-08-29T15:14:24.769068+0000","last_unstale":"2024-08-29T15:16:02.767901+0000","last_undegraded":"2024-08-29T15:16:02.767901+0000","last_fullsized":"2024-08-29T15:16:02.767901+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:03:30.509384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745873+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:16:12.745873+0000","last_peered":"2024-08-29T15:16:12.745873+0000","last_clean":"2024-08-29T15:16:12.745873+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:16:12.745873+0000","last_undegraded":"2024-08-29T15:16:12.745873+0000","last_fullsized":"2024-08-29T15:16:12.745873+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745206+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:16:12.745206+0000","last_peered":"2024-08-29T15:16:12.745206+0000","last_clean":"2024-08-29T15:16:12.745206+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:16:12.745206+0000","last_undegraded":"2024-08-29T15:16:12.745206+0000","last_fullsized":"2024-08-29T15:16:12.745206+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":138,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.768397+0000","last_change":"2024-08-29T15:14:41.708389+0000","last_active":"2024-08-29T15:16:02.768397+0000","last_peered":"2024-08-29T15:16:02.768397+0000","last_clean":"2024-08-29T15:16:02.768397+0000","last_became_active":"2024-08-29T15:14:41.708074+0000","last_became_peered":"2024-08-29T15:14:41.708074+0000","last_unstale":"2024-08-29T15:16:02.768397+0000","last_undegraded":"2024-08-29T15:16:02.768397+0000","last_fullsized":"2024-08-29T15:16:02.768397+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:36:56.089658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":26,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.750104+0000","last_change":"2024-08-29T15:15:30.439343+0000","last_active":"2024-08-29T15:16:12.750104+0000","last_peered":"2024-08-29T15:16:12.750104+0000","last_clean":"2024-08-29T15:16:12.750104+0000","last_became_active":"2024-08-29T15:15:30.438934+0000","last_became_peered":"2024-08-29T15:15:30.438934+0000","last_unstale":"2024-08-29T15:16:12.750104+0000","last_undegraded":"2024-08-29T15:16:12.750104+0000","last_fullsized":"2024-08-29T15:16:12.750104+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:44:45.377680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":87,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745392+0000","last_change":"2024-08-29T15:14:41.720900+0000","last_active":"2024-08-29T15:16:12.745392+0000","last_peered":"2024-08-29T15:16:12.745392+0000","last_clean":"2024-08-29T15:16:12.745392+0000","last_became_active":"2024-08-29T15:14:41.715217+0000","last_became_peered":"2024-08-29T15:14:41.715217+0000","last_unstale":"2024-08-29T15:16:12.745392+0000","last_undegraded":"2024-08-29T15:16:12.745392+0000","last_fullsized":"2024-08-29T15:16:12.745392+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:21:13.058909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.a","version":"180'4925","reported_seq":4946,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.808831+0000","last_change":"2024-08-29T15:15:18.754051+0000","last_active":"2024-08-29T15:16:19.808831+0000","last_peered":"2024-08-29T15:16:19.808831+0000","last_clean":"2024-08-29T15:16:19.808831+0000","last_became_active":"2024-08-29T15:15:18.753647+0000","last_became_peered":"2024-08-29T15:15:18.753647+0000","last_unstale":"2024-08-29T15:16:19.808831+0000","last_undegraded":"2024-08-29T15:16:19.808831+0000","last_fullsized":"2024-08-29T15:16:19.808831+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4925,"log_dups_size":0,"ondisk_log_size":4925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:34:29.574588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322764800,"num_objects":4925,"num_object_clones":0,"num_object_copies":14775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4925,"num_write_kb":315200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":146,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.707103+0000","last_change":"2024-08-29T15:07:20.816034+0000","last_active":"2024-08-29T15:16:02.707103+0000","last_peered":"2024-08-29T15:16:02.707103+0000","last_clean":"2024-08-29T15:16:02.707103+0000","last_became_active":"2024-08-29T15:07:20.815678+0000","last_became_peered":"2024-08-29T15:07:20.815678+0000","last_unstale":"2024-08-29T15:16:02.707103+0000","last_undegraded":"2024-08-29T15:16:02.707103+0000","last_fullsized":"2024-08-29T15:16:02.707103+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:24.804814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766913+0000","last_change":"2024-08-29T15:14:41.707030+0000","last_active":"2024-08-29T15:16:02.766913+0000","last_peered":"2024-08-29T15:16:02.766913+0000","last_clean":"2024-08-29T15:16:02.766913+0000","last_became_active":"2024-08-29T15:14:41.706699+0000","last_became_peered":"2024-08-29T15:14:41.706699+0000","last_unstale":"2024-08-29T15:16:02.766913+0000","last_undegraded":"2024-08-29T15:16:02.766913+0000","last_fullsized":"2024-08-29T15:16:02.766913+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:14:00.323667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"180'4611","reported_seq":4646,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:16.659413+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:16:16.659413+0000","last_peered":"2024-08-29T15:16:16.659413+0000","last_clean":"2024-08-29T15:16:16.659413+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:16:16.659413+0000","last_undegraded":"2024-08-29T15:16:16.659413+0000","last_fullsized":"2024-08-29T15:16:16.659413+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4611,"log_dups_size":0,"ondisk_log_size":4611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302186496,"num_objects":4611,"num_object_clones":0,"num_object_copies":13833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4611,"num_write_kb":295104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":147,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.707149+0000","last_change":"2024-08-29T15:07:52.142518+0000","last_active":"2024-08-29T15:16:02.707149+0000","last_peered":"2024-08-29T15:16:02.707149+0000","last_clean":"2024-08-29T15:16:02.707149+0000","last_became_active":"2024-08-29T15:07:52.142151+0000","last_became_peered":"2024-08-29T15:07:52.142151+0000","last_unstale":"2024-08-29T15:16:02.707149+0000","last_undegraded":"2024-08-29T15:16:02.707149+0000","last_fullsized":"2024-08-29T15:16:02.707149+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:12:56.940922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":158,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.764602+0000","last_change":"2024-08-29T15:08:58.885413+0000","last_active":"2024-08-29T15:16:02.764602+0000","last_peered":"2024-08-29T15:16:02.764602+0000","last_clean":"2024-08-29T15:16:02.764602+0000","last_became_active":"2024-08-29T15:08:58.885119+0000","last_became_peered":"2024-08-29T15:08:58.885119+0000","last_unstale":"2024-08-29T15:16:02.764602+0000","last_undegraded":"2024-08-29T15:16:02.764602+0000","last_fullsized":"2024-08-29T15:16:02.764602+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:29.223332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"180'4865","reported_seq":4886,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.567334+0000","last_change":"2024-08-29T15:15:18.757178+0000","last_active":"2024-08-29T15:16:19.567334+0000","last_peered":"2024-08-29T15:16:19.567334+0000","last_clean":"2024-08-29T15:16:19.567334+0000","last_became_active":"2024-08-29T15:15:18.756864+0000","last_became_peered":"2024-08-29T15:15:18.756864+0000","last_unstale":"2024-08-29T15:16:19.567334+0000","last_undegraded":"2024-08-29T15:16:19.567334+0000","last_fullsized":"2024-08-29T15:16:19.567334+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4865,"log_dups_size":0,"ondisk_log_size":4865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:43.382827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4865,"num_write_kb":311360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":170,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767981+0000","last_change":"2024-08-29T15:14:41.712300+0000","last_active":"2024-08-29T15:16:02.767981+0000","last_peered":"2024-08-29T15:16:02.767981+0000","last_clean":"2024-08-29T15:16:02.767981+0000","last_became_active":"2024-08-29T15:14:41.708025+0000","last_became_peered":"2024-08-29T15:14:41.708025+0000","last_unstale":"2024-08-29T15:16:02.767981+0000","last_undegraded":"2024-08-29T15:16:02.767981+0000","last_fullsized":"2024-08-29T15:16:02.767981+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:27:44.258784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":113,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.743743+0000","last_change":"2024-08-29T15:14:41.710873+0000","last_active":"2024-08-29T15:16:12.743743+0000","last_peered":"2024-08-29T15:16:12.743743+0000","last_clean":"2024-08-29T15:16:12.743743+0000","last_became_active":"2024-08-29T15:14:41.710534+0000","last_became_peered":"2024-08-29T15:14:41.710534+0000","last_unstale":"2024-08-29T15:16:12.743743+0000","last_undegraded":"2024-08-29T15:16:12.743743+0000","last_fullsized":"2024-08-29T15:16:12.743743+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:50.786506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"180'4864","reported_seq":4898,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:18.843241+0000","last_change":"2024-08-29T15:15:18.753804+0000","last_active":"2024-08-29T15:16:18.843241+0000","last_peered":"2024-08-29T15:16:18.843241+0000","last_clean":"2024-08-29T15:16:18.843241+0000","last_became_active":"2024-08-29T15:15:18.753257+0000","last_became_peered":"2024-08-29T15:15:18.753257+0000","last_unstale":"2024-08-29T15:16:18.843241+0000","last_undegraded":"2024-08-29T15:16:18.843241+0000","last_fullsized":"2024-08-29T15:16:18.843241+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4864,"log_dups_size":0,"ondisk_log_size":4864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:59:57.157511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318767104,"num_objects":4864,"num_object_clones":0,"num_object_copies":14592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4864,"num_write_kb":311296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":52,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767567+0000","last_change":"2024-08-29T15:14:41.712203+0000","last_active":"2024-08-29T15:16:02.767567+0000","last_peered":"2024-08-29T15:16:02.767567+0000","last_clean":"2024-08-29T15:16:02.767567+0000","last_became_active":"2024-08-29T15:14:41.711831+0000","last_became_peered":"2024-08-29T15:14:41.711831+0000","last_unstale":"2024-08-29T15:16:02.767567+0000","last_undegraded":"2024-08-29T15:16:02.767567+0000","last_fullsized":"2024-08-29T15:16:02.767567+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:54:29.075989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745082+0000","last_change":"2024-08-29T15:07:52.133957+0000","last_active":"2024-08-29T15:16:12.745082+0000","last_peered":"2024-08-29T15:16:12.745082+0000","last_clean":"2024-08-29T15:16:12.745082+0000","last_became_active":"2024-08-29T15:07:52.133628+0000","last_became_peered":"2024-08-29T15:07:52.133628+0000","last_unstale":"2024-08-29T15:16:12.745082+0000","last_undegraded":"2024-08-29T15:16:12.745082+0000","last_fullsized":"2024-08-29T15:16:12.745082+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:00:18.471899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"180'4890","reported_seq":4911,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.659794+0000","last_change":"2024-08-29T15:15:18.752004+0000","last_active":"2024-08-29T15:16:19.659794+0000","last_peered":"2024-08-29T15:16:19.659794+0000","last_clean":"2024-08-29T15:16:19.659794+0000","last_became_active":"2024-08-29T15:15:18.751547+0000","last_became_peered":"2024-08-29T15:15:18.751547+0000","last_unstale":"2024-08-29T15:16:19.659794+0000","last_undegraded":"2024-08-29T15:16:19.659794+0000","last_fullsized":"2024-08-29T15:16:19.659794+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4890,"log_dups_size":0,"ondisk_log_size":4890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:43:33.859489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320471040,"num_objects":4890,"num_object_clones":0,"num_object_copies":14670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4890,"num_write_kb":312960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":216,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.746453+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:16:12.746453+0000","last_peered":"2024-08-29T15:16:12.746453+0000","last_clean":"2024-08-29T15:16:12.746453+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:16:12.746453+0000","last_undegraded":"2024-08-29T15:16:12.746453+0000","last_fullsized":"2024-08-29T15:16:12.746453+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":152,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.765581+0000","last_change":"2024-08-29T15:14:41.712153+0000","last_active":"2024-08-29T15:16:02.765581+0000","last_peered":"2024-08-29T15:16:02.765581+0000","last_clean":"2024-08-29T15:16:02.765581+0000","last_became_active":"2024-08-29T15:14:41.711815+0000","last_became_peered":"2024-08-29T15:14:41.711815+0000","last_unstale":"2024-08-29T15:16:02.765581+0000","last_undegraded":"2024-08-29T15:16:02.765581+0000","last_fullsized":"2024-08-29T15:16:02.765581+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:28:05.338836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"180'4841","reported_seq":4876,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.085737+0000","last_change":"2024-08-29T15:15:18.751470+0000","last_active":"2024-08-29T15:16:19.085737+0000","last_peered":"2024-08-29T15:16:19.085737+0000","last_clean":"2024-08-29T15:16:19.085737+0000","last_became_active":"2024-08-29T15:15:18.751188+0000","last_became_peered":"2024-08-29T15:15:18.751188+0000","last_unstale":"2024-08-29T15:16:19.085737+0000","last_undegraded":"2024-08-29T15:16:19.085737+0000","last_fullsized":"2024-08-29T15:16:19.085737+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4841,"log_dups_size":0,"ondisk_log_size":4841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:03:58.623718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317259776,"num_objects":4841,"num_object_clones":0,"num_object_copies":14523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4841,"num_write_kb":309824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":89,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.764968+0000","last_change":"2024-08-29T15:14:41.714417+0000","last_active":"2024-08-29T15:16:02.764968+0000","last_peered":"2024-08-29T15:16:02.764968+0000","last_clean":"2024-08-29T15:16:02.764968+0000","last_became_active":"2024-08-29T15:14:41.712773+0000","last_became_peered":"2024-08-29T15:14:41.712773+0000","last_unstale":"2024-08-29T15:16:02.764968+0000","last_undegraded":"2024-08-29T15:16:02.764968+0000","last_fullsized":"2024-08-29T15:16:02.764968+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:29:37.353192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":161,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.765284+0000","last_change":"2024-08-29T15:14:06.704490+0000","last_active":"2024-08-29T15:16:02.765284+0000","last_peered":"2024-08-29T15:16:02.765284+0000","last_clean":"2024-08-29T15:16:02.765284+0000","last_became_active":"2024-08-29T15:14:06.703917+0000","last_became_peered":"2024-08-29T15:14:06.703917+0000","last_unstale":"2024-08-29T15:16:02.765284+0000","last_undegraded":"2024-08-29T15:16:02.765284+0000","last_fullsized":"2024-08-29T15:16:02.765284+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:55:25.179472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"180'4620","reported_seq":4655,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:16.679054+0000","last_change":"2024-08-29T15:15:18.751951+0000","last_active":"2024-08-29T15:16:16.679054+0000","last_peered":"2024-08-29T15:16:16.679054+0000","last_clean":"2024-08-29T15:16:16.679054+0000","last_became_active":"2024-08-29T15:15:18.751645+0000","last_became_peered":"2024-08-29T15:15:18.751645+0000","last_unstale":"2024-08-29T15:16:16.679054+0000","last_undegraded":"2024-08-29T15:16:16.679054+0000","last_fullsized":"2024-08-29T15:16:16.679054+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4620,"log_dups_size":0,"ondisk_log_size":4620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:40:24.969000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4620,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":144,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.764678+0000","last_change":"2024-08-29T15:08:25.187770+0000","last_active":"2024-08-29T15:16:02.764678+0000","last_peered":"2024-08-29T15:16:02.764678+0000","last_clean":"2024-08-29T15:16:02.764678+0000","last_became_active":"2024-08-29T15:08:25.187313+0000","last_became_peered":"2024-08-29T15:08:25.187313+0000","last_unstale":"2024-08-29T15:16:02.764678+0000","last_undegraded":"2024-08-29T15:16:02.764678+0000","last_fullsized":"2024-08-29T15:16:02.764678+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:54.149968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":152,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766651+0000","last_change":"2024-08-29T15:14:41.713336+0000","last_active":"2024-08-29T15:16:02.766651+0000","last_peered":"2024-08-29T15:16:02.766651+0000","last_clean":"2024-08-29T15:16:02.766651+0000","last_became_active":"2024-08-29T15:14:41.712786+0000","last_became_peered":"2024-08-29T15:14:41.712786+0000","last_unstale":"2024-08-29T15:16:02.766651+0000","last_undegraded":"2024-08-29T15:16:02.766651+0000","last_fullsized":"2024-08-29T15:16:02.766651+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:17:48.490496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"180'4970","reported_seq":4991,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.790670+0000","last_change":"2024-08-29T15:15:18.755857+0000","last_active":"2024-08-29T15:16:19.790670+0000","last_peered":"2024-08-29T15:16:19.790670+0000","last_clean":"2024-08-29T15:16:19.790670+0000","last_became_active":"2024-08-29T15:15:18.755586+0000","last_became_peered":"2024-08-29T15:15:18.755586+0000","last_unstale":"2024-08-29T15:16:19.790670+0000","last_undegraded":"2024-08-29T15:16:19.790670+0000","last_fullsized":"2024-08-29T15:16:19.790670+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4970,"log_dups_size":0,"ondisk_log_size":4970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:52:17.577528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325713920,"num_objects":4970,"num_object_clones":0,"num_object_copies":14910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4970,"num_write_kb":318080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":219,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.747301+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:16:12.747301+0000","last_peered":"2024-08-29T15:16:12.747301+0000","last_clean":"2024-08-29T15:16:12.747301+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:16:12.747301+0000","last_undegraded":"2024-08-29T15:16:12.747301+0000","last_fullsized":"2024-08-29T15:16:12.747301+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":197,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.743856+0000","last_change":"2024-08-29T15:08:58.882144+0000","last_active":"2024-08-29T15:16:12.743856+0000","last_peered":"2024-08-29T15:16:12.743856+0000","last_clean":"2024-08-29T15:16:12.743856+0000","last_became_active":"2024-08-29T15:08:58.881833+0000","last_became_peered":"2024-08-29T15:08:58.881833+0000","last_unstale":"2024-08-29T15:16:12.743856+0000","last_undegraded":"2024-08-29T15:16:12.743856+0000","last_fullsized":"2024-08-29T15:16:12.743856+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:45:49.553848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"180'4631","reported_seq":4652,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.814308+0000","last_change":"2024-08-29T15:15:18.752994+0000","last_active":"2024-08-29T15:16:19.814308+0000","last_peered":"2024-08-29T15:16:19.814308+0000","last_clean":"2024-08-29T15:16:19.814308+0000","last_became_active":"2024-08-29T15:15:18.752519+0000","last_became_peered":"2024-08-29T15:15:18.752519+0000","last_unstale":"2024-08-29T15:16:19.814308+0000","last_undegraded":"2024-08-29T15:16:19.814308+0000","last_fullsized":"2024-08-29T15:16:19.814308+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4631,"log_dups_size":0,"ondisk_log_size":4631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:13:25.171934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303497216,"num_objects":4631,"num_object_clones":0,"num_object_copies":13893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4631,"num_write_kb":296384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":499,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766517+0000","last_change":"2024-08-29T15:14:44.744053+0000","last_active":"2024-08-29T15:16:02.766517+0000","last_peered":"2024-08-29T15:16:02.766517+0000","last_clean":"2024-08-29T15:16:02.766517+0000","last_became_active":"2024-08-29T15:14:44.743581+0000","last_became_peered":"2024-08-29T15:14:44.743581+0000","last_unstale":"2024-08-29T15:16:02.766517+0000","last_undegraded":"2024-08-29T15:16:02.766517+0000","last_fullsized":"2024-08-29T15:16:02.766517+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:02:28.470398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":643,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767253+0000","last_change":"2024-08-29T15:14:27.711565+0000","last_active":"2024-08-29T15:16:02.767253+0000","last_peered":"2024-08-29T15:16:02.767253+0000","last_clean":"2024-08-29T15:16:02.767253+0000","last_became_active":"2024-08-29T15:14:27.711044+0000","last_became_peered":"2024-08-29T15:14:27.711044+0000","last_unstale":"2024-08-29T15:16:02.767253+0000","last_undegraded":"2024-08-29T15:16:02.767253+0000","last_fullsized":"2024-08-29T15:16:02.767253+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:54:00.170761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":188,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.746289+0000","last_change":"2024-08-29T15:14:41.714021+0000","last_active":"2024-08-29T15:16:12.746289+0000","last_peered":"2024-08-29T15:16:12.746289+0000","last_clean":"2024-08-29T15:16:12.746289+0000","last_became_active":"2024-08-29T15:14:41.713427+0000","last_became_peered":"2024-08-29T15:14:41.713427+0000","last_unstale":"2024-08-29T15:16:12.746289+0000","last_undegraded":"2024-08-29T15:16:12.746289+0000","last_fullsized":"2024-08-29T15:16:12.746289+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:43:27.654536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":37,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.765020+0000","last_change":"2024-08-29T15:14:41.709125+0000","last_active":"2024-08-29T15:16:02.765020+0000","last_peered":"2024-08-29T15:16:02.765020+0000","last_clean":"2024-08-29T15:16:02.765020+0000","last_became_active":"2024-08-29T15:14:41.707940+0000","last_became_peered":"2024-08-29T15:14:41.707940+0000","last_unstale":"2024-08-29T15:16:02.765020+0000","last_undegraded":"2024-08-29T15:16:02.765020+0000","last_fullsized":"2024-08-29T15:16:02.765020+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:05:13.324130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"180'4623","reported_seq":4657,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:15.554527+0000","last_change":"2024-08-29T15:15:18.756055+0000","last_active":"2024-08-29T15:16:15.554527+0000","last_peered":"2024-08-29T15:16:15.554527+0000","last_clean":"2024-08-29T15:16:15.554527+0000","last_became_active":"2024-08-29T15:15:18.755737+0000","last_became_peered":"2024-08-29T15:15:18.755737+0000","last_unstale":"2024-08-29T15:16:15.554527+0000","last_undegraded":"2024-08-29T15:16:15.554527+0000","last_fullsized":"2024-08-29T15:16:15.554527+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4623,"log_dups_size":0,"ondisk_log_size":4623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:34.363816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302972928,"num_objects":4623,"num_object_clones":0,"num_object_copies":13869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4623,"num_write_kb":295872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":872,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.706733+0000","last_change":"2024-08-29T15:07:52.138219+0000","last_active":"2024-08-29T15:16:02.706733+0000","last_peered":"2024-08-29T15:16:02.706733+0000","last_clean":"2024-08-29T15:16:02.706733+0000","last_became_active":"2024-08-29T15:07:52.137653+0000","last_became_peered":"2024-08-29T15:07:52.137653+0000","last_unstale":"2024-08-29T15:16:02.706733+0000","last_undegraded":"2024-08-29T15:16:02.706733+0000","last_fullsized":"2024-08-29T15:16:02.706733+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:31:31.216034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745329+0000","last_change":"2024-08-29T15:08:58.888268+0000","last_active":"2024-08-29T15:16:12.745329+0000","last_peered":"2024-08-29T15:16:12.745329+0000","last_clean":"2024-08-29T15:16:12.745329+0000","last_became_active":"2024-08-29T15:08:58.887934+0000","last_became_peered":"2024-08-29T15:08:58.887934+0000","last_unstale":"2024-08-29T15:16:12.745329+0000","last_undegraded":"2024-08-29T15:16:12.745329+0000","last_fullsized":"2024-08-29T15:16:12.745329+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:41:07.542272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":219,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745867+0000","last_change":"2024-08-29T15:08:25.188451+0000","last_active":"2024-08-29T15:16:12.745867+0000","last_peered":"2024-08-29T15:16:12.745867+0000","last_clean":"2024-08-29T15:16:12.745867+0000","last_became_active":"2024-08-29T15:08:25.188161+0000","last_became_peered":"2024-08-29T15:08:25.188161+0000","last_unstale":"2024-08-29T15:16:12.745867+0000","last_undegraded":"2024-08-29T15:16:12.745867+0000","last_fullsized":"2024-08-29T15:16:12.745867+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:37.868467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"180'4867","reported_seq":4888,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.580137+0000","last_change":"2024-08-29T15:15:18.757881+0000","last_active":"2024-08-29T15:16:19.580137+0000","last_peered":"2024-08-29T15:16:19.580137+0000","last_clean":"2024-08-29T15:16:19.580137+0000","last_became_active":"2024-08-29T15:15:18.757483+0000","last_became_peered":"2024-08-29T15:15:18.757483+0000","last_unstale":"2024-08-29T15:16:19.580137+0000","last_undegraded":"2024-08-29T15:16:19.580137+0000","last_fullsized":"2024-08-29T15:16:19.580137+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4867,"log_dups_size":0,"ondisk_log_size":4867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:39:19.468364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318963712,"num_objects":4867,"num_object_clones":0,"num_object_copies":14601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4867,"num_write_kb":311488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":683,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.742331+0000","last_change":"2024-08-29T15:08:58.881724+0000","last_active":"2024-08-29T15:16:12.742331+0000","last_peered":"2024-08-29T15:16:12.742331+0000","last_clean":"2024-08-29T15:16:12.742331+0000","last_became_active":"2024-08-29T15:08:58.881158+0000","last_became_peered":"2024-08-29T15:08:58.881158+0000","last_unstale":"2024-08-29T15:16:12.742331+0000","last_undegraded":"2024-08-29T15:16:12.742331+0000","last_fullsized":"2024-08-29T15:16:12.742331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:39.697768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":206,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.743620+0000","last_change":"2024-08-29T15:14:41.711008+0000","last_active":"2024-08-29T15:16:12.743620+0000","last_peered":"2024-08-29T15:16:12.743620+0000","last_clean":"2024-08-29T15:16:12.743620+0000","last_became_active":"2024-08-29T15:14:41.709892+0000","last_became_peered":"2024-08-29T15:14:41.709892+0000","last_unstale":"2024-08-29T15:16:12.743620+0000","last_undegraded":"2024-08-29T15:16:12.743620+0000","last_fullsized":"2024-08-29T15:16:12.743620+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:16.428341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":245,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.743634+0000","last_change":"2024-08-29T15:14:24.771674+0000","last_active":"2024-08-29T15:16:12.743634+0000","last_peered":"2024-08-29T15:16:12.743634+0000","last_clean":"2024-08-29T15:16:12.743634+0000","last_became_active":"2024-08-29T15:14:24.771246+0000","last_became_peered":"2024-08-29T15:14:24.771246+0000","last_unstale":"2024-08-29T15:16:12.743634+0000","last_undegraded":"2024-08-29T15:16:12.743634+0000","last_fullsized":"2024-08-29T15:16:12.743634+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:19:31.634147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"180'4721","reported_seq":4755,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:16.671092+0000","last_change":"2024-08-29T15:15:18.752654+0000","last_active":"2024-08-29T15:16:16.671092+0000","last_peered":"2024-08-29T15:16:16.671092+0000","last_clean":"2024-08-29T15:16:16.671092+0000","last_became_active":"2024-08-29T15:15:18.752238+0000","last_became_peered":"2024-08-29T15:15:18.752238+0000","last_unstale":"2024-08-29T15:16:16.671092+0000","last_undegraded":"2024-08-29T15:16:16.671092+0000","last_fullsized":"2024-08-29T15:16:16.671092+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4721,"log_dups_size":0,"ondisk_log_size":4721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:59:30.256649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309395456,"num_objects":4721,"num_object_clones":0,"num_object_copies":14163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4721,"num_write_kb":302144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":88,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767368+0000","last_change":"2024-08-29T15:14:41.724584+0000","last_active":"2024-08-29T15:16:02.767368+0000","last_peered":"2024-08-29T15:16:02.767368+0000","last_clean":"2024-08-29T15:16:02.767368+0000","last_became_active":"2024-08-29T15:14:41.711085+0000","last_became_peered":"2024-08-29T15:14:41.711085+0000","last_unstale":"2024-08-29T15:16:02.767368+0000","last_undegraded":"2024-08-29T15:16:02.767368+0000","last_fullsized":"2024-08-29T15:16:02.767368+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:00:29.527981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":153,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766534+0000","last_change":"2024-08-29T15:14:41.714189+0000","last_active":"2024-08-29T15:16:02.766534+0000","last_peered":"2024-08-29T15:16:02.766534+0000","last_clean":"2024-08-29T15:16:02.766534+0000","last_became_active":"2024-08-29T15:14:41.713833+0000","last_became_peered":"2024-08-29T15:14:41.713833+0000","last_unstale":"2024-08-29T15:16:02.766534+0000","last_undegraded":"2024-08-29T15:16:02.766534+0000","last_fullsized":"2024-08-29T15:16:02.766534+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:51.314860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767332+0000","last_change":"2024-08-29T15:15:30.024329+0000","last_active":"2024-08-29T15:16:02.767332+0000","last_peered":"2024-08-29T15:16:02.767332+0000","last_clean":"2024-08-29T15:16:02.767332+0000","last_became_active":"2024-08-29T15:15:30.024056+0000","last_became_peered":"2024-08-29T15:15:30.024056+0000","last_unstale":"2024-08-29T15:16:02.767332+0000","last_undegraded":"2024-08-29T15:16:02.767332+0000","last_fullsized":"2024-08-29T15:16:02.767332+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:25:39.008078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"180'4882","reported_seq":4917,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:18.834531+0000","last_change":"2024-08-29T15:15:18.753466+0000","last_active":"2024-08-29T15:16:18.834531+0000","last_peered":"2024-08-29T15:16:18.834531+0000","last_clean":"2024-08-29T15:16:18.834531+0000","last_became_active":"2024-08-29T15:15:18.753190+0000","last_became_peered":"2024-08-29T15:15:18.753190+0000","last_unstale":"2024-08-29T15:16:18.834531+0000","last_undegraded":"2024-08-29T15:16:18.834531+0000","last_fullsized":"2024-08-29T15:16:18.834531+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:39:23.536331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319946752,"num_objects":4882,"num_object_clones":0,"num_object_copies":14646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4882,"num_write_kb":312448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":43,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767975+0000","last_change":"2024-08-29T15:14:24.771445+0000","last_active":"2024-08-29T15:16:02.767975+0000","last_peered":"2024-08-29T15:16:02.767975+0000","last_clean":"2024-08-29T15:16:02.767975+0000","last_became_active":"2024-08-29T15:14:24.770188+0000","last_became_peered":"2024-08-29T15:14:24.770188+0000","last_unstale":"2024-08-29T15:16:02.767975+0000","last_undegraded":"2024-08-29T15:16:02.767975+0000","last_fullsized":"2024-08-29T15:16:02.767975+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:39:43.261706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":649,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744218+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:16:12.744218+0000","last_peered":"2024-08-29T15:16:12.744218+0000","last_clean":"2024-08-29T15:16:12.744218+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:16:12.744218+0000","last_undegraded":"2024-08-29T15:16:12.744218+0000","last_fullsized":"2024-08-29T15:16:12.744218+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":201,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744301+0000","last_change":"2024-08-29T15:14:24.768088+0000","last_active":"2024-08-29T15:16:12.744301+0000","last_peered":"2024-08-29T15:16:12.744301+0000","last_clean":"2024-08-29T15:16:12.744301+0000","last_became_active":"2024-08-29T15:14:24.767617+0000","last_became_peered":"2024-08-29T15:14:24.767617+0000","last_unstale":"2024-08-29T15:16:12.744301+0000","last_undegraded":"2024-08-29T15:16:12.744301+0000","last_fullsized":"2024-08-29T15:16:12.744301+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:01:44.687737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"180'4622","reported_seq":4657,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:15.541777+0000","last_change":"2024-08-29T15:15:18.753416+0000","last_active":"2024-08-29T15:16:15.541777+0000","last_peered":"2024-08-29T15:16:15.541777+0000","last_clean":"2024-08-29T15:16:15.541777+0000","last_became_active":"2024-08-29T15:15:18.753105+0000","last_became_peered":"2024-08-29T15:15:18.753105+0000","last_unstale":"2024-08-29T15:16:15.541777+0000","last_undegraded":"2024-08-29T15:16:15.541777+0000","last_fullsized":"2024-08-29T15:16:15.541777+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4622,"log_dups_size":0,"ondisk_log_size":4622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:39:35.527281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302907392,"num_objects":4622,"num_object_clones":0,"num_object_copies":13866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4622,"num_write_kb":295808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":213,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744038+0000","last_change":"2024-08-29T15:14:24.771408+0000","last_active":"2024-08-29T15:16:12.744038+0000","last_peered":"2024-08-29T15:16:12.744038+0000","last_clean":"2024-08-29T15:16:12.744038+0000","last_became_active":"2024-08-29T15:14:24.770270+0000","last_became_peered":"2024-08-29T15:14:24.770270+0000","last_unstale":"2024-08-29T15:16:12.744038+0000","last_undegraded":"2024-08-29T15:16:12.744038+0000","last_fullsized":"2024-08-29T15:16:12.744038+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:09:10.706502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":569,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766717+0000","last_change":"2024-08-29T15:07:20.816913+0000","last_active":"2024-08-29T15:16:02.766717+0000","last_peered":"2024-08-29T15:16:02.766717+0000","last_clean":"2024-08-29T15:16:02.766717+0000","last_became_active":"2024-08-29T15:07:20.816469+0000","last_became_peered":"2024-08-29T15:07:20.816469+0000","last_unstale":"2024-08-29T15:16:02.766717+0000","last_undegraded":"2024-08-29T15:16:02.766717+0000","last_fullsized":"2024-08-29T15:16:02.766717+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:18:54.484142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":211,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744958+0000","last_change":"2024-08-29T15:14:24.771845+0000","last_active":"2024-08-29T15:16:12.744958+0000","last_peered":"2024-08-29T15:16:12.744958+0000","last_clean":"2024-08-29T15:16:12.744958+0000","last_became_active":"2024-08-29T15:14:24.771443+0000","last_became_peered":"2024-08-29T15:14:24.771443+0000","last_unstale":"2024-08-29T15:16:12.744958+0000","last_undegraded":"2024-08-29T15:16:12.744958+0000","last_fullsized":"2024-08-29T15:16:12.744958+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:33:30.018083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"180'4894","reported_seq":4915,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.658927+0000","last_change":"2024-08-29T15:15:18.752272+0000","last_active":"2024-08-29T15:16:19.658927+0000","last_peered":"2024-08-29T15:16:19.658927+0000","last_clean":"2024-08-29T15:16:19.658927+0000","last_became_active":"2024-08-29T15:15:18.751972+0000","last_became_peered":"2024-08-29T15:15:18.751972+0000","last_unstale":"2024-08-29T15:16:19.658927+0000","last_undegraded":"2024-08-29T15:16:19.658927+0000","last_fullsized":"2024-08-29T15:16:19.658927+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4894,"log_dups_size":0,"ondisk_log_size":4894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:40:03.741164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320733184,"num_objects":4894,"num_object_clones":0,"num_object_copies":14682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4894,"num_write_kb":313216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":216,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.751655+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:16:12.751655+0000","last_peered":"2024-08-29T15:16:12.751655+0000","last_clean":"2024-08-29T15:16:12.751655+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:16:12.751655+0000","last_undegraded":"2024-08-29T15:16:12.751655+0000","last_fullsized":"2024-08-29T15:16:12.751655+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":771,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.743842+0000","last_change":"2024-08-29T15:14:44.732925+0000","last_active":"2024-08-29T15:16:12.743842+0000","last_peered":"2024-08-29T15:16:12.743842+0000","last_clean":"2024-08-29T15:16:12.743842+0000","last_became_active":"2024-08-29T15:14:44.732304+0000","last_became_peered":"2024-08-29T15:14:44.732304+0000","last_unstale":"2024-08-29T15:16:12.743842+0000","last_undegraded":"2024-08-29T15:16:12.743842+0000","last_fullsized":"2024-08-29T15:16:12.743842+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:58:17.177306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":213,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.747095+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:16:12.747095+0000","last_peered":"2024-08-29T15:16:12.747095+0000","last_clean":"2024-08-29T15:16:12.747095+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:16:12.747095+0000","last_undegraded":"2024-08-29T15:16:12.747095+0000","last_fullsized":"2024-08-29T15:16:12.747095+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"180'4908","reported_seq":4929,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:19.584653+0000","last_change":"2024-08-29T15:15:18.757134+0000","last_active":"2024-08-29T15:16:19.584653+0000","last_peered":"2024-08-29T15:16:19.584653+0000","last_clean":"2024-08-29T15:16:19.584653+0000","last_became_active":"2024-08-29T15:15:18.756774+0000","last_became_peered":"2024-08-29T15:15:18.756774+0000","last_unstale":"2024-08-29T15:16:19.584653+0000","last_undegraded":"2024-08-29T15:16:19.584653+0000","last_fullsized":"2024-08-29T15:16:19.584653+0000","mapping_epoch":168,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":4908,"log_dups_size":0,"ondisk_log_size":4908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:24:07.106346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321650688,"num_objects":4908,"num_object_clones":0,"num_object_copies":14724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4908,"num_write_kb":314112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":212,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745750+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:16:12.745750+0000","last_peered":"2024-08-29T15:16:12.745750+0000","last_clean":"2024-08-29T15:16:12.745750+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:16:12.745750+0000","last_undegraded":"2024-08-29T15:16:12.745750+0000","last_fullsized":"2024-08-29T15:16:12.745750+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":341,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767550+0000","last_change":"2024-08-29T15:14:41.726971+0000","last_active":"2024-08-29T15:16:02.767550+0000","last_peered":"2024-08-29T15:16:02.767550+0000","last_clean":"2024-08-29T15:16:02.767550+0000","last_became_active":"2024-08-29T15:14:41.709484+0000","last_became_peered":"2024-08-29T15:14:41.709484+0000","last_unstale":"2024-08-29T15:16:02.767550+0000","last_undegraded":"2024-08-29T15:16:02.767550+0000","last_fullsized":"2024-08-29T15:16:02.767550+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:51:28.628784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745411+0000","last_change":"2024-08-29T15:14:41.709697+0000","last_active":"2024-08-29T15:16:12.745411+0000","last_peered":"2024-08-29T15:16:12.745411+0000","last_clean":"2024-08-29T15:16:12.745411+0000","last_became_active":"2024-08-29T15:14:41.707901+0000","last_became_peered":"2024-08-29T15:14:41.707901+0000","last_unstale":"2024-08-29T15:16:12.745411+0000","last_undegraded":"2024-08-29T15:16:12.745411+0000","last_fullsized":"2024-08-29T15:16:12.745411+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:32:42.182970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766324+0000","last_change":"2024-08-29T15:14:41.709215+0000","last_active":"2024-08-29T15:16:02.766324+0000","last_peered":"2024-08-29T15:16:02.766324+0000","last_clean":"2024-08-29T15:16:02.766324+0000","last_became_active":"2024-08-29T15:14:41.707784+0000","last_became_peered":"2024-08-29T15:14:41.707784+0000","last_unstale":"2024-08-29T15:16:02.766324+0000","last_undegraded":"2024-08-29T15:16:02.766324+0000","last_fullsized":"2024-08-29T15:16:02.766324+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:21:24.116890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":170,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.765503+0000","last_change":"2024-08-29T15:14:41.719049+0000","last_active":"2024-08-29T15:16:02.765503+0000","last_peered":"2024-08-29T15:16:02.765503+0000","last_clean":"2024-08-29T15:16:02.765503+0000","last_became_active":"2024-08-29T15:14:41.705898+0000","last_became_peered":"2024-08-29T15:14:41.705898+0000","last_unstale":"2024-08-29T15:16:02.765503+0000","last_undegraded":"2024-08-29T15:16:02.765503+0000","last_fullsized":"2024-08-29T15:16:02.765503+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:40:45.948387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":204,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.743865+0000","last_change":"2024-08-29T15:14:41.711774+0000","last_active":"2024-08-29T15:16:12.743865+0000","last_peered":"2024-08-29T15:16:12.743865+0000","last_clean":"2024-08-29T15:16:12.743865+0000","last_became_active":"2024-08-29T15:14:41.711453+0000","last_became_peered":"2024-08-29T15:14:41.711453+0000","last_unstale":"2024-08-29T15:16:12.743865+0000","last_undegraded":"2024-08-29T15:16:12.743865+0000","last_fullsized":"2024-08-29T15:16:12.743865+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:10:12.878022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":147,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766460+0000","last_change":"2024-08-29T15:08:58.886502+0000","last_active":"2024-08-29T15:16:02.766460+0000","last_peered":"2024-08-29T15:16:02.766460+0000","last_clean":"2024-08-29T15:16:02.766460+0000","last_became_active":"2024-08-29T15:08:58.886155+0000","last_became_peered":"2024-08-29T15:08:58.886155+0000","last_unstale":"2024-08-29T15:16:02.766460+0000","last_undegraded":"2024-08-29T15:16:02.766460+0000","last_fullsized":"2024-08-29T15:16:02.766460+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:33:37.696805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":43,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767349+0000","last_change":"2024-08-29T15:14:24.769581+0000","last_active":"2024-08-29T15:16:02.767349+0000","last_peered":"2024-08-29T15:16:02.767349+0000","last_clean":"2024-08-29T15:16:02.767349+0000","last_became_active":"2024-08-29T15:14:24.769060+0000","last_became_peered":"2024-08-29T15:14:24.769060+0000","last_unstale":"2024-08-29T15:16:02.767349+0000","last_undegraded":"2024-08-29T15:16:02.767349+0000","last_fullsized":"2024-08-29T15:16:02.767349+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:27:52.647907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744727+0000","last_change":"2024-08-29T15:14:41.710653+0000","last_active":"2024-08-29T15:16:12.744727+0000","last_peered":"2024-08-29T15:16:12.744727+0000","last_clean":"2024-08-29T15:16:12.744727+0000","last_became_active":"2024-08-29T15:14:41.708547+0000","last_became_peered":"2024-08-29T15:14:41.708547+0000","last_unstale":"2024-08-29T15:16:12.744727+0000","last_undegraded":"2024-08-29T15:16:12.744727+0000","last_fullsized":"2024-08-29T15:16:12.744727+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:18:57.194953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":222,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745462+0000","last_change":"2024-08-29T15:14:24.770144+0000","last_active":"2024-08-29T15:16:12.745462+0000","last_peered":"2024-08-29T15:16:12.745462+0000","last_clean":"2024-08-29T15:16:12.745462+0000","last_became_active":"2024-08-29T15:14:24.769805+0000","last_became_peered":"2024-08-29T15:14:24.769805+0000","last_unstale":"2024-08-29T15:16:12.745462+0000","last_undegraded":"2024-08-29T15:16:12.745462+0000","last_fullsized":"2024-08-29T15:16:12.745462+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:59:31.140323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":225,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.747018+0000","last_change":"2024-08-29T15:14:41.714831+0000","last_active":"2024-08-29T15:16:12.747018+0000","last_peered":"2024-08-29T15:16:12.747018+0000","last_clean":"2024-08-29T15:16:12.747018+0000","last_became_active":"2024-08-29T15:14:41.714418+0000","last_became_peered":"2024-08-29T15:14:41.714418+0000","last_unstale":"2024-08-29T15:16:12.747018+0000","last_undegraded":"2024-08-29T15:16:12.747018+0000","last_fullsized":"2024-08-29T15:16:12.747018+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:15:29.132215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":158,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767717+0000","last_change":"2024-08-29T15:14:41.710252+0000","last_active":"2024-08-29T15:16:02.767717+0000","last_peered":"2024-08-29T15:16:02.767717+0000","last_clean":"2024-08-29T15:16:02.767717+0000","last_became_active":"2024-08-29T15:14:41.709936+0000","last_became_peered":"2024-08-29T15:14:41.709936+0000","last_unstale":"2024-08-29T15:16:02.767717+0000","last_undegraded":"2024-08-29T15:16:02.767717+0000","last_fullsized":"2024-08-29T15:16:02.767717+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:01:26.996814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":234,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.747220+0000","last_change":"2024-08-29T15:08:58.885893+0000","last_active":"2024-08-29T15:16:12.747220+0000","last_peered":"2024-08-29T15:16:12.747220+0000","last_clean":"2024-08-29T15:16:12.747220+0000","last_became_active":"2024-08-29T15:08:58.885565+0000","last_became_peered":"2024-08-29T15:08:58.885565+0000","last_unstale":"2024-08-29T15:16:12.747220+0000","last_undegraded":"2024-08-29T15:16:12.747220+0000","last_fullsized":"2024-08-29T15:16:12.747220+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:29:01.370313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745298+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:16:12.745298+0000","last_peered":"2024-08-29T15:16:12.745298+0000","last_clean":"2024-08-29T15:16:12.745298+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:16:12.745298+0000","last_undegraded":"2024-08-29T15:16:12.745298+0000","last_fullsized":"2024-08-29T15:16:12.745298+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":198,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744149+0000","last_change":"2024-08-29T15:08:58.883563+0000","last_active":"2024-08-29T15:16:12.744149+0000","last_peered":"2024-08-29T15:16:12.744149+0000","last_clean":"2024-08-29T15:16:12.744149+0000","last_became_active":"2024-08-29T15:08:58.883243+0000","last_became_peered":"2024-08-29T15:08:58.883243+0000","last_unstale":"2024-08-29T15:16:12.744149+0000","last_undegraded":"2024-08-29T15:16:12.744149+0000","last_fullsized":"2024-08-29T15:16:12.744149+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:16:04.642839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":216,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.749251+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:16:12.749251+0000","last_peered":"2024-08-29T15:16:12.749251+0000","last_clean":"2024-08-29T15:16:12.749251+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:16:12.749251+0000","last_undegraded":"2024-08-29T15:16:12.749251+0000","last_fullsized":"2024-08-29T15:16:12.749251+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":159,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.765049+0000","last_change":"2024-08-29T15:08:25.186689+0000","last_active":"2024-08-29T15:16:02.765049+0000","last_peered":"2024-08-29T15:16:02.765049+0000","last_clean":"2024-08-29T15:16:02.765049+0000","last_became_active":"2024-08-29T15:08:25.186346+0000","last_became_peered":"2024-08-29T15:08:25.186346+0000","last_unstale":"2024-08-29T15:16:02.765049+0000","last_undegraded":"2024-08-29T15:16:02.765049+0000","last_fullsized":"2024-08-29T15:16:02.765049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:47.044600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":244,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.750035+0000","last_change":"2024-08-29T15:15:30.439845+0000","last_active":"2024-08-29T15:16:12.750035+0000","last_peered":"2024-08-29T15:16:12.750035+0000","last_clean":"2024-08-29T15:16:12.750035+0000","last_became_active":"2024-08-29T15:15:30.439476+0000","last_became_peered":"2024-08-29T15:15:30.439476+0000","last_unstale":"2024-08-29T15:16:12.750035+0000","last_undegraded":"2024-08-29T15:16:12.750035+0000","last_fullsized":"2024-08-29T15:16:12.750035+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:16:45.599107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":200,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.744854+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:16:12.744854+0000","last_peered":"2024-08-29T15:16:12.744854+0000","last_clean":"2024-08-29T15:16:12.744854+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:16:12.744854+0000","last_undegraded":"2024-08-29T15:16:12.744854+0000","last_fullsized":"2024-08-29T15:16:12.744854+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":157,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.766252+0000","last_change":"2024-08-29T15:08:58.884996+0000","last_active":"2024-08-29T15:16:02.766252+0000","last_peered":"2024-08-29T15:16:02.766252+0000","last_clean":"2024-08-29T15:16:02.766252+0000","last_became_active":"2024-08-29T15:08:58.884587+0000","last_became_peered":"2024-08-29T15:08:58.884587+0000","last_unstale":"2024-08-29T15:16:02.766252+0000","last_undegraded":"2024-08-29T15:16:02.766252+0000","last_fullsized":"2024-08-29T15:16:02.766252+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:35.799624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":47,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767647+0000","last_change":"2024-08-29T15:14:24.778194+0000","last_active":"2024-08-29T15:16:02.767647+0000","last_peered":"2024-08-29T15:16:02.767647+0000","last_clean":"2024-08-29T15:16:02.767647+0000","last_became_active":"2024-08-29T15:14:24.771664+0000","last_became_peered":"2024-08-29T15:14:24.771664+0000","last_unstale":"2024-08-29T15:16:02.767647+0000","last_undegraded":"2024-08-29T15:16:02.767647+0000","last_fullsized":"2024-08-29T15:16:02.767647+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:38:27.921998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":157,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.764892+0000","last_change":"2024-08-29T15:08:58.884919+0000","last_active":"2024-08-29T15:16:02.764892+0000","last_peered":"2024-08-29T15:16:02.764892+0000","last_clean":"2024-08-29T15:16:02.764892+0000","last_became_active":"2024-08-29T15:08:58.884608+0000","last_became_peered":"2024-08-29T15:08:58.884608+0000","last_unstale":"2024-08-29T15:16:02.764892+0000","last_undegraded":"2024-08-29T15:16:02.764892+0000","last_fullsized":"2024-08-29T15:16:02.764892+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:58:04.591754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":60,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.767824+0000","last_change":"2024-08-29T15:14:41.715411+0000","last_active":"2024-08-29T15:16:02.767824+0000","last_peered":"2024-08-29T15:16:02.767824+0000","last_clean":"2024-08-29T15:16:02.767824+0000","last_became_active":"2024-08-29T15:14:41.706868+0000","last_became_peered":"2024-08-29T15:14:41.706868+0000","last_unstale":"2024-08-29T15:16:02.767824+0000","last_undegraded":"2024-08-29T15:16:02.767824+0000","last_fullsized":"2024-08-29T15:16:02.767824+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:47:08.936672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":237,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745060+0000","last_change":"2024-08-29T15:08:58.888010+0000","last_active":"2024-08-29T15:16:12.745060+0000","last_peered":"2024-08-29T15:16:12.745060+0000","last_clean":"2024-08-29T15:16:12.745060+0000","last_became_active":"2024-08-29T15:08:58.886351+0000","last_became_peered":"2024-08-29T15:08:58.886351+0000","last_unstale":"2024-08-29T15:16:12.745060+0000","last_undegraded":"2024-08-29T15:16:12.745060+0000","last_fullsized":"2024-08-29T15:16:12.745060+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:39.544734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":232,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.751428+0000","last_change":"2024-08-29T15:14:41.706396+0000","last_active":"2024-08-29T15:16:12.751428+0000","last_peered":"2024-08-29T15:16:12.751428+0000","last_clean":"2024-08-29T15:16:12.751428+0000","last_became_active":"2024-08-29T15:14:41.706027+0000","last_became_peered":"2024-08-29T15:14:41.706027+0000","last_unstale":"2024-08-29T15:16:12.751428+0000","last_undegraded":"2024-08-29T15:16:12.751428+0000","last_fullsized":"2024-08-29T15:16:12.751428+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:22:11.025293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":151,"reported_epoch":179,"state":"active+clean","last_fresh":"2024-08-29T15:16:02.764534+0000","last_change":"2024-08-29T15:14:41.715832+0000","last_active":"2024-08-29T15:16:02.764534+0000","last_peered":"2024-08-29T15:16:02.764534+0000","last_clean":"2024-08-29T15:16:02.764534+0000","last_became_active":"2024-08-29T15:14:41.715329+0000","last_became_peered":"2024-08-29T15:14:41.715329+0000","last_unstale":"2024-08-29T15:16:02.764534+0000","last_undegraded":"2024-08-29T15:16:02.764534+0000","last_fullsized":"2024-08-29T15:16:02.764534+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:04.911775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":238,"reported_epoch":180,"state":"active+clean","last_fresh":"2024-08-29T15:16:12.745342+0000","last_change":"2024-08-29T15:14:24.781220+0000","last_active":"2024-08-29T15:16:12.745342+0000","last_peered":"2024-08-29T15:16:12.745342+0000","last_clean":"2024-08-29T15:16:12.745342+0000","last_became_active":"2024-08-29T15:14:24.771475+0000","last_became_peered":"2024-08-29T15:14:24.771475+0000","last_unstale":"2024-08-29T15:16:12.745342+0000","last_undegraded":"2024-08-29T15:16:12.745342+0000","last_fullsized":"2024-08-29T15:16:12.745342+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:46:20.615745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5028839424,"num_objects":76734,"num_object_clones":0,"num_object_copies":230202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":76734,"num_write_kb":4910976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15057747968,"data_stored":15057747968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76734,"ondisk_log_size":76734,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":6130,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18303,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234260,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2644904,"kb_used_data":2469408,"kb_used_omap":1,"kb_used_meta":175486,"kb_avail":91100248,"statfs":{"total":95995035648,"available":93286653952,"internally_reserved":0,"allocated":2528673792,"data_stored":2527760879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":179698123},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568495896,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757533,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1827280,"kb_used_data":1553352,"kb_used_omap":1,"kb_used_meta":273918,"kb_avail":91917872,"statfs":{"total":95995035648,"available":94123900928,"internally_reserved":0,"allocated":1590632448,"data_stored":1589738935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":280492486},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019170,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1807772,"kb_used_data":1551892,"kb_used_omap":1,"kb_used_meta":255870,"kb_avail":91937380,"statfs":{"total":95995035648,"available":94143877120,"internally_reserved":0,"allocated":1589137408,"data_stored":1588235896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":262011334},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435354,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3078544,"kb_used_data":2942792,"kb_used_omap":4,"kb_used_meta":135739,"kb_avail":90666608,"statfs":{"total":95995035648,"available":92842606592,"internally_reserved":0,"allocated":3013419008,"data_stored":3012525327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":138997001},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":104,"seq":446676598881,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3313040,"kb_used_data":3068936,"kb_used_omap":7,"kb_used_meta":244088,"kb_avail":90432112,"statfs":{"total":95995035648,"available":92602482688,"internally_reserved":0,"allocated":3142590464,"data_stored":3141700745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7857,"internal_metadata":249946447},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762407,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1930768,"kb_used_data":1760264,"kb_used_omap":7,"kb_used_meta":170488,"kb_avail":91814384,"statfs":{"total":95995035648,"available":94017929216,"internally_reserved":0,"allocated":1802510336,"data_stored":1801617058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":174580066},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925934,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1500368,"kb_used_data":1238216,"kb_used_omap":7,"kb_used_meta":262136,"kb_avail":92244784,"statfs":{"total":95995035648,"available":94458658816,"internally_reserved":0,"allocated":1267933184,"data_stored":1267039069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":268427407},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2341,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1279000576,"data_stored":1279000576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1809907712,"data_stored":1809907712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3184066560,"data_stored":3184066560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3047358464,"data_stored":3047358464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1586823168,"data_stored":1586823168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1593442304,"data_stored":1593442304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2557149184,"data_stored":2557149184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:16:24.634 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-29T15:16:24.634 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:16:24.634 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-29T15:16:24.634 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-29T15:16:24.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2 2024-08-29T15:16:24.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:24.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:24.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:24.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:24.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:24.853 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:24 smithi027 systemd[1]: Stopping Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:16:24.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:24.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:24.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:24.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:25.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:25.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:25.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:25.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:25.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:25.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:25.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:25.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:25.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:25.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:25.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:25.474 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:25.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:25.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:25.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 54 16 84388 84372 97.6309 66.875 0.0135087 0.0102373 2024-08-29T15:16:25.601 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:25.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:25.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:25.687 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:25.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:25.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:25.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:25.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:25.858 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:25.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:25.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:25.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:26.008 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:26.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:25 smithi171 ceph-mon[90031]: pgmap v541: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:16:26.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:25 smithi027 ceph-mon[98970]: pgmap v541: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:16:26.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:25 smithi027 ceph-mon[93921]: pgmap v541: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:16:26.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:26.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:26.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:26.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:26.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:26.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:26.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:26.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:26.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:26.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:26.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 55 16 85406 85390 97.0126 63.625 0.00794107 0.0103026 2024-08-29T15:16:26.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:26.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:26.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:26.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:26.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:26.788 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:16:26.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:26.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:26.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:26.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:26.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:26.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3943740300' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:26 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3943740300' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:26 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:27.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:26 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[119051]: 2024-08-29T15:16:26.868+0000 7f5f8da85640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:16:27.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:26 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[119051]: 2024-08-29T15:16:26.868+0000 7f5f8da85640 -1 osd.2 181 *** Got signal Terminated *** 2024-08-29T15:16:27.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:26 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[119051]: 2024-08-29T15:16:26.868+0000 7f5f8da85640 -1 osd.2 181 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:16:27.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:27.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:27.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3943740300' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:26 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:27.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:27.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:27.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:27.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:27.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:27.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:27.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:27.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:27.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:27.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:27.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 56 16 86556 86540 96.5637 71.875 0.0141217 0.0103509 2024-08-29T15:16:27.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:27.703 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:27.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:27.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:27.831 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:27.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:27.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:27.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:27.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[93921]: pgmap v542: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[93921]: osdmap e181: 8 total, 8 up, 7 in 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[93921]: osd.2 marked itself down and dead 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[98970]: pgmap v542: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[98970]: osdmap e181: 8 total, 8 up, 7 in 2024-08-29T15:16:27.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:27 smithi027 ceph-mon[98970]: osd.2 marked itself down and dead 2024-08-29T15:16:28.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:28.077 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:28.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:28.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:28.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:27 smithi171 ceph-mon[90031]: pgmap v542: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:16:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:27 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:16:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:27 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:16:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:27 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:27 smithi171 ceph-mon[90031]: osdmap e181: 8 total, 8 up, 7 in 2024-08-29T15:16:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:27 smithi171 ceph-mon[90031]: osd.2 marked itself down and dead 2024-08-29T15:16:28.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:28.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:28.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:28.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:28.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:28.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:28.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 57 16 86752 86736 95.0847 12.25 0.00321962 0.0103654 2024-08-29T15:16:28.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:28.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:28.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:28.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:28.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:28.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:28.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:28.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:29.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:29.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:16:29] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-08-29T15:16:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[98970]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:16:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[98970]: osdmap e182: 8 total, 7 up, 7 in 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[98970]: osdmap e183: 8 total, 7 up, 7 in 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[93921]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[93921]: osdmap e182: 8 total, 7 up, 7 in 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:29.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:28 smithi027 ceph-mon[93921]: osdmap e183: 8 total, 7 up, 7 in 2024-08-29T15:16:29.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:29.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:29.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:29.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:29.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:28 smithi171 ceph-mon[90031]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-29T15:16:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:28 smithi171 ceph-mon[90031]: osdmap e182: 8 total, 7 up, 7 in 2024-08-29T15:16:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:28 smithi171 ceph-mon[90031]: osdmap e183: 8 total, 7 up, 7 in 2024-08-29T15:16:29.332 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:29.355 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:29.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:29.457 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:29.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:29.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:29.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:29.588 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 58 16 87905 87889 94.6877 72.0625 0.00414505 0.0105559 2024-08-29T15:16:29.590 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:29.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:29.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:29.732 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:29.769 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:29.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:29.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:29.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:29.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:29.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:30.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:30.042 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:30.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:30.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:30.068 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:29 smithi027 podman[175688]: 2024-08-29 15:16:29.814265532 +0000 UTC m=+3.574355558 container died 6dd209a7d75d1065ccec8bfd1dadd6e2037a030977b6dc803cacb47cec02ee52 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: pgmap v545: 89 pgs: 4 stale+active+clean, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 7]} : dispatch 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 2, 5, 0]} : dispatch 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 2]} : dispatch 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: pgmap v545: 89 pgs: 4 stale+active+clean, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 7]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 2, 5, 0]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 2]} : dispatch 2024-08-29T15:16:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-29T15:16:30.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:30.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: pgmap v545: 89 pgs: 4 stale+active+clean, 85 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 7]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 2, 5, 0]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 2]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-29T15:16:30.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]} : dispatch 2024-08-29T15:16:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:30.171 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:30.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:30.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:16:30.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:30.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:30.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:30.360 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:16:30.360 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:16:30.406 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:16:30.406 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:16:30.407 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:16:30.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:30.445 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:16:30.445 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:16:30.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:30.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:30.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:30.480 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:16:30.481 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:16:30.517 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:16:30.517 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:16:30.547 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:16:30.547 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:16:30.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:30.583 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:16:30.583 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:16:30.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:30.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:30.589 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 59 16 89331 89315 94.5933 89.125 0.0113351 0.0105663 2024-08-29T15:16:30.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:30.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:30.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:30.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:30 smithi027 podman[175688]: 2024-08-29 15:16:30.587035048 +0000 UTC m=+4.347125069 container cleanup 6dd209a7d75d1065ccec8bfd1dadd6e2037a030977b6dc803cacb47cec02ee52 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-29T15:16:30.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:30 smithi027 bash[175688]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2 2024-08-29T15:16:30.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:30.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:30.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:30.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:30.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:31.008 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:31.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:31.110 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:31.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:31.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 7]}]': finished 2024-08-29T15:16:31.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-08-29T15:16:31.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-08-29T15:16:31.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 2, 5, 0]}]': finished 2024-08-29T15:16:31.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 2]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 5]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: osdmap e184: 8 total, 7 up, 7 in 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/86503117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 7]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-08-29T15:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 2, 5, 0]}]': finished 2024-08-29T15:16:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 2]}]': finished 2024-08-29T15:16:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 5]}]': finished 2024-08-29T15:16:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]}]': finished 2024-08-29T15:16:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2024-08-29T15:16:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-08-29T15:16:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: osdmap e184: 8 total, 7 up, 7 in 2024-08-29T15:16:31.229 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/86503117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:31.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:31.264 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:16:31.265 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:16:31.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.0", "id": [1, 7]}]': finished 2024-08-29T15:16:31.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-08-29T15:16:31.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 2]}]': finished 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [7, 2, 5, 0]}]': finished 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.10", "id": [0, 2]}]': finished 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 5]}]': finished 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]}]': finished 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: osdmap e184: 8 total, 7 up, 7 in 2024-08-29T15:16:31.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/86503117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.282 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:31.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:31.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:31.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:31.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.558 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:31.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:31.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:16:31.599048+0000 min lat: 0.00129485 max lat: 1.05023 avg lat: 0.0105752 2024-08-29T15:16:31.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:16:31.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 60 16 90766 90750 94.4953 89.6875 0.00298686 0.0105752 2024-08-29T15:16:31.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:31.698 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:31.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:31.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:31.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:91157 6.9 6:9c5eb681:::benchmark_data_smithi027_169445_object91156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:31.900305+0000", 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.010712615, 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010750078999999999, 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:16:31.950 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 91157 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900305+0000", 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900305+0000", 2024-08-29T15:16:31.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900305+0000", 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7599999999999997e-07 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900328+0000", 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3037999999999999e-05 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900337+0000", 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1449999999999997e-06 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900344+0000", 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3949999999999998e-06 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.904680+0000", 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043359549999999998 2024-08-29T15:16:31.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.904865+0000", 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000185532 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.905025+0000", 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016009600000000001 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:31.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:91162 6.d e186/168 6:bf991d31:::benchmark_data_smithi027_169445_object91161:head v 186'5760, mlcod=186'5758)", 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:31.905251+0000", 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0057663310000000004, 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058791379999999999, 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.905251+0000", 2024-08-29T15:16:31.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.905251+0000", 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.905252+0000", 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3600000000000002e-07 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.907919+0000", 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026666960000000001 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.907929+0000", 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.078e-05 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.907937+0000", 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7530000000000001e-06 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.909611+0000", 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0016741060000000001 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.909631+0000", 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9470000000000002e-05 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.958 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:91170 6.9 6:90c5c971:::benchmark_data_smithi027_169445_object91169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:31.908253+0000", 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.002764112, 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002920125, 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 91170 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:31.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908253+0000", 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908253+0000", 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908255+0000", 2024-08-29T15:16:31.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5960000000000001e-06 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908276+0000", 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.109e-05 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908286+0000", 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0616e-05 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:31.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908293+0000", 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.511e-06 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908430+0000", 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136847 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908568+0000", 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013831600000000001 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.908717+0000", 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014880699999999999 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:91156 6.9 6:96d4c889:::benchmark_data_smithi027_169445_object91155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e186)", 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:31.900097+0000", 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.010919948000000001, 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011113962999999999, 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:31.963 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 91156 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900097+0000", 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900097+0000", 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900099+0000", 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3120000000000001e-06 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900121+0000", 2024-08-29T15:16:31.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2759999999999999e-05 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900141+0000", 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9765000000000001e-05 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.900282+0000", 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014072600000000001 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.901403+0000", 2024-08-29T15:16:31.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001121085 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.901526+0000", 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000123025 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:16:31.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.901654+0000", 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012791199999999999 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:31.907947+0000", 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006293435 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 4 2024-08-29T15:16:31.969 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:16:31.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:31.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:31.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:32.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:32.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:32.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:32.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:32.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:32.252 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:32.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:32.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:31 smithi171 ceph-mon[90031]: pgmap v548: 89 pgs: 2 unknown, 2 peering, 8 active+undersized, 3 stale+active+clean, 11 active+undersized+degraded, 63 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 43954/259863 objects degraded (16.914%) 2024-08-29T15:16:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:31 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:16:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:31 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 43954/259863 objects degraded (16.914%), 11 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:32.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:31 smithi171 ceph-mon[90031]: osdmap e185: 8 total, 7 up, 7 in 2024-08-29T15:16:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[93921]: pgmap v548: 89 pgs: 2 unknown, 2 peering, 8 active+undersized, 3 stale+active+clean, 11 active+undersized+degraded, 63 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 43954/259863 objects degraded (16.914%) 2024-08-29T15:16:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:16:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 43954/259863 objects degraded (16.914%), 11 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[93921]: osdmap e185: 8 total, 7 up, 7 in 2024-08-29T15:16:32.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[98970]: pgmap v548: 89 pgs: 2 unknown, 2 peering, 8 active+undersized, 3 stale+active+clean, 11 active+undersized+degraded, 63 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 43954/259863 objects degraded (16.914%) 2024-08-29T15:16:32.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:16:32.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 43954/259863 objects degraded (16.914%), 11 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:32.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:31 smithi027 ceph-mon[98970]: osdmap e185: 8 total, 7 up, 7 in 2024-08-29T15:16:32.320 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:31 smithi027 podman[175879]: 2024-08-29 15:16:31.848998943 +0000 UTC m=+3.469367918 container remove 6dd209a7d75d1065ccec8bfd1dadd6e2037a030977b6dc803cacb47cec02ee52 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-29T15:16:32.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:32.392 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:32.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:32.493 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:32.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:32.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:32.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:32.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 61 16 92220 92204 94.4359 90.875 0.00435619 0.0105834 2024-08-29T15:16:32.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:32.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:32.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:32.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:32.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:32.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:32.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:32.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:16:32.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:32.933 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:32.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:32.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:32 smithi027 ceph-mon[98970]: osdmap e186: 8 total, 7 up, 7 in 2024-08-29T15:16:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:32 smithi027 ceph-mon[93921]: osdmap e186: 8 total, 7 up, 7 in 2024-08-29T15:16:33.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:33.141 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:33.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:33.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:33.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:16:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:32 smithi171 ceph-mon[90031]: osdmap e186: 8 total, 7 up, 7 in 2024-08-29T15:16:33.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:33.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:33.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:33.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:33.406 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:33.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:33.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:33.551 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:33.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:33.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:33.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 62 16 93547 93531 94.2502 82.9375 0.0136274 0.0106049 2024-08-29T15:16:33.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:33.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:33.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:33.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:33.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:33.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:33.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:33.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:33.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:33.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:33 smithi027 ceph-mon[93921]: pgmap v551: 89 pgs: 2 unknown, 2 peering, 15 active+undersized, 13 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 55316/266454 objects degraded (20.760%) 2024-08-29T15:16:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:33 smithi027 ceph-mon[98970]: pgmap v551: 89 pgs: 2 unknown, 2 peering, 15 active+undersized, 13 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 55316/266454 objects degraded (20.760%) 2024-08-29T15:16:34.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:34.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:34.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:34.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:34.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:34.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:34.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:34.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:33 smithi171 ceph-mon[90031]: pgmap v551: 89 pgs: 2 unknown, 2 peering, 15 active+undersized, 13 active+undersized+degraded, 57 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s; 55316/266454 objects degraded (20.760%) 2024-08-29T15:16:34.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:34.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:34.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:34.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:34.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:34.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:34.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:34.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 63 16 95016 95000 94.2114 91.8125 0.00661792 0.0106094 2024-08-29T15:16:34.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:34.645 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:34.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:34.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:34.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:34.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:34.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:34.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:34.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:34.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:34.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:35.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:35.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:35.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:35.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:35.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:35.167 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:35.204 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:35.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:35.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:35.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:35.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:35.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:35.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:35.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:35.495 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:35.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:35.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:35.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:35 smithi027 podman[176607]: 2024-08-29 15:16:35.354414789 +0000 UTC m=+0.147763307 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:16:35.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:35.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 64 16 96377 96361 94.0684 85.0625 0.0668704 0.0106252 2024-08-29T15:16:35.631 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:35.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:35.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:35.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:35.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:35.864 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:35 smithi027 podman[176607]: 2024-08-29 15:16:35.589816331 +0000 UTC m=+0.383164848 container create 1959461222fd6300ea906a06eccf5e591c73b2062064203a4cfffa43b0d85ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-08-29T15:16:35.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:35.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:35.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:36.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:36.059 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:36.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:36.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:36.179 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:35 smithi027 ceph-mon[98970]: pgmap v552: 89 pgs: 3 active+clean+remapped, 2 unknown, 2 peering, 15 active+undersized, 13 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 56203/269115 objects degraded (20.884%); 1/269115 objects misplaced (0.000%) 2024-08-29T15:16:36.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:35 smithi027 ceph-mon[93921]: pgmap v552: 89 pgs: 3 active+clean+remapped, 2 unknown, 2 peering, 15 active+undersized, 13 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 56203/269115 objects degraded (20.884%); 1/269115 objects misplaced (0.000%) 2024-08-29T15:16:36.202 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:36.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:36.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:35 smithi171 ceph-mon[90031]: pgmap v552: 89 pgs: 3 active+clean+remapped, 2 unknown, 2 peering, 15 active+undersized, 13 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 56203/269115 objects degraded (20.884%); 1/269115 objects misplaced (0.000%) 2024-08-29T15:16:36.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:36.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:36.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:36.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:36.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:36.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:36.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:36.573 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:36.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 65 16 97718 97702 93.9105 83.8125 0.01147 0.0106433 2024-08-29T15:16:36.608 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:36.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:36.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:36.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:36.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:36.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:36.841 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:36.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:36.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:36.892 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:16:36.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:37.003 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:37.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/174256373' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:16:37.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:36 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:16:37.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:36 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 56203/269115 objects degraded (20.884%), 13 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:37.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/174256373' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:16:37.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:36 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:16:37.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:36 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 56203/269115 objects degraded (20.884%), 13 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:37.105 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:37.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:37.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/174256373' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:16:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:36 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:16:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:36 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 56203/269115 objects degraded (20.884%), 13 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:37.280 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:37.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:37.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:37.411 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:37.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:37.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:37.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:37.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 66 16 99165 99149 93.8578 90.4375 0.00576511 0.0106491 2024-08-29T15:16:37.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:37.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:37.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:37.824 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:37.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:37.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:37.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:37.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:37.964 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[93921]: pgmap v553: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 59358/282486 objects degraded (21.013%); 11/282486 objects misplaced (0.004%); 720 B/s, 1 keys/s, 0 objects/s recovering 2024-08-29T15:16:37.964 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:16:37.964 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:37.964 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:16:37.964 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[93921]: osdmap e187: 8 total, 7 up, 7 in 2024-08-29T15:16:37.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[98970]: pgmap v553: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 59358/282486 objects degraded (21.013%); 11/282486 objects misplaced (0.004%); 720 B/s, 1 keys/s, 0 objects/s recovering 2024-08-29T15:16:37.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:16:37.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:37.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:16:37.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:37 smithi027 ceph-mon[98970]: osdmap e187: 8 total, 7 up, 7 in 2024-08-29T15:16:37.965 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:37 smithi027 podman[176607]: 2024-08-29 15:16:37.637126786 +0000 UTC m=+2.430475320 container init 1959461222fd6300ea906a06eccf5e591c73b2062064203a4cfffa43b0d85ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:16:37.965 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:37 smithi027 podman[176607]: 2024-08-29 15:16:37.64659002 +0000 UTC m=+2.439938546 container start 1959461222fd6300ea906a06eccf5e591c73b2062064203a4cfffa43b0d85ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-29T15:16:37.966 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:37 smithi027 podman[176607]: 2024-08-29 15:16:37.752110966 +0000 UTC m=+2.545459492 container attach 1959461222fd6300ea906a06eccf5e591c73b2062064203a4cfffa43b0d85ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:16:38.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:38.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:38.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:38.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:38.229 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:38.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:37 smithi171 ceph-mon[90031]: pgmap v553: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 59358/282486 objects degraded (21.013%); 11/282486 objects misplaced (0.004%); 720 B/s, 1 keys/s, 0 objects/s recovering 2024-08-29T15:16:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:37 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:16:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:37 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:37 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:16:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:37 smithi171 ceph-mon[90031]: osdmap e187: 8 total, 7 up, 7 in 2024-08-29T15:16:38.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:38.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:38.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:38.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:38.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:38.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:38.558 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:38 smithi027 podman[176607]: 2024-08-29 15:16:38.30373644 +0000 UTC m=+3.097084973 container died 1959461222fd6300ea906a06eccf5e591c73b2062064203a4cfffa43b0d85ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=) 2024-08-29T15:16:38.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:38.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 67 16 100528 100512 93.7283 85.1875 0.017566 0.0106636 2024-08-29T15:16:38.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:38.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:38.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:38.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:38.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:38.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:38.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:38.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:38.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:38.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:38.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:39.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:39.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:39.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:39.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:39.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:39.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:39.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:39.183 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:16:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:16:39] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-08-29T15:16:39.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:39.290 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:39.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:39.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:39.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:39.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:39.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:39.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:39.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:39.599 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 68 16 101834 101818 93.5503 81.625 0.0157571 0.0106842 2024-08-29T15:16:39.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:39.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:39.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:39.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:39.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:39.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:39.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:39.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:40.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:40.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:40.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service' 2024-08-29T15:16:40.134 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:40.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:40.163 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:39 smithi027 ceph-mon[98970]: pgmap v555: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 59765/287655 objects degraded (20.777%); 11/287655 objects misplaced (0.004%); 1.2 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-29T15:16:40.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:39 smithi027 ceph-mon[93921]: pgmap v555: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 59765/287655 objects degraded (20.777%); 11/287655 objects misplaced (0.004%); 1.2 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-29T15:16:40.164 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:40 smithi027 podman[176854]: 2024-08-29 15:16:40.096883616 +0000 UTC m=+2.115999818 container remove 1959461222fd6300ea906a06eccf5e591c73b2062064203a4cfffa43b0d85ed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-29T15:16:40.164 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:40 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service: Deactivated successfully. 2024-08-29T15:16:40.164 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:40 smithi027 systemd[1]: Stopped Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:16:40.164 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:16:40 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service: Consumed 9min 12.282s CPU time. 2024-08-29T15:16:40.230 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:16:40.230 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-29T15:16:40.236 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:40.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:39 smithi171 ceph-mon[90031]: pgmap v555: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 59765/287655 objects degraded (20.777%); 11/287655 objects misplaced (0.004%); 1.2 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-29T15:16:40.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:40.392 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:40.407 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:16:40.407 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:16:40.445 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:16:40.446 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:16:40.450 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:16:40.488 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:16:40.488 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:16:40.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:40.522 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:16:40.522 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:16:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:40.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:40.557 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:16:40.557 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:16:40.593 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:16:40.593 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:16:40.600 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 69 16 103012 102996 93.2615 73.625 0.00963273 0.0107173 2024-08-29T15:16:40.626 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:16:40.627 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:16:40.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:40.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:40.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:40.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:40.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:40.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:40.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:41.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:41.083 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:41.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:41.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:41.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:41.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:41.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:41.219 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3112420396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:41.220 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3112420396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:41.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:16:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3112420396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:41.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:41.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:41.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:41.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:41.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:41.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:41.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:41.600 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 70 16 104192 104176 92.9827 73.75 0.00578629 0.0107493 2024-08-29T15:16:41.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:41.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:41.715 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:41.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:41.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:41.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:41.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:41.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:41.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:42.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:42.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:42.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:42.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:42.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:41 smithi171 ceph-mon[90031]: pgmap v556: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 63204/300459 objects degraded (21.036%); 11/300459 objects misplaced (0.004%); 1.1 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-29T15:16:42.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:41 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 63204/300459 objects degraded (21.036%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:42.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:41 smithi027 ceph-mon[98970]: pgmap v556: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 63204/300459 objects degraded (21.036%); 11/300459 objects misplaced (0.004%); 1.1 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-29T15:16:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:41 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 63204/300459 objects degraded (21.036%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:41 smithi027 ceph-mon[93921]: pgmap v556: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 63204/300459 objects degraded (21.036%); 11/300459 objects misplaced (0.004%); 1.1 KiB/s, 1 keys/s, 1 objects/s recovering 2024-08-29T15:16:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:41 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 63204/300459 objects degraded (21.036%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:42.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:42.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:42.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:42.547 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:16:42.547 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:16:42.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:42.600 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 71 16 105515 105499 92.8377 82.6875 0.0052223 0.0107667 2024-08-29T15:16:42.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:42.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.702 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:42.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:42.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:42.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:42.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:42.949 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:16:42.949 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:16:42.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:42.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:42.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:43.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:43.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:43.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:43.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:43.225 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:43.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:43.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:43.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:43.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:43.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:43.492 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:43.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:43.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:43.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:43.600 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 72 16 106856 106840 92.7115 83.8125 0.00614846 0.0107809 2024-08-29T15:16:43.641 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:43.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:43.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:43.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:16:43.744 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:43.771 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:43.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:43.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:43.863 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:43.873 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:43.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:43.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:43.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:44.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:43 smithi027 ceph-mon[98970]: pgmap v557: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s; 64065/307872 objects degraded (20.809%); 11/307872 objects misplaced (0.004%); 954 B/s, 0 keys/s, 0 objects/s recovering 2024-08-29T15:16:44.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:44.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:44.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:43 smithi027 ceph-mon[93921]: pgmap v557: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s; 64065/307872 objects degraded (20.809%); 11/307872 objects misplaced (0.004%); 954 B/s, 0 keys/s, 0 objects/s recovering 2024-08-29T15:16:44.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:44.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:44.004 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:44.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:44.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:44.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:44.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:43 smithi171 ceph-mon[90031]: pgmap v557: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 86 MiB/s wr, 1.38k op/s; 64065/307872 objects degraded (20.809%); 11/307872 objects misplaced (0.004%); 954 B/s, 0 keys/s, 0 objects/s recovering 2024-08-29T15:16:44.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:16:44.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:44.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:44.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:44.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:44.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:44.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:16:44.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:44.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:44.601 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 73 16 108074 108058 92.4832 76.125 0.00971627 0.0108074 2024-08-29T15:16:44.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:44.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:44.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:44.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:44.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:44.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:44.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:44.993 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:44.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:45.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:45.129 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:45.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:45.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:45.231 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5] out_osds: [6] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2024-08-29T15:16:45.231 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:16:45.231 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-08-29T15:16:45.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd in 6 2024-08-29T15:16:45.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:45.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:45.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:45.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:45.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:45.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:45.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:45.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:45.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:45.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:45.601 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 74 16 109354 109338 92.3145 80 0.00696891 0.0108275 2024-08-29T15:16:45.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:45.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:45.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:45.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:45.789 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:45.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:45.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:45.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:45.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:45.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:46.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:46.053 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:46.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:46.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:46.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:46.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:45 smithi171 ceph-mon[90031]: pgmap v558: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s; 64836/310185 objects degraded (20.902%); 11/310185 objects misplaced (0.004%); 954 B/s, 0 keys/s, 0 objects/s recovering 2024-08-29T15:16:46.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:45 smithi027 ceph-mon[93921]: pgmap v558: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s; 64836/310185 objects degraded (20.902%); 11/310185 objects misplaced (0.004%); 954 B/s, 0 keys/s, 0 objects/s recovering 2024-08-29T15:16:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:45 smithi027 ceph-mon[98970]: pgmap v558: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s; 64836/310185 objects degraded (20.902%); 11/310185 objects misplaced (0.004%); 954 B/s, 0 keys/s, 0 objects/s recovering 2024-08-29T15:16:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:46.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:46.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:46.454 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:46.556 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:46.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:46.601 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 75 16 110703 110687 92.2077 84.3125 0.00697158 0.0108398 2024-08-29T15:16:46.692 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:46.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:46.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:46.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:46.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:46.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:46.991 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:46.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:46.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:47.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:47.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:47.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:47.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:47.261 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:47.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:47.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:46 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 64836/310185 objects degraded (20.902%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:47.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:46 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 64836/310185 objects degraded (20.902%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:47.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:46 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 64836/310185 objects degraded (20.902%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:47.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:47.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:47.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:47.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:47.527 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:47.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:47.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 76 16 112002 111986 92.0626 81.1875 0.0181682 0.0108568 2024-08-29T15:16:47.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:47.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:47.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:47.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:47.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:47.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:47.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:47.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:47.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:48.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:48.054 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:48.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:48.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:48.192 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:47 smithi027 ceph-mon[93921]: pgmap v559: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 67540/321828 objects degraded (20.986%); 11/321828 objects misplaced (0.003%) 2024-08-29T15:16:48.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:47 smithi027 ceph-mon[98970]: pgmap v559: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 67540/321828 objects degraded (20.986%); 11/321828 objects misplaced (0.003%) 2024-08-29T15:16:48.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:48.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:48.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:47 smithi171 ceph-mon[90031]: pgmap v559: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 67540/321828 objects degraded (20.986%); 11/321828 objects misplaced (0.003%) 2024-08-29T15:16:48.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:48.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:48.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:48.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:48.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:48.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:48.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 77 16 113203 113187 91.8418 75.0625 0.0134749 0.0108834 2024-08-29T15:16:48.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:48.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:48.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:48.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:48.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:48.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:48.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:48.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:48.924 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:48.924 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:48.964 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:48.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:49.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:49.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:16:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:16:49] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.51.0" 2024-08-29T15:16:49.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:49.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:49.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:49.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:49.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:49.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:49.347 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:49.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:49.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:49.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:49.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:49.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:49.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:49.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 78 16 114409 114393 91.6306 75.375 0.015415 0.010908 2024-08-29T15:16:49.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:49.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:49.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:49.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:49.780 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:49.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:49.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:49.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:49.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:49.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:49.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:49.954 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:16:50.023 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:50.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:50.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:50.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:50.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:50.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:49 smithi171 ceph-mon[90031]: pgmap v560: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 67959/326670 objects degraded (20.804%); 11/326670 objects misplaced (0.003%) 2024-08-29T15:16:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3802248' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:49 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:50.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:49 smithi027 ceph-mon[98970]: pgmap v560: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 67959/326670 objects degraded (20.804%); 11/326670 objects misplaced (0.003%) 2024-08-29T15:16:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:49 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3802248' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:49 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:49 smithi027 ceph-mon[93921]: pgmap v560: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 67959/326670 objects degraded (20.804%); 11/326670 objects misplaced (0.003%) 2024-08-29T15:16:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3802248' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:49 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:16:50.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:50.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:50.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:50.475 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:50.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:50.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:50.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:50.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 79 16 115661 115645 91.4612 78.25 0.00524077 0.0109276 2024-08-29T15:16:50.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:50.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:50.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:50.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:50.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:50.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:50.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:50.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:50.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:50.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:51.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:51.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:51.146 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:51.183 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:50 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:51.183 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:50 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:51.183 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:50 smithi027 ceph-mon[93921]: osdmap e188: 8 total, 7 up, 7 in 2024-08-29T15:16:51.183 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:50 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:51.183 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:50 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:51.184 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:50 smithi027 ceph-mon[98970]: osdmap e188: 8 total, 7 up, 7 in 2024-08-29T15:16:51.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:50 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:16:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:50 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:16:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:50 smithi171 ceph-mon[90031]: osdmap e188: 8 total, 7 up, 7 in 2024-08-29T15:16:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.282 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:51.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:51.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:51.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:51.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:51.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:51.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:16:51.602316+0000 min lat: 0.00129485 max lat: 1.05023 avg lat: 0.0109484 2024-08-29T15:16:51.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:16:51.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 80 16 116912 116896 91.2953 78.1875 0.00976799 0.0109484 2024-08-29T15:16:51.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:51.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:51.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:51.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:51.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:51.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:51.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:51.974 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:51.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:51.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:51 smithi027 ceph-mon[93921]: pgmap v562: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s; 71203/338712 objects degraded (21.022%); 11/338712 objects misplaced (0.003%) 2024-08-29T15:16:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:51 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 71203/338712 objects degraded (21.022%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:51 smithi027 ceph-mon[98970]: pgmap v562: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s; 71203/338712 objects degraded (21.022%); 11/338712 objects misplaced (0.003%) 2024-08-29T15:16:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:51 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 71203/338712 objects degraded (21.022%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:52.075 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:52.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:52.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:52.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:52.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:52.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:52.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:51 smithi171 ceph-mon[90031]: pgmap v562: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s; 71203/338712 objects degraded (21.022%); 11/338712 objects misplaced (0.003%) 2024-08-29T15:16:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:51 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 71203/338712 objects degraded (21.022%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:52.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:52.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:52.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:52.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:52.506 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:52.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:52.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 81 16 118208 118192 91.1681 81 0.0229943 0.0109634 2024-08-29T15:16:52.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:52.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:52.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:52.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:52.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:52.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:52.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:52.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:52.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:52.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:52.968 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:16:52.968 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:16:53.012 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:16:53.012 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:16:53.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:53.013 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:16:53.047 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:16:53.048 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:16:53.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1332690390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:53.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1332690390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:53.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:53.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:53.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:53.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:16:53.086 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:16:53.121 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:16:53.121 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:16:53.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:53.158 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:16:53.158 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:16:53.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:53.195 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:16:53.195 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:16:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:53.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1332690390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:16:53.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:53.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:53.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:53.432 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:53.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:53.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:53.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:53.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:53.599 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:53.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:53.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:53.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 82 16 119366 119350 90.9389 72.375 0.00752959 0.0109916 2024-08-29T15:16:53.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:53.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:53.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:53.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:53.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:53.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:53.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:54.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:54.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:54.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:54.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:54.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:16:54.249 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:53 smithi027 ceph-mon[98970]: pgmap v563: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 71989/345801 objects degraded (20.818%); 11/345801 objects misplaced (0.003%) 2024-08-29T15:16:54.250 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:53 smithi027 ceph-mon[93921]: pgmap v563: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 71989/345801 objects degraded (20.818%); 11/345801 objects misplaced (0.003%) 2024-08-29T15:16:54.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:53 smithi171 ceph-mon[90031]: pgmap v563: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 71989/345801 objects degraded (20.818%); 11/345801 objects misplaced (0.003%) 2024-08-29T15:16:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:54.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:54.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:54.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:54.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:54.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:54.602 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 83 16 120579 120563 90.7566 75.8125 0.0207454 0.0110131 2024-08-29T15:16:54.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:54.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:54.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:54.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:16:54.834 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:16:54.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:54.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:54.987 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:54.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:55.059 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:16:55.059 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:16:55.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:55.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:55.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:55.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:55.260 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:55.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:16:55.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:55.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:55.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:55.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:55.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:55.539 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:55.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:55.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 84 16 121737 121721 90.5377 72.375 0.00542325 0.0110397 2024-08-29T15:16:55.641 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:55.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:55.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:55.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:55.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:55.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:55.811 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:16:55.811 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:16:55.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:16:55.811 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:16:55.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:23216 5.3 5:c08b9121:::benchmark_data_smithi027_135490_object23215:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:10:40.961706+0000", 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 374.84623294800002, 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.137816141, 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 23216 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.812 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961706+0000", 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961706+0000", 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961707+0000", 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.031e-06 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961728+0000", 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1568000000000001e-05 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961738+0000", 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5410000000000006e-06 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:40.961745+0000", 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3379999999999999e-06 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093391+0000", 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13164532300000001 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093512+0000", 2024-08-29T15:16:55.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120995 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.093632+0000", 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012078099999999999 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.096349+0000", 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002716825 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.096525+0000", 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017614099999999999 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.098729+0000", 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022036999999999998 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.099473+0000", 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00074389300000000003 2024-08-29T15:16:55.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:10:41.099522+0000", 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9005000000000003e-05 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62785 5.3 5:c3da9dee:::benchmark_data_smithi027_135490_object62784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 355.10987734700001, 2024-08-29T15:16:55.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.23727498499999999, 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62785 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698064+0000", 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2879999999999999e-06 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698106+0000", 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2039999999999997e-05 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698128+0000", 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2064000000000001e-05 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698136+0000", 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3790000000000006e-06 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.700871+0000", 2024-08-29T15:16:55.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002734972 2024-08-29T15:16:55.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.701013+0000", 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141607 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.701141+0000", 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012853499999999999 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.704979+0000", 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038374239999999999 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.716497+0000", 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011518188 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935036+0000", 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.218538912 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935095+0000", 2024-08-29T15:16:55.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8857000000000002e-05 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935336+0000", 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024171899999999999 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62814 5.3 5:c9d52f91:::benchmark_data_smithi027_135490_object62813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:16:55.825 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 355.081453952, 2024-08-29T15:16:55.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.210827087, 2024-08-29T15:16:55.826 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.826 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.826 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.826 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.826 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.826 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62814 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726486+0000", 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5e-06 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726507+0000", 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0245e-05 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726519+0000", 2024-08-29T15:16:55.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2074e-05 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726526+0000", 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7980000000000006e-06 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726593+0000", 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6962e-05 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726740+0000", 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014641200000000001 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726858+0000", 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011843 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.728600+0000", 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017419340000000001 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.728790+0000", 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018932899999999999 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937147+0000", 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20835720599999999 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937263+0000", 2024-08-29T15:16:55.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116499 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937312+0000", 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8698e-05 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62826 5.3 5:c75bdcb4:::benchmark_data_smithi027_135490_object62825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:16:55.832 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 355.05663653099998, 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.188340805, 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62826 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:16:55.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751303+0000", 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.201e-06 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751324+0000", 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0271e-05 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751356+0000", 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2428999999999997e-05 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751364+0000", 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9039999999999992e-06 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751427+0000", 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3304000000000002e-05 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751579+0000", 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015205000000000001 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751757+0000", 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017760700000000001 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.754471+0000", 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027138589999999999 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.756577+0000", 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021064460000000001 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939494+0000", 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18291691700000001 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939598+0000", 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010330700000000001 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939643+0000", 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5510000000000003e-05 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62843 5.3 5:c10518e0:::benchmark_data_smithi027_135490_object62842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 355.04566612000002, 2024-08-29T15:16:55.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17756435800000001, 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62843 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:16:55.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762274+0000", 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.116e-06 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762294+0000", 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9743000000000001e-05 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762308+0000", 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4959e-05 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762316+0000", 2024-08-29T15:16:55.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1829999999999997e-06 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762372+0000", 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6172999999999999e-05 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762497+0000", 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012541700000000001 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762631+0000", 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013369200000000001 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.765559+0000", 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029280959999999998 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.765658+0000", 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8919999999999995e-05 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939688+0000", 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17403020299999999 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939785+0000", 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6971000000000004e-05 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939837+0000", 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1885000000000002e-05 2024-08-29T15:16:55.844 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62883 5.3 5:c29effb9:::benchmark_data_smithi027_135490_object62882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 355.01919723200001, 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15126415900000001, 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.845 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62883 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788744+0000", 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5919999999999999e-06 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788770+0000", 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5687999999999999e-05 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788788+0000", 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8099000000000001e-05 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788798+0000", 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7659999999999993e-06 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788890+0000", 2024-08-29T15:16:55.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2724000000000002e-05 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.789064+0000", 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000174048 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.789215+0000", 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015093600000000001 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.791500+0000", 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00228502 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.791885+0000", 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038479299999999997 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939879+0000", 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14799395700000001 2024-08-29T15:16:55.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939965+0000", 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6304999999999994e-05 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940006+0000", 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0231000000000002e-05 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62891 5.3 5:cec157a3:::benchmark_data_smithi027_135490_object62890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.797135+0000", 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 355.010803657, 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14327178700000001, 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62891 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.852 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797135+0000", 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797135+0000", 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797137+0000", 2024-08-29T15:16:55.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0619999999999999e-06 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797160+0000", 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2721000000000002e-05 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797170+0000", 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9359999999999994e-06 2024-08-29T15:16:55.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797175+0000", 2024-08-29T15:16:55.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3430000000000003e-06 2024-08-29T15:16:55.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797229+0000", 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.342e-05 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797374+0000", 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014522499999999999 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.797534+0000", 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000159679 2024-08-29T15:16:55.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.799392+0000", 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018581100000000001 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.799781+0000", 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038978999999999999 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940045+0000", 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14026403400000001 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940364+0000", 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031812799999999999 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940407+0000", 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3339e-05 2024-08-29T15:16:55.858 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68893 5.5 5:a47e561b:::benchmark_data_smithi027_135490_object68892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.906386107, 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14711858999999999, 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.859 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.860 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.860 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68893 2024-08-29T15:16:55.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.860 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901554+0000", 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.24e-06 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901576+0000", 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2155999999999999e-05 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901587+0000", 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1258e-05 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901596+0000", 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3930000000000008e-06 2024-08-29T15:16:55.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904408+0000", 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028119130000000001 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904518+0000", 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010994500000000001 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904644+0000", 2024-08-29T15:16:55.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012621299999999999 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907190+0000", 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025458640000000001 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048542+0000", 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.141352177 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048577+0000", 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4686e-05 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048622+0000", 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5382e-05 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048671+0000", 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9363000000000002e-05 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.865 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68895 5.3 5:c72cbd73:::benchmark_data_smithi027_135490_object68894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.90576112500003, 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.151393162, 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68895 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902179+0000", 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8700000000000004e-07 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902204+0000", 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5474999999999999e-05 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2199e-05 2024-08-29T15:16:55.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902224+0000", 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1289999999999996e-06 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904177+0000", 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00195218 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904303+0000", 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012648 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904431+0000", 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012815400000000001 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909174+0000", 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047423450000000002 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909243+0000", 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9497000000000002e-05 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053476+0000", 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.144233477 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053526+0000", 2024-08-29T15:16:55.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9477000000000001e-05 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053571+0000", 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4861999999999998e-05 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68897 5.5 5:adfd8055:::benchmark_data_smithi027_135490_object68896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.90565919199997, 2024-08-29T15:16:55.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.147127652, 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68897 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:16:55.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7599999999999997e-07 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902303+0000", 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2676000000000001e-05 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902313+0000", 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7529999999999996e-06 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902319+0000", 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6290000000000004e-06 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907215+0000", 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048957189999999998 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907355+0000", 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139775 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907500+0000", 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014554800000000001 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049212+0000", 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14171125700000001 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049280+0000", 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8319999999999996e-05 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049292+0000", 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1849e-05 2024-08-29T15:16:55.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049363+0000", 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1616e-05 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049407+0000", 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3934000000000003e-05 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.44573.0:68893 5.5 e117/111 ondisk, result = 0)", 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.908863+0000", 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.89907561199999, 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13968623199999999, 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908863+0000", 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908863+0000", 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908864+0000", 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0200000000000002e-07 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908867+0000", 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9239999999999999e-06 2024-08-29T15:16:55.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908876+0000", 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5230000000000001e-06 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908882+0000", 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9100000000000002e-06 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048526+0000", 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13964390500000001 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048535+0000", 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2369999999999998e-06 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048549+0000", 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4131000000000001e-05 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.882 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68907 5.3 5:cf461b57:::benchmark_data_smithi027_135490_object68906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.908921+0000", 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.899017958, 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.145667885, 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.883 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68907 2024-08-29T15:16:55.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.884 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908921+0000", 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908921+0000", 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908922+0000", 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.147e-06 2024-08-29T15:16:55.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908942+0000", 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0259e-05 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908951+0000", 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2099999999999999e-06 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.908957+0000", 2024-08-29T15:16:55.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.468e-06 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909276+0000", 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031944699999999999 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048670+0000", 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.139393394 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048861+0000", 2024-08-29T15:16:55.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000190982 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053583+0000", 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047221590000000001 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053644+0000", 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1029000000000001e-05 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054502+0000", 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00085849299999999999 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054552+0000", 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9160999999999999e-05 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.054589+0000", 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7135999999999998e-05 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.889 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:68906 5.e e117/111 5:73dcaf78:::benchmark_data_smithi027_135490_object68905:head v 117'4372, mlcod=117'4370)", 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.909133+0000", 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.89880550800001, 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13943607699999999, 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909133+0000", 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909133+0000", 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909134+0000", 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8400000000000004e-07 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909164+0000", 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9853999999999999e-05 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909172+0000", 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4999999999999999e-06 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909178+0000", 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9059999999999999e-06 2024-08-29T15:16:55.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909239+0000", 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1291999999999995e-05 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909267+0000", 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7735000000000001e-05 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048521+0000", 2024-08-29T15:16:55.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13925380900000001 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048569+0000", 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8396999999999998e-05 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:68896 5.b e117/111 5:d4a8571b:::benchmark_data_smithi027_135490_object68895:head v 117'4385, mlcod=117'4382)", 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.910870+0000", 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.89706904100001, 2024-08-29T15:16:55.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.142751935, 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910870+0000", 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910870+0000", 2024-08-29T15:16:55.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910871+0000", 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4099999999999997e-07 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.910895+0000", 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4634000000000002e-05 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048477+0000", 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.137582223 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048489+0000", 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1814000000000001e-05 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048682+0000", 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019234800000000001 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048706+0000", 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4893e-05 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053584+0000", 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048777220000000001 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053622+0000", 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7459999999999997e-05 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.898 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:68909 5.7 e117/111 5:eeb8103f:::benchmark_data_smithi027_135490_object68908:head v 117'4363, mlcod=117'4360)", 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.912355+0000", 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.89558380699998, 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.143279975, 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912355+0000", 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912355+0000", 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912356+0000", 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9800000000000003e-07 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.912422+0000", 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5881000000000002e-05 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048786+0000", 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.136363978 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048794+0000", 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8300000000000002e-06 2024-08-29T15:16:55.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048860+0000", 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6100999999999996e-05 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048881+0000", 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0682999999999998e-05 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055514+0000", 2024-08-29T15:16:55.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006632827 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055635+0000", 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120977 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239396 5.3 5:c28abb26:::benchmark_data_smithi027_135490_object105644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:16:55.903 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 170.06496284599999, 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4671054730000002, 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239396 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742978+0000", 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.674e-06 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742984+0000", 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7530000000000004e-06 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742995+0000", 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0967e-05 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.743005+0000", 2024-08-29T15:16:55.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4690000000000003e-06 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.751912+0000", 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008907491 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.751924+0000", 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1246999999999999e-05 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692438+0000", 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94051403300000003 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692450+0000", 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.269e-05 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.176340+0000", 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.483889434 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.176472+0000", 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013255499999999999 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.177116+0000", 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064412200000000001 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.201804+0000", 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024687298999999999 2024-08-29T15:16:55.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.201859+0000", 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5201000000000002e-05 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.206234+0000", 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043748629999999997 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.209955+0000", 2024-08-29T15:16:55.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037214819999999999 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.209993+0000", 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7898999999999997e-05 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210081+0000", 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8294000000000005e-05 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.911 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239400 5.3 5:c91c85c5:::benchmark_data_smithi027_135490_object105648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 170.04959238399999, 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4617042850000002, 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.912 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239400 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758348+0000", 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4130000000000001e-06 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758354+0000", 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0510000000000002e-06 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758382+0000", 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7864999999999998e-05 2024-08-29T15:16:55.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758391+0000", 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2390000000000008e-06 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758468+0000", 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7152999999999998e-05 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758481+0000", 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2927e-05 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692477+0000", 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93399591900000001 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692485+0000", 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6340000000000001e-06 2024-08-29T15:16:55.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210333+0000", 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5178479149999999 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210442+0000", 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108992 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:16:55.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210625+0000", 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018322499999999999 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219736+0000", 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0091110099999999992 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219805+0000", 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8861000000000004e-05 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219816+0000", 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1392999999999999e-05 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219858+0000", 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2416000000000002e-05 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219940+0000", 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1883000000000002e-05 2024-08-29T15:16:55.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220051+0000", 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110389 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239422 5.3 5:cc7358ac:::benchmark_data_smithi027_135490_object105670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 167.7881677, 2024-08-29T15:16:55.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21943114, 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239422 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019773+0000", 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2479999999999999e-06 2024-08-29T15:16:55.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019782+0000", 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0650000000000005e-06 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019802+0000", 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9653999999999999e-05 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019811+0000", 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4369999999999996e-06 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019893+0000", 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1891999999999995e-05 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019903+0000", 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0003999999999999e-05 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220071+0000", 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20016803999999999 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220151+0000", 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9785999999999998e-05 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220220+0000", 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8520000000000001e-05 2024-08-29T15:16:55.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228770+0000", 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085506139999999998 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228784+0000", 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3207000000000001e-05 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228816+0000", 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2806999999999999e-05 2024-08-29T15:16:55.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239075+0000", 2024-08-29T15:16:55.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010258751 2024-08-29T15:16:55.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:55.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239111+0000", 2024-08-29T15:16:55.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5917999999999997e-05 2024-08-29T15:16:55.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239202+0000", 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1197000000000001e-05 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239427 5.3 5:cac43ec6:::benchmark_data_smithi027_135490_object105675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 167.765213652, 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.197921239, 2024-08-29T15:16:55.929 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239427 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042726+0000", 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.215e-06 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042730+0000", 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9469999999999996e-06 2024-08-29T15:16:55.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042741+0000", 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0995999999999999e-05 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042748+0000", 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.652e-06 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042842+0000", 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4467000000000002e-05 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042852+0000", 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9080000000000007e-06 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.223936+0000", 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18108415 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.224043+0000", 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106591 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.224167+0000", 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012385599999999999 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228861+0000", 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046937619999999998 2024-08-29T15:16:55.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228899+0000", 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8325000000000003e-05 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239219+0000", 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010320424999999999 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240479+0000", 2024-08-29T15:16:55.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001259825 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240537+0000", 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7386000000000002e-05 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240646+0000", 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109734 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.936 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239430 5.3 5:c7a58ecd:::benchmark_data_smithi027_135490_object105678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 167.754846614, 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18776626399999999, 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:16:55.937 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239430 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053094+0000", 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4100000000000001e-06 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053100+0000", 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2979999999999997e-06 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053111+0000", 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0664e-05 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053117+0000", 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7479999999999996e-06 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053200+0000", 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2795999999999998e-05 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053210+0000", 2024-08-29T15:16:55.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0399e-05 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228935+0000", 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.175724134 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.229054+0000", 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119704 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:16:55.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.229207+0000", 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015269400000000001 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240172+0000", 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010965088 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240433+0000", 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026095699999999998 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240446+0000", 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3414000000000001e-05 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240699+0000", 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025234500000000001 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240748+0000", 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9118e-05 2024-08-29T15:16:55.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240858+0000", 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011049500000000001 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:16:55.944 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:16:55.989 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:55.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:56.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:56.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:56.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:56.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:56 smithi171 ceph-mon[90031]: pgmap v564: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 72724/348006 objects degraded (20.897%); 11/348006 objects misplaced (0.003%) 2024-08-29T15:16:56.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:56.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:56.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:56 smithi027 ceph-mon[98970]: pgmap v564: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 72724/348006 objects degraded (20.897%); 11/348006 objects misplaced (0.003%) 2024-08-29T15:16:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:56 smithi027 ceph-mon[93921]: pgmap v564: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 72724/348006 objects degraded (20.897%); 11/348006 objects misplaced (0.003%) 2024-08-29T15:16:56.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:56.419 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:56.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:56.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:56.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:56.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:56.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:56.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 85 16 122848 122832 90.2894 69.4375 0.013587 0.0110708 2024-08-29T15:16:56.670 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:56.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:56.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:56.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:56.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:56.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:16:56.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:56.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:56.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:56.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:56.988 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:56.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:56.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:57.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:57.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:57.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:57.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:57.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:57.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 72724/348006 objects degraded (20.897%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:57.256 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 72724/348006 objects degraded (20.897%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:57.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:57.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:57.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 72724/348006 objects degraded (20.897%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:16:57.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:16:57.295 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:16:57.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:57.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:57.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:57.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:57.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:57.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:57.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 86 16 123956 123940 90.0447 69.25 0.0220186 0.0110996 2024-08-29T15:16:57.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:57.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:57.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:57.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:57.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:57.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:57.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:57.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:57.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:57.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:57.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:58.023 INFO:teuthology.orchestra.run.smithi027.stderr:marked in osd.6. 2024-08-29T15:16:58.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:58.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:58 smithi027 ceph-mon[98970]: pgmap v565: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 75383/359559 objects degraded (20.965%); 11/359559 objects misplaced (0.003%) 2024-08-29T15:16:58.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2981380243' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-29T15:16:58.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:58 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-29T15:16:58.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:57 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:16:57.988+0000 7f3f0d183640 -1 mon.a@0(leader).osd e188 definitely_dead 0 2024-08-29T15:16:58.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:58 smithi027 ceph-mon[93921]: pgmap v565: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 75383/359559 objects degraded (20.965%); 11/359559 objects misplaced (0.003%) 2024-08-29T15:16:58.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2981380243' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-29T15:16:58.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:58 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-29T15:16:58.064 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:58.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:16:58.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:58.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:58.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:58.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:58.312 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:58.380 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:58 smithi171 ceph-mon[90031]: pgmap v565: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 75383/359559 objects degraded (20.965%); 11/359559 objects misplaced (0.003%) 2024-08-29T15:16:58.380 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2981380243' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-29T15:16:58.380 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:58 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-29T15:16:58.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:58.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:58.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:58.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:58.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:58.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 87 16 124461 124445 89.3726 31.5625 0.00375423 0.0111153 2024-08-29T15:16:58.607 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:16:58.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:58.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:16:58.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:58.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:58.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:58.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:58.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:16:58.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:58.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:58.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:16:58.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:59.001 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:59.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:59.103 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:16:59.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:16:59.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:59.249 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:59.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:59 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-29T15:16:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:59 smithi171 ceph-mon[90031]: osdmap e189: 8 total, 7 up, 8 in 2024-08-29T15:16:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:16:59 smithi171 ceph-mon[90031]: osdmap e190: 8 total, 7 up, 8 in 2024-08-29T15:16:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-29T15:16:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[93921]: osdmap e189: 8 total, 7 up, 8 in 2024-08-29T15:16:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[93921]: osdmap e190: 8 total, 7 up, 8 in 2024-08-29T15:16:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:16:59] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.51.0" 2024-08-29T15:16:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-29T15:16:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[98970]: osdmap e189: 8 total, 7 up, 8 in 2024-08-29T15:16:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:16:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:16:59 smithi027 ceph-mon[98970]: osdmap e190: 8 total, 7 up, 8 in 2024-08-29T15:16:59.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:16:59.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:59.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:16:59.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:59.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:59.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:59.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:59.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 88 16 124461 124445 88.3572 0 - 0.0111153 2024-08-29T15:16:59.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:59.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:16:59.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:59.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:16:59.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:16:59.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:16:59.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:16:59.971 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:16:59.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:16:59.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:16:59.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:00.096 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:00.126 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:00.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:00.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:00.250 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:00.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:00.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[93921]: pgmap v567: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 62 MiB/s wr, 985 op/s; 75746/363939 objects degraded (20.813%); 11/363939 objects misplaced (0.003%) 2024-08-29T15:17:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[93921]: osdmap e191: 8 total, 7 up, 8 in 2024-08-29T15:17:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[98970]: pgmap v567: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 62 MiB/s wr, 985 op/s; 75746/363939 objects degraded (20.813%); 11/363939 objects misplaced (0.003%) 2024-08-29T15:17:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:00.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:00 smithi027 ceph-mon[98970]: osdmap e191: 8 total, 7 up, 8 in 2024-08-29T15:17:00.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:00.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:00.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:00.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:00.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:00.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:00.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:00 smithi171 ceph-mon[90031]: pgmap v567: 89 pgs: 9 active+clean+remapped, 15 active+undersized, 12 active+undersized+degraded, 53 active+clean; 7.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 62 MiB/s wr, 985 op/s; 75746/363939 objects degraded (20.813%); 11/363939 objects misplaced (0.003%) 2024-08-29T15:17:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:00 smithi171 ceph-mon[90031]: osdmap e191: 8 total, 7 up, 8 in 2024-08-29T15:17:00.542 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-08-29T15:17:00.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 89 16 124703 124687 87.5345 7.5625 0.00574531 0.0114163 2024-08-29T15:17:00.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:00.649 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:00.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:00.751 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:00.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:00.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:00.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:00.879 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:00.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:00.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:00.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:00.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:01.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:01.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:01.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:01.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:01.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:01 smithi027 ceph-mon[98970]: 6.1 starting backfill to osd.6 from (0'0,0'0] MAX to 188'7780 2024-08-29T15:17:01.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:01 smithi027 ceph-mon[98970]: 6.c starting backfill to osd.6 from (0'0,0'0] MAX to 188'7766 2024-08-29T15:17:01.272 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:01 smithi027 ceph-mon[93921]: 6.1 starting backfill to osd.6 from (0'0,0'0] MAX to 188'7780 2024-08-29T15:17:01.272 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:01 smithi027 ceph-mon[93921]: 6.c starting backfill to osd.6 from (0'0,0'0] MAX to 188'7766 2024-08-29T15:17:01.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:01.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:01.369 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:01 smithi171 ceph-mon[90031]: 6.1 starting backfill to osd.6 from (0'0,0'0] MAX to 188'7780 2024-08-29T15:17:01.369 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:01 smithi171 ceph-mon[90031]: 6.c starting backfill to osd.6 from (0'0,0'0] MAX to 188'7766 2024-08-29T15:17:01.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:01.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:01.584 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:01.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 90 3 125424 125421 87.0717 45.875 0.0505816 0.0114778 2024-08-29T15:17:01.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:01.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:01.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:01.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:01.836 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:01.858 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:01.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:01.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:01.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:01.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:02.051 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[93921]: pgmap v570: 89 pgs: 2 peering, 10 remapped+peering, 3 active+clean+remapped, 13 active+undersized, 9 active+undersized+degraded, 52 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 54339/371385 objects degraded (14.631%); 2/371385 objects misplaced (0.001%); 3 B/s, 0 objects/s recovering 2024-08-29T15:17:02.051 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:17:02.051 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[93921]: osdmap e192: 8 total, 7 up, 8 in 2024-08-29T15:17:02.051 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 54339/371385 objects degraded (14.631%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:02.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:02.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:02.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:02.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:02.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:02.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[98970]: pgmap v570: 89 pgs: 2 peering, 10 remapped+peering, 3 active+clean+remapped, 13 active+undersized, 9 active+undersized+degraded, 52 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 54339/371385 objects degraded (14.631%); 2/371385 objects misplaced (0.001%); 3 B/s, 0 objects/s recovering 2024-08-29T15:17:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:17:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[98970]: osdmap e192: 8 total, 7 up, 8 in 2024-08-29T15:17:02.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 54339/371385 objects degraded (14.631%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:02.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:02.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:02.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:02.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:02.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:02.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:02.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:02.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:02 smithi171 ceph-mon[90031]: pgmap v570: 89 pgs: 2 peering, 10 remapped+peering, 3 active+clean+remapped, 13 active+undersized, 9 active+undersized+degraded, 52 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 54339/371385 objects degraded (14.631%); 2/371385 objects misplaced (0.001%); 3 B/s, 0 objects/s recovering 2024-08-29T15:17:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:02 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:17:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:02 smithi171 ceph-mon[90031]: osdmap e192: 8 total, 7 up, 8 in 2024-08-29T15:17:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 54339/371385 objects degraded (14.631%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:02.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:02.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total time run: 90.0389 2024-08-29T15:17:02.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total writes made: 125424 2024-08-29T15:17:02.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Write size: 65536 2024-08-29T15:17:02.603 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object size: 65536 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Bandwidth (MB/sec): 87.0624 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Bandwidth: 32.2527 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max bandwidth (MB/sec): 190.688 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min bandwidth (MB/sec): 0 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average IOPS: 1392 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev IOPS: 516.043 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max IOPS: 3051 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min IOPS: 0 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average Latency(s): 0.0114788 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Latency(s): 0.0288931 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max latency(s): 2.31553 2024-08-29T15:17:02.604 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min latency(s): 0.00129485 2024-08-29T15:17:02.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:02.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:02.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:02.608 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Cleaning up (deleting benchmark objects) 2024-08-29T15:17:02.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:02.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:02.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:02.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:02.841 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:02.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:02.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:02.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:02.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:03.014 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:03.116 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:03.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:03.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:03.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:03.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:03.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:03.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:03 smithi027 ceph-mon[98970]: osdmap e193: 8 total, 7 up, 8 in 2024-08-29T15:17:03.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:03 smithi027 ceph-mon[93921]: osdmap e193: 8 total, 7 up, 8 in 2024-08-29T15:17:03.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:03.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:03.452 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:03.502 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:17:03.502 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:17:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:03 smithi171 ceph-mon[90031]: osdmap e193: 8 total, 7 up, 8 in 2024-08-29T15:17:03.540 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:17:03.540 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:17:03.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:03.543 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:17:03.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:03.578 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:17:03.579 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:17:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:03.615 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:17:03.615 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:17:03.648 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:17:03.649 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:17:03.679 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:17:03.679 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:17:03.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:03.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:03.709 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:17:03.709 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:17:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:03.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:03.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:03.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:03.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:03.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:03.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:04.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:04.082 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:17:04.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:04.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:04.215 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:04.252 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:04.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:04.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[93921]: pgmap v573: 89 pgs: 1 active+undersized+remapped, 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 remapped+peering, 5 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 50 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 54609/374277 objects degraded (14.591%); 15628/374277 objects misplaced (4.176%); 864 KiB/s, 16 objects/s recovering 2024-08-29T15:17:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2210160036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/619241764' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[98970]: pgmap v573: 89 pgs: 1 active+undersized+remapped, 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 remapped+peering, 5 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 50 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 54609/374277 objects degraded (14.591%); 15628/374277 objects misplaced (4.176%); 864 KiB/s, 16 objects/s recovering 2024-08-29T15:17:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2210160036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/619241764' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:04 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:04.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:04.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:04.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:04.483 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:04.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:04 smithi171 ceph-mon[90031]: pgmap v573: 89 pgs: 1 active+undersized+remapped, 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 remapped+peering, 5 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 50 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 54609/374277 objects degraded (14.591%); 15628/374277 objects misplaced (4.176%); 864 KiB/s, 16 objects/s recovering 2024-08-29T15:17:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2210160036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:04.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/619241764' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:04.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:04 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:04.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:04.619 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:04.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:04.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:04.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:04.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:04.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:04.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:04.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:04.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:04.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:04.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:05.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:05.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:05.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:05.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:05.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:05.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:05.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:05.297 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:05 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:17:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:05 smithi027 ceph-mon[98970]: osdmap e194: 8 total, 7 up, 8 in 2024-08-29T15:17:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:05 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:17:05.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:05 smithi027 ceph-mon[93921]: osdmap e194: 8 total, 7 up, 8 in 2024-08-29T15:17:05.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:05.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:05.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:05.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:05.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:05 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:17:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:05 smithi171 ceph-mon[90031]: osdmap e194: 8 total, 7 up, 8 in 2024-08-29T15:17:05.543 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2024-08-29T15:17:05.543 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:17:05.543 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-29T15:17:05.543 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.153813 2024-08-29T15:17:05.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 2 0.15381322743165093 2024-08-29T15:17:05.572 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:05.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:05.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:05.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:05.749 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:05.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:05.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:05.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:05.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:05.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:06.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:06.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:06.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:06.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:06.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:06.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:06 smithi027 ceph-mon[98970]: pgmap v575: 89 pgs: 1 active+undersized+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 7 remapped+peering, 8 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 51 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 15 MiB/s wr, 282 op/s; 54598/374244 objects degraded (14.589%); 15629/374244 objects misplaced (4.176%); 775 KiB/s, 17 objects/s recovering 2024-08-29T15:17:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:06 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:17:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:06 smithi027 ceph-mon[93921]: pgmap v575: 89 pgs: 1 active+undersized+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 7 remapped+peering, 8 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 51 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 15 MiB/s wr, 282 op/s; 54598/374244 objects degraded (14.589%); 15629/374244 objects misplaced (4.176%); 775 KiB/s, 17 objects/s recovering 2024-08-29T15:17:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:06 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:17:06.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:17:06.415 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:17:06.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:06.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:06 smithi171 ceph-mon[90031]: pgmap v575: 89 pgs: 1 active+undersized+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 7 remapped+peering, 8 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 51 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 15 MiB/s wr, 282 op/s; 54598/374244 objects degraded (14.589%); 15629/374244 objects misplaced (4.176%); 775 KiB/s, 17 objects/s recovering 2024-08-29T15:17:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:06 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:17:06.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:06.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:06.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:06.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:06.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:06.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:06.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:06.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:06.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:07.086 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:17:07.087 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:17:07.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:07.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:07.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:07.225 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:07.269 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:07.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:07.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 54598/374244 objects degraded (14.589%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 54598/374244 objects degraded (14.589%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:07.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:07.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:07.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:07.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 54598/374244 objects degraded (14.589%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:07.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:07.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:07.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:07.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:07.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:07.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:07.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:17:07.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:07.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:07.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:07.854 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:07.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:07.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:07.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:07.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:08.061 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:08.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:08.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:08.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:08.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:08.238 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:08.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:08.313 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:08 smithi027 ceph-mon[98970]: pgmap v576: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 17 MiB/s wr, 501 op/s; 77606/372306 objects degraded (20.845%); 23258/372306 objects misplaced (6.247%); 1.8 MiB/s, 32 objects/s recovering 2024-08-29T15:17:08.313 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:08 smithi027 ceph-mon[93921]: pgmap v576: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 17 MiB/s wr, 501 op/s; 77606/372306 objects degraded (20.845%); 23258/372306 objects misplaced (6.247%); 1.8 MiB/s, 32 objects/s recovering 2024-08-29T15:17:08.340 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:08.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:08.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:08.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:08.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:17:08.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:08.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:08.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:08.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:08 smithi171 ceph-mon[90031]: pgmap v576: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 17 MiB/s wr, 501 op/s; 77606/372306 objects degraded (20.845%); 23258/372306 objects misplaced (6.247%); 1.8 MiB/s, 32 objects/s recovering 2024-08-29T15:17:08.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:08.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:08.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:08.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:08.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:08.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:08.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:08.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:08.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:08.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:09.002 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:09.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:17:09.036 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:17:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:17:09] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.51.0" 2024-08-29T15:17:09.039 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:09.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:09.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:09.147 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:09.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:09.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:09.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:09.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:09.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:09.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:09.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:09.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:09.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:09.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.722 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:09.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:09.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:09.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:09.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:09.993 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:09.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:10.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:10.130 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:10.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:10.184 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:10.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:10.263 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:10.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:10.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:10.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:10 smithi027 ceph-mon[98970]: pgmap v577: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 137 B/s rd, 11 MiB/s wr, 463 op/s; 77405/369885 objects degraded (20.927%); 23052/369885 objects misplaced (6.232%); 1.0 MiB/s, 17 objects/s recovering 2024-08-29T15:17:10.398 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:10 smithi027 ceph-mon[93921]: pgmap v577: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 137 B/s rd, 11 MiB/s wr, 463 op/s; 77405/369885 objects degraded (20.927%); 23052/369885 objects misplaced (6.232%); 1.0 MiB/s, 17 objects/s recovering 2024-08-29T15:17:10.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:10.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:10.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:10 smithi171 ceph-mon[90031]: pgmap v577: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 137 B/s rd, 11 MiB/s wr, 463 op/s; 77405/369885 objects degraded (20.927%); 23052/369885 objects misplaced (6.232%); 1.0 MiB/s, 17 objects/s recovering 2024-08-29T15:17:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:10.539 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:10.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:10.640 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:10.680 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:10.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:10.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:10.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:10.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:10.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:10.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:10.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:10.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:10.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:10.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:11.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:11.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:11.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:11.098 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.2 primary-affinity to 0.153813 (8100802) 2024-08-29T15:17:11.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:11.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:11.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:11.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:11.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:11.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:11.435 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:11.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:11.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/65607715' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093} : dispatch 2024-08-29T15:17:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093} : dispatch 2024-08-29T15:17:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/65607715' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093} : dispatch 2024-08-29T15:17:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093} : dispatch 2024-08-29T15:17:11.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/65607715' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093} : dispatch 2024-08-29T15:17:11.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093} : dispatch 2024-08-29T15:17:11.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:11.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:11.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:11.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:11.708 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:11.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:11.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:11.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:11.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:11.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:11.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:12.014 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:12.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:12.116 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:12.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:12.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:12.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:12.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[93921]: pgmap v578: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 121 B/s rd, 5.1 MiB/s wr, 639 op/s; 75800/362286 objects degraded (20.923%); 23052/362286 objects misplaced (6.363%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093}]': finished 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[93921]: osdmap e195: 8 total, 7 up, 8 in 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 75800/362286 objects degraded (20.923%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[98970]: pgmap v578: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 121 B/s rd, 5.1 MiB/s wr, 639 op/s; 75800/362286 objects degraded (20.923%); 23052/362286 objects misplaced (6.363%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093}]': finished 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[98970]: osdmap e195: 8 total, 7 up, 8 in 2024-08-29T15:17:12.317 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 75800/362286 objects degraded (20.923%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:12.396 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:12.399 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:12 smithi171 ceph-mon[90031]: pgmap v578: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 121 B/s rd, 5.1 MiB/s wr, 639 op/s; 75800/362286 objects degraded (20.923%); 23052/362286 objects misplaced (6.363%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:17:12.399 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:12 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.15381322743165093}]': finished 2024-08-29T15:17:12.399 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:12 smithi171 ceph-mon[90031]: osdmap e195: 8 total, 7 up, 8 in 2024-08-29T15:17:12.400 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 75800/362286 objects degraded (20.923%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:12.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:12.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:12.575 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:12.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:12.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:12.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:12.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:12.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:12.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:12.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:12.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:12.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:13.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:13.134 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:13.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:13.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:13.236 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:13.267 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:13.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:13.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:13.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:13.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:13.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:13.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:13.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:13.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:13.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:13.583 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:13.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:13.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:13.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:13.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:13.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:13.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:13.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:13.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:13.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:13.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:14.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:14.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:14.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:14.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:14.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:14.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:14.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:14.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:14.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:14.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:14.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:14.503 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:14.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:14 smithi171 ceph-mon[90031]: pgmap v580: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 121 B/s rd, 4.1 MiB/s wr, 781 op/s; 75293/357846 objects degraded (21.041%); 22526/357846 objects misplaced (6.295%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:17:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:14 smithi027 ceph-mon[98970]: pgmap v580: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 121 B/s rd, 4.1 MiB/s wr, 781 op/s; 75293/357846 objects degraded (21.041%); 22526/357846 objects misplaced (6.295%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:17:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:14.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:14 smithi027 ceph-mon[93921]: pgmap v580: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 121 B/s rd, 4.1 MiB/s wr, 781 op/s; 75293/357846 objects degraded (21.041%); 22526/357846 objects misplaced (6.295%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:17:14.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:14.605 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:14.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:14.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:14.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:14.793 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:14.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:14.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:14.895 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:14.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:14.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:15.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:15.060 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:15.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:15.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:15.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:15.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:15.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:15.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:15.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:15.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:15.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:15.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:15.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:15.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:15.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:15.598 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:15.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:15.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:15.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:15.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:15.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:15.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:15.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:15.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:15.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:15.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:16.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:16.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:16.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:16.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:16.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:16.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:16.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:16.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:16 smithi027 ceph-mon[98970]: pgmap v581: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 3.4 MiB/s wr, 708 op/s; 74777/356298 objects degraded (20.987%); 22526/356298 objects misplaced (6.322%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:17:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:16 smithi027 ceph-mon[93921]: pgmap v581: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 3.4 MiB/s wr, 708 op/s; 74777/356298 objects degraded (20.987%); 22526/356298 objects misplaced (6.322%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:17:16.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:16.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:16.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:16.489 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:17:16.489 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:17:16.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:16.516 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:17:16.516 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:17:16.517 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:17:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:16 smithi171 ceph-mon[90031]: pgmap v581: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 3.4 MiB/s wr, 708 op/s; 74777/356298 objects degraded (20.987%); 22526/356298 objects misplaced (6.322%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:17:16.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:16.553 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:17:16.553 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:17:16.593 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:17:16.593 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:17:16.639 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:17:16.639 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:17:16.652 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:16.674 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:17:16.674 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:17:16.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:16.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:16.711 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:17:16.711 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:17:16.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:16.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:16.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:16.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:16.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:16.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:17.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:17.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:17.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:17.208 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:17.221 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:17:17.221 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:17:17.221 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:17:17.221 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:17:17.221 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:17:17.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:17.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:17.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:17.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:17.384 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:17.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:17.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:17.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/314522723' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 74777/356298 objects degraded (20.987%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:17.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:17.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/314522723' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 74777/356298 objects degraded (20.987%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:17.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/314522723' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:17.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 74777/356298 objects degraded (20.987%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:17.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:17.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:17.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:17.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:17.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:17.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:17.883 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:17:17.884 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:17:17.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:17.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:17.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:17.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:18.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:18.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:18.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:18.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:18.195 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:18.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:18 smithi027 ceph-mon[98970]: pgmap v582: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 73391/348684 objects degraded (21.048%); 22287/348684 objects misplaced (6.392%); 1004 KiB/s, 15 objects/s recovering 2024-08-29T15:17:18.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:18 smithi027 ceph-mon[93921]: pgmap v582: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 73391/348684 objects degraded (21.048%); 22287/348684 objects misplaced (6.392%); 1004 KiB/s, 15 objects/s recovering 2024-08-29T15:17:18.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:18.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:18.336 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:18.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:18.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:18.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:18.499 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:18.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:18 smithi171 ceph-mon[90031]: pgmap v582: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 73391/348684 objects degraded (21.048%); 22287/348684 objects misplaced (6.392%); 1004 KiB/s, 15 objects/s recovering 2024-08-29T15:17:18.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:18.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:18.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:18.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:17:18.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:18.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:18.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:18.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:18.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:18.906 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:18.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:19.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:19.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:19.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:19.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:19.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:19.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:19.289 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:17:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:17:19] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.51.0" 2024-08-29T15:17:19.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:19.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:19.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:19.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:19.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:19.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:19.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:19.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:19.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:19.662 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2024-08-29T15:17:19.663 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:17:19.663 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 3, 4, 5, 6, 7] 2024-08-29T15:17:19.663 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-29T15:17:19.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.5 2024-08-29T15:17:19.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:19.759 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:19.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:19.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:19.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:17:19.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:19.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:19.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:20.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:19 smithi171 systemd[1]: Stopping Ceph osd.5 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:17:20.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:20.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:20.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:20.249 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:20 smithi027 ceph-mon[98970]: pgmap v583: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 73128/345684 objects degraded (21.155%); 22052/345684 objects misplaced (6.379%); 1004 KiB/s, 15 objects/s recovering 2024-08-29T15:17:20.250 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:20 smithi027 ceph-mon[93921]: pgmap v583: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 73128/345684 objects degraded (21.155%); 22052/345684 objects misplaced (6.379%); 1004 KiB/s, 15 objects/s recovering 2024-08-29T15:17:20.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:20.304 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:20.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:20.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:20.416 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:20.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:20.517 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[33336]: 2024-08-29T15:17:20.210+0000 7fb2af5b1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:17:20.517 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[33336]: 2024-08-29T15:17:20.210+0000 7fb2af5b1640 -1 osd.5 195 *** Got signal Terminated *** 2024-08-29T15:17:20.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[33336]: 2024-08-29T15:17:20.210+0000 7fb2af5b1640 -1 osd.5 195 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:17:20.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:20 smithi171 ceph-mon[90031]: pgmap v583: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 73128/345684 objects degraded (21.155%); 22052/345684 objects misplaced (6.379%); 1004 KiB/s, 15 objects/s recovering 2024-08-29T15:17:20.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:20.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:20.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:20.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:20.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:20.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:20.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:20.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:20.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:20.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:21.037 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:21.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:21.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:21.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:21.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:21.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:21.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:21 smithi027 ceph-mon[93921]: osd.5 marked itself down and dead 2024-08-29T15:17:21.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:21.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:21.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:21 smithi027 ceph-mon[98970]: osd.5 marked itself down and dead 2024-08-29T15:17:21.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:21.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:21.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:21.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:21.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:21.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:21.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:21.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:21.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:21.503 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:21.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:21 smithi171 ceph-mon[90031]: osd.5 marked itself down and dead 2024-08-29T15:17:21.605 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:21.627 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:21.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:21.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:21.751 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:21.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:21.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:21.758 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:17:21.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:21.879 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:21.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:21.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:21.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:22.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:22.087 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:21 smithi171 podman[130329]: 2024-08-29 15:17:21.766900044 +0000 UTC m=+1.734180636 container died 570b002456a53b364454fe15998aa5c2adafa69b7e959118ff0d6473455464aa (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-fe43f1c, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.1, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=) 2024-08-29T15:17:22.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:22.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:22.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:22.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:22.254 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:22.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:22.356 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:22.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:22.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:22.447 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:22 smithi171 podman[130329]: 2024-08-29 15:17:22.18594709 +0000 UTC m=+2.153227659 container cleanup 570b002456a53b364454fe15998aa5c2adafa69b7e959118ff0d6473455464aa (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, GIT_CLEAN=True, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-29T15:17:22.447 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:22 smithi171 bash[130329]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5 2024-08-29T15:17:22.448 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: pgmap v584: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 71561/338166 objects degraded (21.162%); 22052/338166 objects misplaced (6.521%); 1024 KiB/s, 15 objects/s recovering 2024-08-29T15:17:22.448 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: Health check update: 2 osds down (OSD_DOWN) 2024-08-29T15:17:22.448 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: osdmap e196: 8 total, 6 up, 8 in 2024-08-29T15:17:22.448 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:17:22.448 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:17:22.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:22.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2225049005' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:22.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 71561/338166 objects degraded (21.162%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:22.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:17:22.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2225049005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:22.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:22 smithi171 ceph-mon[90031]: osdmap e197: 8 total, 6 up, 8 in 2024-08-29T15:17:22.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:22.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: pgmap v584: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 71561/338166 objects degraded (21.162%); 22052/338166 objects misplaced (6.521%); 1024 KiB/s, 15 objects/s recovering 2024-08-29T15:17:22.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: Health check update: 2 osds down (OSD_DOWN) 2024-08-29T15:17:22.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: osdmap e196: 8 total, 6 up, 8 in 2024-08-29T15:17:22.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:17:22.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:17:22.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:22.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2225049005' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:22.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 71561/338166 objects degraded (21.162%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:22.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:17:22.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2225049005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:22.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[98970]: osdmap e197: 8 total, 6 up, 8 in 2024-08-29T15:17:22.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: pgmap v584: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 8 active+undersized+degraded, 54 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 71561/338166 objects degraded (21.162%); 22052/338166 objects misplaced (6.521%); 1024 KiB/s, 15 objects/s recovering 2024-08-29T15:17:22.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: Health check update: 2 osds down (OSD_DOWN) 2024-08-29T15:17:22.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: osdmap e196: 8 total, 6 up, 8 in 2024-08-29T15:17:22.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:17:22.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:17:22.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:22.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2225049005' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:22.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 71561/338166 objects degraded (21.162%), 12 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:22.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:17:22.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2225049005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:22.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:22 smithi027 ceph-mon[93921]: osdmap e197: 8 total, 6 up, 8 in 2024-08-29T15:17:22.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:22.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:22.607 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:22.631 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:22.732 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:22.754 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:22.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:22.767 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:22 smithi171 podman[130444]: 2024-08-29 15:17:22.44618852 +0000 UTC m=+0.669128860 container remove 570b002456a53b364454fe15998aa5c2adafa69b7e959118ff0d6473455464aa (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, org.label-schema.schema-version=1.0) 2024-08-29T15:17:22.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:22.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:22.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:22.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:23.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:23.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:23.126 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:22 smithi171 podman[130646]: 2024-08-29 15:17:22.73508537 +0000 UTC m=+0.017947669 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T15:17:23.127 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:22 smithi171 podman[130646]: 2024-08-29 15:17:22.856149256 +0000 UTC m=+0.139011543 container create 1b77e45328bd75aa07b68a4c18a77da83a0109374cb2688a79c44dd43c88fc8e (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c) 2024-08-29T15:17:23.131 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:23.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:23.255 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:23.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:23.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:23.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:23 smithi171 ceph-mon[90031]: 6.c continuing backfill to osd.6 from (176'1400,195'8736] MIN to 195'8736 2024-08-29T15:17:23.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:23 smithi171 ceph-mon[90031]: osdmap e198: 8 total, 6 up, 8 in 2024-08-29T15:17:23.449 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:23 smithi171 podman[130646]: 2024-08-29 15:17:23.12626797 +0000 UTC m=+0.409130269 container init 1b77e45328bd75aa07b68a4c18a77da83a0109374cb2688a79c44dd43c88fc8e (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:17:23.449 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:23 smithi171 podman[130646]: 2024-08-29 15:17:23.13169062 +0000 UTC m=+0.414552904 container start 1b77e45328bd75aa07b68a4c18a77da83a0109374cb2688a79c44dd43c88fc8e (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-fe43f1c, ceph=True, io.buildah.version=1.37.1, GIT_CLEAN=True) 2024-08-29T15:17:23.449 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:23 smithi171 podman[130646]: 2024-08-29 15:17:23.253400102 +0000 UTC m=+0.536262405 container attach 1b77e45328bd75aa07b68a4c18a77da83a0109374cb2688a79c44dd43c88fc8e (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-08-29T15:17:23.449 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:23 smithi171 podman[130646]: 2024-08-29 15:17:23.365480868 +0000 UTC m=+0.648343161 container died 1b77e45328bd75aa07b68a4c18a77da83a0109374cb2688a79c44dd43c88fc8e (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c) 2024-08-29T15:17:23.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:23.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:23.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:23.558 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:23 smithi027 ceph-mon[93921]: 6.c continuing backfill to osd.6 from (176'1400,195'8736] MIN to 195'8736 2024-08-29T15:17:23.558 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:23 smithi027 ceph-mon[93921]: osdmap e198: 8 total, 6 up, 8 in 2024-08-29T15:17:23.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:23 smithi027 ceph-mon[98970]: 6.c continuing backfill to osd.6 from (176'1400,195'8736] MIN to 195'8736 2024-08-29T15:17:23.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:23 smithi027 ceph-mon[98970]: osdmap e198: 8 total, 6 up, 8 in 2024-08-29T15:17:23.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:23.632 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:23.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:23.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:23.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:23.882 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:23.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:23.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:23.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:24.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:24.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:24.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.5.service' 2024-08-29T15:17:24.102 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:17:24.102 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-29T15:17:24.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:24.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:24.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:24.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:24.276 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:24 smithi027 ceph-mon[93921]: pgmap v587: 89 pgs: 44 active+clean, 2 undersized+peered, 8 active+undersized+degraded, 10 active+undersized, 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 stale+active+clean, 1 stale+active+undersized+degraded, 2 stale+active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 stale+active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 84732/333840 objects degraded (25.381%); 21602/333840 objects misplaced (6.471%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:17:24.277 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:24 smithi027 ceph-mon[98970]: pgmap v587: 89 pgs: 44 active+clean, 2 undersized+peered, 8 active+undersized+degraded, 10 active+undersized, 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 stale+active+clean, 1 stale+active+undersized+degraded, 2 stale+active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 stale+active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 84732/333840 objects degraded (25.381%); 21602/333840 objects misplaced (6.471%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:17:24.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:24.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:24.497 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:24.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:17:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:24 smithi171 ceph-mon[90031]: pgmap v587: 89 pgs: 44 active+clean, 2 undersized+peered, 8 active+undersized+degraded, 10 active+undersized, 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 stale+active+clean, 1 stale+active+undersized+degraded, 2 stale+active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 stale+active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 84732/333840 objects degraded (25.381%); 21602/333840 objects misplaced (6.471%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:17:24.520 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:24.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68914 5.c 5:3b301fb0:::benchmark_data_smithi027_135490_object68913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.916458+0000", 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 379.61725740700001, 2024-08-29T15:17:24.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13917643499999999, 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68914 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916458+0000", 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916458+0000", 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916459+0000", 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.333e-06 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916479+0000", 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0097000000000001e-05 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916490+0000", 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0786e-05 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916496+0000", 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5690000000000004e-06 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916540+0000", 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3767999999999998e-05 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916626+0000", 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6602999999999997e-05 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916730+0000", 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104308 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.918642+0000", 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019117649999999999 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.918979+0000", 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033689100000000002 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055434+0000", 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13645533700000001 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055589+0000", 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015502100000000001 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055634+0000", 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4956999999999997e-05 2024-08-29T15:17:24.554 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68920 5.c 5:320cd6cb:::benchmark_data_smithi027_135490_object68919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.921040+0000", 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 379.612675723, 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134813617, 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.555 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68920 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921040+0000", 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921040+0000", 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921041+0000", 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3740000000000001e-06 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921061+0000", 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9623000000000002e-05 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921071+0000", 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0229e-05 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921076+0000", 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4779999999999999e-06 2024-08-29T15:17:24.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921121+0000", 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4415000000000001e-05 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921247+0000", 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012585 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921345+0000", 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8319999999999994e-05 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.923113+0000", 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001768071 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.924140+0000", 2024-08-29T15:17:24.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010273820000000001 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055681+0000", 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.131540405 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055808+0000", 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127222 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055853+0000", 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5247999999999998e-05 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68927 5.c 5:3864dffc:::benchmark_data_smithi027_135490_object68926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.929290+0000", 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 379.60442585099997, 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.127866969, 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.562 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68927 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929290+0000", 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929290+0000", 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929291+0000", 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5149999999999999e-06 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929311+0000", 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9524000000000001e-05 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929320+0000", 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2070000000000001e-06 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929325+0000", 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2360000000000004e-06 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929368+0000", 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3155999999999998e-05 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929457+0000", 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8553000000000004e-05 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929549+0000", 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2368999999999996e-05 2024-08-29T15:17:24.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.931187+0000", 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001637388 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.931438+0000", 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025143000000000002 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.057018+0000", 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12558042799999999 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.057071+0000", 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3007000000000003e-05 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.057157+0000", 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5155999999999996e-05 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.568 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:237081 5.0 e137/111 5:0dda9191:::benchmark_data_smithi027_135490_object103329:head v 137'14454, mlcod=137'14452)", 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.798340+0000", 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.73537492400001, 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11104399, 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798340+0000", 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798340+0000", 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798341+0000", 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5899999999999995e-07 2024-08-29T15:17:24.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798348+0000", 2024-08-29T15:17:24.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4749999999999998e-06 2024-08-29T15:17:24.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798358+0000", 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.063e-05 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798365+0000", 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5440000000000004e-06 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798459+0000", 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4293e-05 2024-08-29T15:17:24.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798477+0000", 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7669999999999999e-05 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909340+0000", 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.110863032 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909384+0000", 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4487000000000003e-05 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:237080 5.8 e137/111 5:144f62ea:::benchmark_data_smithi027_135490_object103328:head v 137'14601, mlcod=137'14597)", 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.802875+0000", 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.73084038600001, 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.106458555, 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.575 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802875+0000", 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802875+0000", 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802877+0000", 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5039999999999999e-06 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802884+0000", 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9440000000000005e-06 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802898+0000", 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3244999999999999e-05 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802906+0000", 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3149999999999992e-06 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802966+0000", 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0229000000000002e-05 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802988+0000", 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1809000000000001e-05 2024-08-29T15:17:24.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.813444+0000", 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010456045000000001 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909334+0000", 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.095889463999999994 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:237085 5.0 e137/111 5:02701778:::benchmark_data_smithi027_135490_object103333:head v 137'14455, mlcod=137'14452)", 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.804867+0000", 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.72884812999999, 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10456966199999999, 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804867+0000", 2024-08-29T15:17:24.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804867+0000", 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804868+0000", 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8299999999999995e-07 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804877+0000", 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3100000000000001e-06 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804889+0000", 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2812e-05 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804897+0000", 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7039999999999994e-06 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.809114+0000", 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042168689999999998 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.809133+0000", 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8963000000000001e-05 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909387+0000", 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.100254073 2024-08-29T15:17:24.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909437+0000", 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9947999999999997e-05 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237090 5.c 5:357a930e:::benchmark_data_smithi027_135490_object103338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.807682+0000", 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.72603389599999, 2024-08-29T15:17:24.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.106071554, 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237090 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807682+0000", 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807682+0000", 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807684+0000", 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0109999999999999e-06 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807691+0000", 2024-08-29T15:17:24.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3710000000000002e-06 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807709+0000", 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7995e-05 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807718+0000", 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1200000000000008e-06 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807787+0000", 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9233999999999995e-05 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807924+0000", 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013661200000000001 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.808040+0000", 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115635 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.812900+0000", 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048600780000000003 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909344+0000", 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.096444385999999993 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.913576+0000", 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042316749999999998 2024-08-29T15:17:24.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.913686+0000", 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011013100000000001 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.913753+0000", 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7305999999999996e-05 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.590 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239392 5.7 5:e745e56f:::benchmark_data_smithi027_135490_object105640:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.749988+0000", 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 198.783727457, 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1819669820000001, 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239392 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749988+0000", 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749988+0000", 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749989+0000", 2024-08-29T15:17:24.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3939999999999999e-06 2024-08-29T15:17:24.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749994+0000", 2024-08-29T15:17:24.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1670000000000001e-06 2024-08-29T15:17:24.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750008+0000", 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4151999999999999e-05 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750018+0000", 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0678e-05 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754784+0000", 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047659549999999997 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754794+0000", 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0009e-05 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700248+0000", 2024-08-29T15:17:24.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94545362099999997 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700260+0000", 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1853e-05 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.928959+0000", 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2286991270000001 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.931955+0000", 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029960260000000002 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239398 5.7 5:ed13b1ac:::benchmark_data_smithi027_135490_object105646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.750029+0000", 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 198.78368604600001, 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2354920040000001, 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.598 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239398 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750029+0000", 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750029+0000", 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750030+0000", 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6400000000000002e-07 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750033+0000", 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6850000000000001e-06 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750040+0000", 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750045+0000", 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2939999999999996e-06 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754817+0000", 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047715550000000002 2024-08-29T15:17:24.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754839+0000", 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2296000000000001e-05 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700288+0000", 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94544935299999999 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700297+0000", 2024-08-29T15:17:24.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1990000000000006e-06 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.932308+0000", 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.232011145 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933364+0000", 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001056476 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4,5", 2024-08-29T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933939+0000", 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00057509599999999998 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.961786+0000", 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027846619 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.965789+0000", 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040033589999999997 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.983390+0000", 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017600437999999999 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.985244+0000", 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001854198 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.985391+0000", 2024-08-29T15:17:24.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014689600000000001 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.985521+0000", 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013052300000000001 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:242791 5.7 5:e93ed9d7:::benchmark_data_smithi027_135490_object109039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:12.856201+0000", 2024-08-29T15:17:24.606 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 191.67751423300001, 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13081316600000001, 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 242791 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856201+0000", 2024-08-29T15:17:24.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856201+0000", 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856203+0000", 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7549999999999999e-06 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856207+0000", 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5349999999999998e-06 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856218+0000", 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0399e-05 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856224+0000", 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4910000000000002e-06 2024-08-29T15:17:24.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865071+0000", 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088467279999999999 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865215+0000", 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014368000000000001 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4,5", 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865360+0000", 2024-08-29T15:17:24.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144938 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.869315+0000", 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039552889999999999 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.876209+0000", 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068940850000000003 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.877487+0000", 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001277891 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.986727+0000", 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.109240306 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.986887+0000", 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015986400000000001 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.987014+0000", 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127205 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:242793 5.c 5:30095f22:::benchmark_data_smithi027_135490_object109041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:12.865875+0000", 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 191.667840278, 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.102709334, 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.613 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 242793 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865875+0000", 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865875+0000", 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865876+0000", 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3039999999999999e-06 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865882+0000", 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2109999999999997e-06 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865898+0000", 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6033e-05 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865905+0000", 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5809999999999999e-06 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.866112+0000", 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000206405 2024-08-29T15:17:24.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.866211+0000", 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9601999999999999e-05 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.866315+0000", 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103751 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.871304+0000", 2024-08-29T15:17:24.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049892490000000003 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.879733+0000", 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084289050000000004 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.968476+0000", 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.088742677000000006 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.968538+0000", 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2026999999999994e-05 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.968584+0000", 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6588999999999997e-05 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:68822 6.1 6:86e313fb:::benchmark_data_smithi027_169445_object68821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2024-08-29T15:17:24.619 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 73.167307492999996, 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.107834163, 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68822 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366410+0000", 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8899999999999999e-06 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366438+0000", 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8589e-05 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366453+0000", 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5085e-05 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366463+0000", 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1179999999999998e-06 2024-08-29T15:17:24.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366622+0000", 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000159746 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366765+0000", 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000142237 2024-08-29T15:17:24.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366907+0000", 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014195399999999999 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.369648+0000", 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002740957 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.370870+0000", 2024-08-29T15:17:24.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001222835 2024-08-29T15:17:24.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474094+0000", 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10322338 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474180+0000", 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6434999999999995e-05 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474242+0000", 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1937e-05 2024-08-29T15:17:24.626 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124391 6.1 6:80fa3010:::benchmark_data_smithi027_169445_object124390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 26.522520660000001, 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1538895120000001, 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.627 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124391 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:17:24.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011196+0000", 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3300000000000001e-07 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011216+0000", 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0068000000000002e-05 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011228+0000", 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.243e-05 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011234+0000", 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2890000000000003e-06 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.018714+0000", 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007479388 2024-08-29T15:17:24.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.020057+0000", 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001342768 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.020202+0000", 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014581300000000001 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.035616+0000", 2024-08-29T15:17:24.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015413736000000001 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.035626+0000", 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0301e-05 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025526+0000", 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98989923499999999 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025540+0000", 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4188e-05 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.164991+0000", 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139451225 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165084+0000", 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3238000000000004e-05 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.633 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124405 6.1 6:81409361:::benchmark_data_smithi027_169445_object124404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 26.505927302, 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1459181940000001, 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.634 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124405 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027789+0000", 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.24e-06 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027810+0000", 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0891999999999999e-05 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027833+0000", 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3217000000000001e-05 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027843+0000", 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1419999999999999e-06 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027929+0000", 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6318999999999998e-05 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.028098+0000", 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016911500000000001 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.028250+0000", 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015169000000000001 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.036231+0000", 2024-08-29T15:17:24.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079816360000000003 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.036241+0000", 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9119999999999993e-06 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025573+0000", 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98933212100000001 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025582+0000", 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2900000000000002e-06 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165270+0000", 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1396881759999999 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165781+0000", 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00051096900000000001 2024-08-29T15:17:24.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173600+0000", 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0078194070000000004 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173706+0000", 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106068 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124438 6.1 6:8a3680bc:::benchmark_data_smithi027_169445_object124437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 26.464013624, 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.104159272, 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124438 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069704+0000", 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.719e-06 2024-08-29T15:17:24.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069728+0000", 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4142999999999999e-05 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069747+0000", 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9177e-05 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069755+0000", 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4859999999999997e-06 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069844+0000", 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8729999999999999e-05 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069853+0000", 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0489999999999993e-06 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025603+0000", 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.955749607 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025609+0000", 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5390000000000004e-06 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165847+0000", 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1402372869999999 2024-08-29T15:17:24.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165953+0000", 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106016 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.166064+0000", 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111908 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.168561+0000", 2024-08-29T15:17:24.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024969829999999999 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.169970+0000", 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014081149999999999 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173730+0000", 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037604190000000001 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173779+0000", 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9508000000000001e-05 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173861+0000", 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1586000000000001e-05 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140304 6.4 6:2a4ae72d:::benchmark_data_smithi027_169445_object14877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:17:24.649 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.3540136779999998, 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63546416900000002, 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140304 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:17:24.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179704+0000", 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0020000000000001e-06 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179709+0000", 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6659999999999996e-06 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179727+0000", 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7693000000000002e-05 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179735+0000", 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7819999999999994e-06 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179797+0000", 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1838e-05 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179805+0000", 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3289999999999994e-06 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810640+0000", 2024-08-29T15:17:24.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63083506700000003 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810759+0000", 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011892699999999999 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3", 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810883+0000", 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124098 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814964+0000", 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040809540000000004 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815029+0000", 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4528000000000003e-05 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815081+0000", 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2151999999999998e-05 2024-08-29T15:17:24.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815166+0000", 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5133e-05 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140324 6.d 6:b4e34602:::benchmark_data_smithi027_169445_object14897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.3327030139999998, 2024-08-29T15:17:24.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.61402909999999999, 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140324 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201014+0000", 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6649999999999999e-06 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201019+0000", 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0340000000000004e-06 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201033+0000", 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4331000000000001e-05 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201041+0000", 2024-08-29T15:17:24.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2200000000000003e-06 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201098+0000", 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7484000000000001e-05 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201106+0000", 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8949999999999998e-06 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.786966+0000", 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58585960400000003 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.790090+0000", 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003124258 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.790556+0000", 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046574499999999999 2024-08-29T15:17:24.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806640+0000", 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016084317000000001 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814905+0000", 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082646869999999997 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814955+0000", 2024-08-29T15:17:24.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9997999999999998e-05 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815042+0000", 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6861999999999996e-05 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140335 6.d 6:b3ea1342:::benchmark_data_smithi027_169445_object14908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:17:24.663 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.3161541319999999, 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59763891800000002, 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140335 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:17:24.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217563+0000", 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.88e-06 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217570+0000", 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0049999999999998e-06 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217589+0000", 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8686e-05 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217598+0000", 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6419999999999996e-06 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217665+0000", 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7447000000000007e-05 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217675+0000", 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7799999999999995e-06 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802631+0000", 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58495612900000005 2024-08-29T15:17:24.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802735+0000", 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104397 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802810+0000", 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5044999999999997e-05 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814892+0000", 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01208188 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815054+0000", 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016216000000000001 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815094+0000", 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.015e-05 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815200+0000", 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105717 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140337 6.d 6:bf19c6d4:::benchmark_data_smithi027_169445_object14910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.312828449, 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59452165300000004, 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.670 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140337 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220889+0000", 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.708e-06 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220896+0000", 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8249999999999999e-06 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220914+0000", 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8476000000000001e-05 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220923+0000", 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1209999999999996e-06 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.221258+0000", 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033503 2024-08-29T15:17:24.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.221267+0000", 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7930000000000003e-06 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806696+0000", 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58542917100000003 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806843+0000", 2024-08-29T15:17:24.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000146623 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806962+0000", 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011940200000000001 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815264+0000", 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083018880000000003 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815277+0000", 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2835e-05 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815315+0000", 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7991000000000003e-05 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815409+0000", 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3789999999999995e-05 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.676 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140341 6.d 6:ba4d184c:::benchmark_data_smithi027_169445_object14914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.298650737, 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58419336799999999, 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:24.677 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140341 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235067+0000", 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.984e-06 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235074+0000", 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0840000000000003e-06 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235095+0000", 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1307000000000001e-05 2024-08-29T15:17:24.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235106+0000", 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0665999999999999e-05 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235173+0000", 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7220999999999999e-05 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235185+0000", 2024-08-29T15:17:24.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2189e-05 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810910+0000", 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57572489900000001 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.811020+0000", 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110179 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.811153+0000", 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013316200000000001 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815416+0000", 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042625670000000001 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819099+0000", 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003683151 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819162+0000", 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2558000000000007e-05 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819258+0000", 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6401000000000001e-05 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:24.683 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:17:24.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:24.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:24.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:24.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:24.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:24.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:24.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:24.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:25.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:25.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:25.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:25.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:25.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:25.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:25.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:25.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:25.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:25.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:25.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:25.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:25.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:25.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:25.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:25.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:25.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:25.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:25.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:26.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:26.081 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:26.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:26.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:26.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:26.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:26.411 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:26 smithi027 ceph-mon[98970]: pgmap v589: 89 pgs: 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 7 active+undersized+degraded, 12 active+undersized, 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 stale+active+clean, 1 stale+active+undersized+degraded, 2 stale+active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 stale+active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 84493/333117 objects degraded (25.364%); 21602/333117 objects misplaced (6.485%); 821 KiB/s, 12 objects/s recovering 2024-08-29T15:17:26.411 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:26 smithi027 ceph-mon[93921]: pgmap v589: 89 pgs: 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 7 active+undersized+degraded, 12 active+undersized, 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 stale+active+clean, 1 stale+active+undersized+degraded, 2 stale+active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 stale+active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 84493/333117 objects degraded (25.364%); 21602/333117 objects misplaced (6.485%); 821 KiB/s, 12 objects/s recovering 2024-08-29T15:17:26.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:26.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:26.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:26 smithi171 ceph-mon[90031]: pgmap v589: 89 pgs: 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 7 active+undersized+degraded, 12 active+undersized, 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 8 stale+active+clean, 1 stale+active+undersized+degraded, 2 stale+active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 stale+active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 84493/333117 objects degraded (25.364%); 21602/333117 objects misplaced (6.485%); 821 KiB/s, 12 objects/s recovering 2024-08-29T15:17:26.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:26.581 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:17:26.581 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:17:26.581 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:17:26.582 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:17:26.582 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:17:26.582 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:17:26.582 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:17:26.582 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:17:26.582 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:17:26.582 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-08-29T15:17:26.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:26.597 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:17:26.597 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:17:26.614 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:17:26.614 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:17:26.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:26.709 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:26.811 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:26.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:26.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:26.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:26.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:27.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:27.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:27.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:27.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:27.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:17:27.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:27.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:27.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:27.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:27.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:27.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3441694921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 84493/333117 objects degraded (25.364%), 14 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:27.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:27.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3441694921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:27.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 84493/333117 objects degraded (25.364%), 14 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3441694921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 84493/333117 objects degraded (25.364%), 14 pgs degraded (PG_DEGRADED) 2024-08-29T15:17:27.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:27.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:27.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:27.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:27.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:27.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:27.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:27.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:27.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:27.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:27.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:28.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:28.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:28.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:17:28.210 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:17:28.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:28.223 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:28.228 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:28.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:28.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:28.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:28.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:28.490 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:28 smithi027 ceph-mon[98970]: pgmap v590: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 103785/331338 objects degraded (31.323%); 21577/331338 objects misplaced (6.512%); 2.3 MiB/s, 28 objects/s recovering 2024-08-29T15:17:28.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:28 smithi027 ceph-mon[93921]: pgmap v590: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 103785/331338 objects degraded (31.323%); 21577/331338 objects misplaced (6.512%); 2.3 MiB/s, 28 objects/s recovering 2024-08-29T15:17:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:28 smithi171 ceph-mon[90031]: pgmap v590: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 103785/331338 objects degraded (31.323%); 21577/331338 objects misplaced (6.512%); 2.3 MiB/s, 28 objects/s recovering 2024-08-29T15:17:28.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:28.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:28.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:28.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:28.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:28.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:28.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:28.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:28.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:28.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:28.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:28.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:29.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:29.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:29.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:29.103 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [2, 5] live_osds: [0, 1, 3, 4, 6, 7] 2024-08-29T15:17:29.103 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:17:29.103 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2024-08-29T15:17:29.103 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2024-08-29T15:17:29.103 DEBUG:teuthology.orchestra.run.smithi171:osd.5> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.5.service 2024-08-29T15:17:29.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl start ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.5 2024-08-29T15:17:29.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:29.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:29.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:29.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:29.252 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:17:29] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.51.0" 2024-08-29T15:17:29.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:29.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:29.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:29.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:29.432 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:29 smithi171 systemd[1]: Starting Ceph osd.5 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:17:29.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:29.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:29.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:29.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:29.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:29.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:29.683 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:29 smithi171 podman[131271]: 2024-08-29 15:17:29.334847106 +0000 UTC m=+0.017476031 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T15:17:29.683 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:29 smithi171 podman[131271]: 2024-08-29 15:17:29.458224579 +0000 UTC m=+0.140853501 container create 6fd986c850afd12f41210bd932958da8ebcc69e201681b291db7fec75dd37d87 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, ceph=True, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-29T15:17:29.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:29.750 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:29.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:29.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:29.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:29.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:29.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:29.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:29.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.004 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:30.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:29 smithi171 ceph-mon[90031]: pgmap v591: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 103700/330927 objects degraded (31.336%); 21548/330927 objects misplaced (6.511%); 1.9 MiB/s, 23 objects/s recovering 2024-08-29T15:17:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:29 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-08-29T15:17:30.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:29 smithi171 podman[131271]: 2024-08-29 15:17:29.72011769 +0000 UTC m=+0.402746619 container init 6fd986c850afd12f41210bd932958da8ebcc69e201681b291db7fec75dd37d87 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-29T15:17:30.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:29 smithi171 podman[131271]: 2024-08-29 15:17:29.724700564 +0000 UTC m=+0.407329488 container start 6fd986c850afd12f41210bd932958da8ebcc69e201681b291db7fec75dd37d87 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=reef-fe43f1c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-08-29T15:17:30.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:29 smithi171 podman[131271]: 2024-08-29 15:17:29.819981236 +0000 UTC m=+0.502610160 container attach 6fd986c850afd12f41210bd932958da8ebcc69e201681b291db7fec75dd37d87 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate, RELEASE=reef-fe43f1c, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.1) 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[98970]: pgmap v591: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 103700/330927 objects degraded (31.336%); 21548/330927 objects misplaced (6.511%); 1.9 MiB/s, 23 objects/s recovering 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[93921]: pgmap v591: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 259 op/s; 103700/330927 objects degraded (31.336%); 21548/330927 objects misplaced (6.511%); 1.9 MiB/s, 23 objects/s recovering 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:30.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:29 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-08-29T15:17:30.105 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:30.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:30.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:17:30.231 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:30.253 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:30.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:30.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:30.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:30.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:30.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:30.517 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate[131302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 bash[131271]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate[131302]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 bash[131271]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate[131302]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 bash[131271]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate[131302]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 bash[131271]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate[131302]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 bash[131271]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate[131302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 bash[131271]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate[131302]: --> ceph-volume raw activate successful for osd ID: 5 2024-08-29T15:17:30.518 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 bash[131271]: --> ceph-volume raw activate successful for osd ID: 5 2024-08-29T15:17:30.519 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 podman[131271]: 2024-08-29 15:17:30.156732072 +0000 UTC m=+0.839361010 container died 6fd986c850afd12f41210bd932958da8ebcc69e201681b291db7fec75dd37d87 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.1, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c, org.label-schema.vendor=CentOS) 2024-08-29T15:17:30.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:30.627 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:30.729 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:30.750 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:30.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:30.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:30.868 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:30 smithi171 podman[131482]: 2024-08-29 15:17:30.866854493 +0000 UTC m=+0.704074914 container remove 6fd986c850afd12f41210bd932958da8ebcc69e201681b291db7fec75dd37d87 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-activate, GIT_CLEAN=True, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:17:30.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:30.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:30.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:31.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:31.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:31.128 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:31.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:31.135 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 podman[131573]: 2024-08-29 15:17:31.003480388 +0000 UTC m=+0.022737514 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T15:17:31.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:31.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:31.327 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:17:31.327 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:17:31.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:31.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:31.407 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 podman[131573]: 2024-08-29 15:17:31.134630339 +0000 UTC m=+0.153887447 container create db94cc15457ad714d1f99f9dc40d3c58479465ff4d188b709c06a72c3a04ad23 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, org.label-schema.vendor=CentOS, RELEASE=reef-fe43f1c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:17:31.408 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 podman[131573]: 2024-08-29 15:17:31.379616616 +0000 UTC m=+0.398873736 container init db94cc15457ad714d1f99f9dc40d3c58479465ff4d188b709c06a72c3a04ad23 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-29T15:17:31.408 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 podman[131573]: 2024-08-29 15:17:31.383537877 +0000 UTC m=+0.402794994 container start db94cc15457ad714d1f99f9dc40d3c58479465ff4d188b709c06a72c3a04ad23 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-fe43f1c, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-29T15:17:31.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:31.485 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:17:31.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:31.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:31.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:31.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:31.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:31.686 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 ceph-osd[131624]: -- 172.21.15.171:0/1532614300 <== mon.1 v2:172.21.15.171:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564d29c771e0 con 0x564d2a9f8000 2024-08-29T15:17:31.686 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 bash[131573]: db94cc15457ad714d1f99f9dc40d3c58479465ff4d188b709c06a72c3a04ad23 2024-08-29T15:17:31.686 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 systemd[1]: Started Ceph osd.5 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:17:31.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:31.743 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:17:31.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:31.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:31 smithi027 ceph-mon[93921]: pgmap v592: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 103212/330264 objects degraded (31.251%); 21548/330264 objects misplaced (6.524%); 3.7 MiB/s, 52 objects/s recovering 2024-08-29T15:17:31.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:31 smithi027 ceph-mon[98970]: pgmap v592: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 103212/330264 objects degraded (31.251%); 21548/330264 objects misplaced (6.524%); 3.7 MiB/s, 52 objects/s recovering 2024-08-29T15:17:31.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:31.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:31.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:31.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:32.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:31 smithi171 ceph-mon[90031]: pgmap v592: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 103212/330264 objects degraded (31.251%); 21548/330264 objects misplaced (6.524%); 3.7 MiB/s, 52 objects/s recovering 2024-08-29T15:17:32.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:31 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:17:31.927+0000 7f0627b96640 -1 Falling back to public interface 2024-08-29T15:17:32.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:32.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:32.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:32.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:32.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:32.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:32.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:17:32.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:32.384 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:32.453 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:32.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:32.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:32.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:32.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:32.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:32.694 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:17:32.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:32.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:32.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:32.861 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:32.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:32.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:32.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.008 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:33.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/497409061' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:32 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 103212/330264 objects degraded (31.251%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:33.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/497409061' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:33.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:32 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:33.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 103212/330264 objects degraded (31.251%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/497409061' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:32 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 103212/330264 objects degraded (31.251%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:33.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:33.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:33.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:33.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:33.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:33.384 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:33.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:33.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:33.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:33.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:33.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:33.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:33.757 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:33.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[93921]: pgmap v593: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 103030/329547 objects degraded (31.264%); 21491/329547 objects misplaced (6.521%); 3.2 MiB/s, 46 objects/s recovering 2024-08-29T15:17:33.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:33.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:17:33.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[93921]: osdmap e199: 8 total, 6 up, 8 in 2024-08-29T15:17:33.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[98970]: pgmap v593: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 103030/329547 objects degraded (31.264%); 21491/329547 objects misplaced (6.521%); 3.2 MiB/s, 46 objects/s recovering 2024-08-29T15:17:33.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:33.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:17:33.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:33 smithi027 ceph-mon[98970]: osdmap e199: 8 total, 6 up, 8 in 2024-08-29T15:17:33.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:33.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:33.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:33.959 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:33 smithi171 ceph-mon[90031]: pgmap v593: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 103030/329547 objects degraded (31.264%); 21491/329547 objects misplaced (6.521%); 3.2 MiB/s, 46 objects/s recovering 2024-08-29T15:17:33.959 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:33 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:33.959 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:33 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:17:33.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:33 smithi171 ceph-mon[90031]: osdmap e199: 8 total, 6 up, 8 in 2024-08-29T15:17:33.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:34.007 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:34.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:34.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:34.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:34.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:34.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:34.257 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:34.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:34.258 INFO:teuthology.orchestra.run.smithi171.stderr:no valid command found; 10 closest matches: 2024-08-29T15:17:34.258 INFO:teuthology.orchestra.run.smithi171.stderr:0 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:1 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:2 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:abort 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:assert 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:config diff 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:config diff get 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:config get 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:config help [] 2024-08-29T15:17:34.259 INFO:teuthology.orchestra.run.smithi171.stderr:config set ... 2024-08-29T15:17:34.260 INFO:teuthology.orchestra.run.smithi171.stderr:admin_socket: invalid command 2024-08-29T15:17:34.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:34.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:34.483 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:34.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:34.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:34.640 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:34.659 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:34.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:34.765 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:34.867 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:34.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:34.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:34.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:34.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:35.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:35.094 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:17:35.094 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2024-08-29T15:17:35.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:35.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:35.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:35.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:35.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:35.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:35.369 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:17:35.369 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:17:35.374 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:17:35.375 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:17:35.375 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:17:35.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:35.393 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:17:35.393 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:17:35.400 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:35.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:35.411 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:17:35.412 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:17:35.412 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.5 2024-08-29T15:17:35.429 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:17:35.429 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:17:35.446 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:17:35.447 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:17:35.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:35.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:35.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:35.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:17:35.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:35.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:35.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:35.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:35.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:35.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:35.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:35 smithi027 ceph-mon[93921]: pgmap v595: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 102988/329421 objects degraded (31.263%); 21491/329421 objects misplaced (6.524%); 3.2 MiB/s, 46 objects/s recovering 2024-08-29T15:17:35.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3904787191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:35.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:35 smithi027 ceph-mon[98970]: pgmap v595: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 102988/329421 objects degraded (31.263%); 21491/329421 objects misplaced (6.524%); 3.2 MiB/s, 46 objects/s recovering 2024-08-29T15:17:35.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3904787191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:35.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:35.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:35.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:36.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:35 smithi171 ceph-mon[90031]: pgmap v595: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 102988/329421 objects degraded (31.263%); 21491/329421 objects misplaced (6.524%); 3.2 MiB/s, 46 objects/s recovering 2024-08-29T15:17:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3904787191' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:36.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:36.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:36.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:36.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:36.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:36.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:36.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:36.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:36.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:36.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:36.530 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:36.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:36.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:36.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:36.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:36.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:36.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:36.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:36.906 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:36.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:36.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:37.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:37.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:37.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:37.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:37.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:37.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:37.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:37.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:37.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:17:37.388 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:17:37.406 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:37.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:37.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:37.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:37.605 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:17:37.605 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:17:37.605 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:17:37.606 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:17:37.606 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:17:37.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:37.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:37.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:37.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:37.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:37.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:37.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:38.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:37 smithi171 ceph-mon[90031]: pgmap v596: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 102478/328668 objects degraded (31.180%); 21460/328668 objects misplaced (6.529%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:17:38.021 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 102478/328668 objects degraded (31.180%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:38.038 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:38.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:37 smithi027 ceph-mon[93921]: pgmap v596: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 102478/328668 objects degraded (31.180%); 21460/328668 objects misplaced (6.529%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:17:38.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 102478/328668 objects degraded (31.180%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:38.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:37 smithi027 ceph-mon[98970]: pgmap v596: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 88 op/s; 102478/328668 objects degraded (31.180%); 21460/328668 objects misplaced (6.529%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:17:38.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 102478/328668 objects degraded (31.180%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:38.139 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:38.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:38.264 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:38.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:38.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:38.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:38.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:38.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:38.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:17:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:38.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:38.605 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:38.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:38.660 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:38.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:38.693 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:17:38.324+0000 7f0627b96640 -1 osd.5 195 log_to_monitors true 2024-08-29T15:17:38.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:38.784 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:38 smithi027 ceph-mon[93921]: from='osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-29T15:17:38.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:38 smithi027 ceph-mon[93921]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-29T15:17:38.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:38 smithi027 ceph-mon[98970]: from='osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-29T15:17:38.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:38 smithi027 ceph-mon[98970]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-29T15:17:38.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:38.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:38.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:38.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:39.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:38 smithi171 ceph-mon[90031]: from='osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-29T15:17:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:38 smithi171 ceph-mon[90031]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-08-29T15:17:39.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:17:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:17:38.711+0000 7f061f940640 -1 osd.5 195 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T15:17:39.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:39.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:39.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:39.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:39.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:39.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:39.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:17:39] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.51.0" 2024-08-29T15:17:39.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:39.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:17:39.407 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:39.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:39.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:39.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:39.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:39.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:39.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:39.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:39.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:39.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:39.793 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:39.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:39.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:39.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:39.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:40.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:39 smithi171 ceph-mon[90031]: pgmap v597: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 89 op/s; 102395/328245 objects degraded (31.195%); 21429/328245 objects misplaced (6.528%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:17:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:39 smithi171 ceph-mon[90031]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-29T15:17:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:39 smithi171 ceph-mon[90031]: osdmap e200: 8 total, 6 up, 8 in 2024-08-29T15:17:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:39 smithi171 ceph-mon[90031]: from='osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]} : dispatch 2024-08-29T15:17:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:39 smithi171 ceph-mon[90031]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]} : dispatch 2024-08-29T15:17:40.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[98970]: pgmap v597: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 89 op/s; 102395/328245 objects degraded (31.195%); 21429/328245 objects misplaced (6.528%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[98970]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[98970]: osdmap e200: 8 total, 6 up, 8 in 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[98970]: from='osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]} : dispatch 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[98970]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]} : dispatch 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[93921]: pgmap v597: 89 pgs: 1 active+recovering+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 18 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 89 op/s; 102395/328245 objects degraded (31.195%); 21429/328245 objects misplaced (6.528%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[93921]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[93921]: osdmap e200: 8 total, 6 up, 8 in 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[93921]: from='osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]} : dispatch 2024-08-29T15:17:40.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:39 smithi027 ceph-mon[93921]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi171", "root=default"]} : dispatch 2024-08-29T15:17:40.033 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:40.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:17:40.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:40.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:40.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:40.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:40.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:40.318 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:17:40.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:40.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:40.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:40.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:40.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:40.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:40.664 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:40.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:40.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:40.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:40.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:40.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:40.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:41.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:41.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:40 smithi171 ceph-mon[90031]: Health check update: 1 osds down (OSD_DOWN) 2024-08-29T15:17:41.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:40 smithi171 ceph-mon[90031]: osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915] boot 2024-08-29T15:17:41.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:40 smithi171 ceph-mon[90031]: osdmap e201: 8 total, 7 up, 8 in 2024-08-29T15:17:41.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:40 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-29T15:17:41.019 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:40 smithi171 ceph-mon[90031]: osdmap e202: 8 total, 7 up, 8 in 2024-08-29T15:17:41.031 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[93921]: Health check update: 1 osds down (OSD_DOWN) 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[93921]: osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915] boot 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[93921]: osdmap e201: 8 total, 7 up, 8 in 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[93921]: osdmap e202: 8 total, 7 up, 8 in 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[98970]: Health check update: 1 osds down (OSD_DOWN) 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[98970]: osd.5 [v2:172.21.15.171:6808/2950085915,v1:172.21.15.171:6809/2950085915] boot 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[98970]: osdmap e201: 8 total, 7 up, 8 in 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-08-29T15:17:41.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:40 smithi027 ceph-mon[98970]: osdmap e202: 8 total, 7 up, 8 in 2024-08-29T15:17:41.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:41.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:41.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:41.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:41.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:41.286 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:41.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:41.388 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:41.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:41.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:41.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:41.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:41.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:41.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:41.708 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:17:41.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:41.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:41.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:41.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:41.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:41.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:42.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:41 smithi171 ceph-mon[90031]: pgmap v600: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 1 active+recovering+undersized+degraded, 1 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 16 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 72 op/s; 102005/327846 objects degraded (31.114%); 21417/327846 objects misplaced (6.533%); 4.5 MiB/s, 71 objects/s recovering 2024-08-29T15:17:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4269188437' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:41 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:41 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:42.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:41 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:42.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:41 smithi171 ceph-mon[90031]: osdmap e203: 8 total, 7 up, 8 in 2024-08-29T15:17:42.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[93921]: pgmap v600: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 1 active+recovering+undersized+degraded, 1 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 16 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 72 op/s; 102005/327846 objects degraded (31.114%); 21417/327846 objects misplaced (6.533%); 4.5 MiB/s, 71 objects/s recovering 2024-08-29T15:17:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4269188437' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[93921]: osdmap e203: 8 total, 7 up, 8 in 2024-08-29T15:17:42.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[98970]: pgmap v600: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 1 active+recovering+undersized+degraded, 1 active+undersized+remapped, 2 undersized+degraded+peered, 36 active+clean, 4 undersized+peered, 13 active+undersized+degraded, 16 active+undersized, 7 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 72 op/s; 102005/327846 objects degraded (31.114%); 21417/327846 objects misplaced (6.533%); 4.5 MiB/s, 71 objects/s recovering 2024-08-29T15:17:42.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4269188437' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:42.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:42.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:42.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:42.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:41 smithi027 ceph-mon[98970]: osdmap e203: 8 total, 7 up, 8 in 2024-08-29T15:17:42.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:42.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:42.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:42.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:42.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:42.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:42.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:42.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:42.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:42.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:42.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:42.638 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:17:42.638 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:17:42.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237081 5.0 5:0dda9191:::benchmark_data_smithi027_135490_object103329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.797802+0000", 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.83828683900001, 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.113892862, 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.639 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237081 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797802+0000", 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797802+0000", 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797807+0000", 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8189999999999999e-06 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797812+0000", 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9509999999999997e-06 2024-08-29T15:17:42.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797860+0000", 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8785999999999997e-05 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797868+0000", 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6550000000000004e-06 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.797934+0000", 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5526999999999998e-05 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798064+0000", 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013022700000000001 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798183+0000", 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119543 2024-08-29T15:17:42.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.813963+0000", 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015779782999999999 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.814006+0000", 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2895999999999997e-05 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911577+0000", 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.097570961999999997 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911648+0000", 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0757e-05 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911695+0000", 2024-08-29T15:17:42.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6956000000000002e-05 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237085 5.0 5:02701778:::benchmark_data_smithi027_135490_object103333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.802468+0000", 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.833620357, 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.109798035, 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.646 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237085 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802468+0000", 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802468+0000", 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802471+0000", 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5330000000000001e-06 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802480+0000", 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8480000000000007e-06 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802503+0000", 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3691000000000001e-05 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802513+0000", 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9189999999999994e-06 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.803202+0000", 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00068899399999999996 2024-08-29T15:17:42.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.803556+0000", 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035358399999999999 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.803995+0000", 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00043872900000000001 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.814150+0000", 2024-08-29T15:17:42.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010155229999999999 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.817712+0000", 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035621939999999999 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.911773+0000", 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.094060560000000001 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.912088+0000", 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031585599999999999 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.912266+0000", 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017789699999999999 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.652 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237116 5.0 5:06a3e46a:::benchmark_data_smithi027_135490_object103364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.838860+0000", 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.79722833100001, 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.090343065, 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:42.653 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.654 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237116 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838860+0000", 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838860+0000", 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838862+0000", 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.984e-06 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838870+0000", 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6639999999999998e-06 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838890+0000", 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9576999999999999e-05 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838898+0000", 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0779999999999996e-06 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.838977+0000", 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9532999999999999e-05 2024-08-29T15:17:42.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.839138+0000", 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016038099999999999 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.839305+0000", 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016747600000000001 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.843249+0000", 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039444700000000003 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.848136+0000", 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004886395 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.928934+0000", 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.080798306 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929076+0000", 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141834 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929203+0000", 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127367 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237118 5.0 5:098193b5:::benchmark_data_smithi027_135490_object103366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.844332+0000", 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.791756569, 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091090242000000002, 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237118 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.661 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844332+0000", 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844332+0000", 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844334+0000", 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.669e-06 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844338+0000", 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6489999999999999e-06 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844352+0000", 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4025e-05 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844361+0000", 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5690000000000005e-06 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844418+0000", 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7178e-05 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844551+0000", 2024-08-29T15:17:42.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133065 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.844702+0000", 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015070400000000001 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.848497+0000", 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037951209999999998 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.849105+0000", 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00060838400000000001 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929264+0000", 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.080158141000000002 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.929399+0000", 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013499200000000001 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.935422+0000", 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060237449999999996 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:119251 6.f 6:f1ca2ca6:::benchmark_data_smithi027_169445_object119250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.451291+0000", 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.184797596000003, 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.080446027000000003, 2024-08-29T15:17:42.667 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 119251 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451291+0000", 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451291+0000", 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451293+0000", 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.818e-06 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451312+0000", 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9281999999999999e-05 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451323+0000", 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0635999999999999e-05 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451330+0000", 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3989999999999998e-06 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451469+0000", 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013882 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451603+0000", 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013386499999999999 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4", 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451747+0000", 2024-08-29T15:17:42.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014433499999999999 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.455319+0000", 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003571524 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.466028+0000", 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010709682 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531644+0000", 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.065615204999999996 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531737+0000", 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3461000000000005e-05 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.673 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:119253 6.b 6:db2a1570:::benchmark_data_smithi027_169445_object119252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.451590+0000", 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.184498716, 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.079809555000000004, 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 119253 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451590+0000", 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451590+0000", 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451591+0000", 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4900000000000005e-07 2024-08-29T15:17:42.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451611+0000", 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9769e-05 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451621+0000", 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9860000000000006e-06 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451627+0000", 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.331e-06 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451681+0000", 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4406000000000001e-05 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451803+0000", 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000122038 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.451942+0000", 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013920299999999999 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.455666+0000", 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037239489999999998 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.460784+0000", 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051173540000000002 2024-08-29T15:17:42.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531295+0000", 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.070511198999999997 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531353+0000", 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7861e-05 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531399+0000", 2024-08-29T15:17:42.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6610000000000003e-05 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:119260 6.b 6:dd30957b:::benchmark_data_smithi027_169445_object119259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.456868+0000", 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.17922111, 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.075425158000000006, 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.680 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 119260 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.456868+0000", 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.456868+0000", 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.456869+0000", 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3549999999999999e-06 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.456892+0000", 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3241e-05 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.456904+0000", 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1883e-05 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.456911+0000", 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.781e-06 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.456971+0000", 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0547000000000002e-05 2024-08-29T15:17:42.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.457101+0000", 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000130085 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.457239+0000", 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013714 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.460798+0000", 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00355967 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531973+0000", 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.071174348999999998 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.532107+0000", 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013489900000000001 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.532250+0000", 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014281400000000001 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.532293+0000", 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2394000000000001e-05 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:119255 6.7 e188/168 6:e541586e:::benchmark_data_smithi027_169445_object119254:head v 188'7216, mlcod=188'7212)", 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.459838+0000", 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.176250342000003, 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.079339359999999998, 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.459838+0000", 2024-08-29T15:17:42.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.459838+0000", 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.459839+0000", 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9500000000000001e-07 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.460041+0000", 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020229499999999999 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.460053+0000", 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2131e-05 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.460060+0000", 2024-08-29T15:17:42.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5570000000000001e-06 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463009+0000", 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029488090000000002 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463024+0000", 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5095e-05 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539143+0000", 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.076119510000000001 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539178+0000", 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4168000000000002e-05 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.691 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:119264 6.f 6:f116c0f2:::benchmark_data_smithi027_169445_object119263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.463238+0000", 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.172850826000001, 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.076085474, 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 119264 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.692 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463238+0000", 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463238+0000", 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463239+0000", 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3370000000000001e-06 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463261+0000", 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.232e-05 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463274+0000", 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2072000000000001e-05 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.463281+0000", 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4610000000000002e-06 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531774+0000", 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.068492534999999993 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.533434+0000", 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0016605890000000001 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4", 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.533588+0000", 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000153459 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539136+0000", 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055481840000000003 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539185+0000", 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9468999999999997e-05 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539237+0000", 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1536000000000003e-05 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539323+0000", 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6512000000000001e-05 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:119268 6.6 6:6668bf6a:::benchmark_data_smithi027_169445_object119267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.464104+0000", 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.171984379999998, 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.075160326, 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.698 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:42.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:42.700 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 119268 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464104+0000", 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464104+0000", 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464105+0000", 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5799999999999999e-07 2024-08-29T15:17:42.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464122+0000", 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7050000000000001e-05 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464129+0000", 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4710000000000001e-06 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464134+0000", 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7369999999999997e-06 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464192+0000", 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7416000000000001e-05 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464350+0000", 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015843199999999999 2024-08-29T15:17:42.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.464508+0000", 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015841799999999999 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.532572+0000", 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.068063969000000002 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.532610+0000", 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7908999999999999e-05 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539086+0000", 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064755539999999997 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539192+0000", 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105652 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539265+0000", 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3059999999999995e-05 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:119270 6.7 e188/168 6:ec9c1621:::benchmark_data_smithi027_169445_object119269:head v 188'7217, mlcod=188'7213)", 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.465222+0000", 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.170867012999999, 2024-08-29T15:17:42.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.082931752999999997, 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.465222+0000", 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.465222+0000", 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.465223+0000", 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2270000000000001e-06 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.465293+0000", 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0297999999999997e-05 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.465304+0000", 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.039e-05 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.465310+0000", 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2559999999999999e-06 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531229+0000", 2024-08-29T15:17:42.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.065919401000000002 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531259+0000", 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0205999999999998e-05 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.548114+0000", 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016854356000000001 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.548153+0000", 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9619000000000001e-05 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:119276 6.f 6:fe9923cf:::benchmark_data_smithi027_169445_object119275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:53.472994+0000", 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.163094985999997, 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.082100514999999999, 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.712 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 119276 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.472994+0000", 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.472994+0000", 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.472995+0000", 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5990000000000001e-06 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.473017+0000", 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1597999999999999e-05 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531437+0000", 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.05842025 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.531446+0000", 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7159999999999992e-06 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539357+0000", 2024-08-29T15:17:42.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079111709999999998 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539518+0000", 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161232 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4", 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.539667+0000", 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000148508 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.543701+0000", 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040340940000000002 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.554854+0000", 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011152746 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.554980+0000", 2024-08-29T15:17:42.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012606800000000001 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:53.555094+0000", 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114533 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124414 6.c 6:305f40c0:::benchmark_data_smithi027_169445_object124413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.043834+0000", 2024-08-29T15:17:42.718 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.592254505, 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.142964321, 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124414 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.043834+0000", 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.043834+0000", 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.043836+0000", 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9690000000000001e-06 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.043861+0000", 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5236e-05 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.043874+0000", 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2310000000000001e-05 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.043881+0000", 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9e-06 2024-08-29T15:17:42.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.044498+0000", 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00061757200000000004 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.044508+0000", 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5519999999999993e-06 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.028579+0000", 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98407099600000003 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.028591+0000", 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2631e-05 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.175316+0000", 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1467245699999999 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.175432+0000", 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116493 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6,7", 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.175519+0000", 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6728e-05 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.180885+0000", 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005365501 2024-08-29T15:17:42.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.181129+0000", 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000243983 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.181205+0000", 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6037000000000004e-05 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.186592+0000", 2024-08-29T15:17:42.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053876710000000001 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.186665+0000", 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2627999999999996e-05 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.186798+0000", 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013354400000000001 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.726 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124419 6.c 6:3d18f905:::benchmark_data_smithi027_169445_object124418:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.047867+0000", 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.588221290999996, 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1392461819999999, 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.727 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124419 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.047867+0000", 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.047867+0000", 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.047868+0000", 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1399999999999996e-07 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.047893+0000", 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4451999999999999e-05 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.047904+0000", 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0995999999999999e-05 2024-08-29T15:17:42.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.047911+0000", 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3220000000000004e-06 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.047990+0000", 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8724e-05 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.048000+0000", 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0322999999999999e-05 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.028777+0000", 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98077656599999996 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.028786+0000", 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0969999999999995e-06 2024-08-29T15:17:42.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.178409+0000", 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1496229769999999 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.178560+0000", 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015121899999999999 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6,7", 2024-08-29T15:17:42.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.178737+0000", 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017699600000000001 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.181224+0000", 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002486837 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.185681+0000", 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044573160000000002 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.186862+0000", 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001180649 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.186917+0000", 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5621999999999997e-05 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.186985+0000", 2024-08-29T15:17:42.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7895e-05 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.187114+0000", 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012837699999999999 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124430 6.c 6:3cd61ad1:::benchmark_data_smithi027_169445_object124429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:17:42.735 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.061485+0000", 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.574603668000002, 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1258973389999998, 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124430 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061485+0000", 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061485+0000", 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061486+0000", 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1799999999999995e-07 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061510+0000", 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4389e-05 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061520+0000", 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6579999999999997e-06 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061526+0000", 2024-08-29T15:17:42.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5520000000000001e-06 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061718+0000", 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019212600000000001 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.061730+0000", 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1779e-05 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.028813+0000", 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96708323900000004 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.028828+0000", 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4783000000000001e-05 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.181252+0000", 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1524242760000001 2024-08-29T15:17:42.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.181400+0000", 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014809400000000001 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6,7", 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.181569+0000", 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000168818 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.183793+0000", 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002223254 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.185764+0000", 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019718330000000001 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.187153+0000", 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013890790000000001 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.187215+0000", 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1821000000000003e-05 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.187275+0000", 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9358000000000003e-05 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.187382+0000", 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107662 2024-08-29T15:17:42.743 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140306 6.c 6:3c01f92e:::benchmark_data_smithi027_169445_object14879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.179791+0000", 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 21.456297712000001, 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.61662402900000002, 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.744 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140306 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179791+0000", 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179791+0000", 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179792+0000", 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1209999999999999e-06 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179796+0000", 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4240000000000003e-06 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179807+0000", 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0633e-05 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179814+0000", 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8700000000000003e-06 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179992+0000", 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000177772 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.180000+0000", 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2230000000000007e-06 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.786990+0000", 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.60699031199999998 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.787132+0000", 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014150299999999999 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.787293+0000", 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161225 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.791544+0000", 2024-08-29T15:17:42.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042511720000000001 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.795771+0000", 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042268100000000001 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.795976+0000", 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020485999999999999 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.796035+0000", 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9426000000000003e-05 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.796415+0000", 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00037967799999999999 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140307 6.7 6:e5668b1d:::benchmark_data_smithi027_169445_object14880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:42.751 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.182617+0000", 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 21.453471519000001, 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.62514394399999995, 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140307 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.182617+0000", 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.182617+0000", 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.182619+0000", 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.697e-06 2024-08-29T15:17:42.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.182625+0000", 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.173e-06 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.182643+0000", 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7941999999999999e-05 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.182652+0000", 2024-08-29T15:17:42.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3899999999999999e-06 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183436+0000", 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00078331799999999997 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183451+0000", 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5519000000000002e-05 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.787799+0000", 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.60434783299999995 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.791044+0000", 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032451770000000001 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4", 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.791493+0000", 2024-08-29T15:17:42.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00044858099999999998 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.803334+0000", 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011840918000000001 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.807451+0000", 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041177610000000002 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.807516+0000", 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4117999999999998e-05 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.807761+0000", 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000245517 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140310 6.7 6:e4fdedea:::benchmark_data_smithi027_169445_object14883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:42.758 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.183674+0000", 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 21.452414227999999, 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.62482996199999996, 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140310 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183674+0000", 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183674+0000", 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183675+0000", 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6199999999999997e-07 2024-08-29T15:17:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183679+0000", 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4999999999999999e-06 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183689+0000", 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0115000000000001e-05 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183696+0000", 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.046e-06 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183748+0000", 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2373000000000002e-05 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.183755+0000", 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3039999999999999e-06 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.803785+0000", 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.62002964599999999 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.803931+0000", 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014615099999999999 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4", 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.804069+0000", 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013736799999999999 2024-08-29T15:17:42.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.808253+0000", 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041849460000000002 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.808385+0000", 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131626 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.808427+0000", 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2188999999999999e-05 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.808504+0000", 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7035999999999999e-05 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140320 6.c 6:388b8e28:::benchmark_data_smithi027_169445_object14893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:17:42.765 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.197539+0000", 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 21.438549598000002, 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59920827899999995, 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140320 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197539+0000", 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197539+0000", 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197540+0000", 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.215e-06 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197545+0000", 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2089999999999996e-06 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197557+0000", 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1388e-05 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197564+0000", 2024-08-29T15:17:42.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.973e-06 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197627+0000", 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3108000000000006e-05 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197635+0000", 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2339999999999994e-06 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.791737+0000", 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.594101977 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.791882+0000", 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014510799999999999 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.792014+0000", 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013211700000000001 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.795907+0000", 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038927409999999999 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.796426+0000", 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00051873999999999996 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.796480+0000", 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4211e-05 2024-08-29T15:17:42.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.796646+0000", 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000165685 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.796747+0000", 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101573 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:141715 6.c 6:357532fc:::benchmark_data_smithi027_169445_object16288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:39.743999+0000", 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 2.89208928, 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9966774539999999, 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:42.773 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 141715 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.743999+0000", 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.743999+0000", 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.744001+0000", 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3519999999999999e-06 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.744007+0000", 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3849999999999999e-06 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.744020+0000", 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2847000000000001e-05 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.744028+0000", 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8960000000000003e-06 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.744100+0000", 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1884999999999994e-05 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:39.744108+0000", 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7180000000000002e-06 2024-08-29T15:17:42.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:40.708251+0000", 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96414262500000003 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:40.708263+0000", 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2255e-05 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:17:42.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.736267+0000", 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.028003824 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.736407+0000", 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000140216 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6,7", 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.736497+0000", 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9843e-05 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.739783+0000", 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032856209999999999 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.739797+0000", 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3787e-05 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.739831+0000", 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3963e-05 2024-08-29T15:17:42.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.740573+0000", 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00074207500000000003 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.740610+0000", 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7342000000000003e-05 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:41.740677+0000", 2024-08-29T15:17:42.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6820000000000001e-05 2024-08-29T15:17:42.781 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.781 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.781 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.781 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:17:42.781 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:17:42.781 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:17:42.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:42.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:42.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:42.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:42.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:42.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:42 smithi171 ceph-mon[90031]: 6.c continuing backfill to osd.6 from (176'1500,200'8826] MIN to 200'8826 2024-08-29T15:17:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 102005/327846 objects degraded (31.114%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:43.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:43.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:42 smithi027 ceph-mon[93921]: 6.c continuing backfill to osd.6 from (176'1500,200'8826] MIN to 200'8826 2024-08-29T15:17:43.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 102005/327846 objects degraded (31.114%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:42 smithi027 ceph-mon[98970]: 6.c continuing backfill to osd.6 from (176'1500,200'8826] MIN to 200'8826 2024-08-29T15:17:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 102005/327846 objects degraded (31.114%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:43.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:43.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:43.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:43.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:43.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:43.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:43.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:43.544 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:43.564 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:43.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:43.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:43.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:17:43.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:43.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:43.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:43.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:43.807 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:43 smithi027 ceph-mon[98970]: pgmap v603: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 remapped+peering, 1 active+recovering+undersized+degraded, 2 undersized+degraded+peered, 36 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 14 active+undersized, 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 47 op/s; 88470/327672 objects degraded (27.000%); 21397/327672 objects misplaced (6.530%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:17:43.808 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:43.808 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:43.808 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:43 smithi027 ceph-mon[93921]: pgmap v603: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 remapped+peering, 1 active+recovering+undersized+degraded, 2 undersized+degraded+peered, 36 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 14 active+undersized, 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 47 op/s; 88470/327672 objects degraded (27.000%); 21397/327672 objects misplaced (6.530%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:17:43.808 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:43.808 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:43.816 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:43.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:43.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:43.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:43.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:43.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:43.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:43 smithi171 ceph-mon[90031]: pgmap v603: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 remapped+peering, 1 active+recovering+undersized+degraded, 2 undersized+degraded+peered, 36 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 14 active+undersized, 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 47 op/s; 88470/327672 objects degraded (27.000%); 21397/327672 objects misplaced (6.530%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:17:43.960 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:17:43.961 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:44.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:44.068 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:44.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:44.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:44.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:44.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:44.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:44.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:44.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:44.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:17:44.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:44.444 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:44.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:44.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:44.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:44.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:44.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:44.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:44.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:44.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:44.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:44.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:44.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:45.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:45.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:45.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:45.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:45.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:45.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:45.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:45.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:45.437 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:17:45.437 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:17:45.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:45.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:45.455 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:17:45.456 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:17:45.456 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:17:45.468 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:45.474 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:17:45.474 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:17:45.492 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:17:45.492 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:17:45.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:45.568 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:45.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:45.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:45.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:45.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:45.727 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:45 smithi027 ceph-mon[93921]: pgmap v604: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 38 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 12 active+undersized, 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 81664/327612 objects degraded (24.927%); 21397/327612 objects misplaced (6.531%); 2.9 MiB/s, 46 objects/s recovering 2024-08-29T15:17:45.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:45.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:45.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:45.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:45.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:45.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:45.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:45 smithi171 ceph-mon[90031]: pgmap v604: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 38 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 12 active+undersized, 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 81664/327612 objects degraded (24.927%); 21397/327612 objects misplaced (6.531%); 2.9 MiB/s, 46 objects/s recovering 2024-08-29T15:17:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3054227011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:46.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3054227011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:45 smithi027 ceph-mon[98970]: pgmap v604: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 38 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 12 active+undersized, 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 36 op/s; 81664/327612 objects degraded (24.927%); 21397/327612 objects misplaced (6.531%); 2.9 MiB/s, 46 objects/s recovering 2024-08-29T15:17:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3054227011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:17:46.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:46.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:46.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:46.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:46.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:46.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:46.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:46.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:46.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:46.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:46.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:46.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:46.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:46.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:46.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:46.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:46.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:46.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:46.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:46.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:47.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:47.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:47.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:47.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:47.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:47.318 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:47.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:47.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:47.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:47.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:47.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:47.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:47.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:47.733 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:17:47.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:47.816 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:47.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:47.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:47.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:47.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:47.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:47 smithi171 ceph-mon[90031]: pgmap v605: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 81407/327543 objects degraded (24.854%); 21401/327543 objects misplaced (6.534%); 5.6 MiB/s, 88 objects/s recovering 2024-08-29T15:17:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 81407/327543 objects degraded (24.854%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/574138238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:48.019 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:47 smithi027 ceph-mon[98970]: pgmap v605: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 81407/327543 objects degraded (24.854%); 21401/327543 objects misplaced (6.534%); 5.6 MiB/s, 88 objects/s recovering 2024-08-29T15:17:48.019 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 81407/327543 objects degraded (24.854%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:48.019 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/574138238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:48.019 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:47 smithi027 ceph-mon[93921]: pgmap v605: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 81407/327543 objects degraded (24.854%); 21401/327543 objects misplaced (6.534%); 5.6 MiB/s, 88 objects/s recovering 2024-08-29T15:17:48.019 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 81407/327543 objects degraded (24.854%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:48.019 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/574138238' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:17:48.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:48.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:48.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:48.191 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:48.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:48.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:48.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:48.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:48.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:48.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:48.568 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:48.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:48.670 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:48.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:48.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:48.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:48.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:48.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:48.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:48.949 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:48.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:48.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:49.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:48 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:49.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/574138238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:17:49.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:48 smithi027 ceph-mon[98970]: osdmap e204: 8 total, 7 up, 8 in 2024-08-29T15:17:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:48 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/574138238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:17:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:48 smithi027 ceph-mon[93921]: osdmap e204: 8 total, 7 up, 8 in 2024-08-29T15:17:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:48 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/574138238' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:17:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:48 smithi171 ceph-mon[90031]: osdmap e204: 8 total, 7 up, 8 in 2024-08-29T15:17:49.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:49.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:49.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:49.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:49.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:17:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:17:49] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.51.0" 2024-08-29T15:17:49.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:49.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:49.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:49.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:49.560 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:49.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:49.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:49.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:49.712 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:49.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:49.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:49.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:49.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:49.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:49.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:50.002 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:17:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:49 smithi171 ceph-mon[90031]: pgmap v607: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 81401/327507 objects degraded (24.855%); 21401/327507 objects misplaced (6.535%); 2.7 MiB/s, 42 objects/s recovering 2024-08-29T15:17:50.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:50.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:49 smithi027 ceph-mon[98970]: pgmap v607: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 81401/327507 objects degraded (24.855%); 21401/327507 objects misplaced (6.535%); 2.7 MiB/s, 42 objects/s recovering 2024-08-29T15:17:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:49 smithi027 ceph-mon[93921]: pgmap v607: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 81401/327507 objects degraded (24.855%); 21401/327507 objects misplaced (6.535%); 2.7 MiB/s, 42 objects/s recovering 2024-08-29T15:17:50.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:50.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:50.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:50.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:50.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:50.350 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:50.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:50.452 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:50.478 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:50.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:50.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:17:50.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:50.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:50.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:50.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:50.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:50.731 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:50.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:50.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:50.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:50.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:50.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:51.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:51.104 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:51.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:51.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:51.228 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:51.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:51.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:51.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:51.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:51.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:51.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:51.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:51.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:51.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:51.706 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:51.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:51.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:51.805 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:51 smithi171 ceph-mon[90031]: pgmap v608: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:17:51.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:51.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:51.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:51.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:51.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:51.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141581 6.a 6.bf61826a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730462+0000", 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595399435999999, 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595421278, 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.042 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141581 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730462+0000", 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.043 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730462+0000", 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730460+0000", 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730473+0000", 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3055e-05 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730482+0000", 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0729999999999994e-06 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730489+0000", 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2419999999999997e-06 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730543+0000", 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3949999999999997e-05 2024-08-29T15:17:52.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730552+0000", 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1309999999999995e-06 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705785+0000", 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97523290500000004 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705794+0000", 2024-08-29T15:17:52.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3249999999999997e-06 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141604 6.a 6.7aa7217a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730497+0000", 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595364349, 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595438013999999, 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.047 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141604 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730497+0000", 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730497+0000", 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730496+0000", 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730499+0000", 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1240000000000001e-06 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730506+0000", 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7970000000000003e-06 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730518+0000", 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1399999999999999e-05 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730590+0000", 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2109999999999999e-05 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730598+0000", 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4090000000000003e-06 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705814+0000", 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97521559300000005 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705821+0000", 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.866e-06 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.051 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141608 6.a 6.2812b0fa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730532+0000", 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595328890999999, 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595445501, 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141608 2024-08-29T15:17:52.052 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730532+0000", 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730532+0000", 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.054 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730532+0000", 2024-08-29T15:17:52.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.055 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.055 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.055 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730534+0000", 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.543e-06 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730541+0000", 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2550000000000003e-06 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730553+0000", 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2505e-05 2024-08-29T15:17:52.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730629+0000", 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6194999999999998e-05 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730635+0000", 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7440000000000004e-06 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705841+0000", 2024-08-29T15:17:52.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97520580700000004 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705847+0000", 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3350000000000003e-06 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141610 6.a 6.393a126a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730557+0000", 2024-08-29T15:17:52.058 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.59530406, 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595462251000001, 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141610 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730557+0000", 2024-08-29T15:17:52.059 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730557+0000", 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730557+0000", 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730559+0000", 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7070000000000001e-06 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730565+0000", 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7540000000000003e-06 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730570+0000", 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1259999999999997e-06 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730663+0000", 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2743000000000003e-05 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730670+0000", 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5610000000000004e-06 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705866+0000", 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97519675299999997 2024-08-29T15:17:52.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705873+0000", 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2439999999999998e-06 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141766 6.a 6.f62f64ca (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e203)", 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:42.049944+0000", 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 9.2759166749999995, 2024-08-29T15:17:52.063 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2761148020000004, 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141766 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:42.049944+0000", 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.064 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:42.049944+0000", 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:42.049943+0000", 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.065 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:42.049951+0000", 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5390000000000002e-06 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:42.049962+0000", 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.186e-05 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:42.049970+0000", 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6380000000000004e-06 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141612 6.a 6.c1d1f5ea (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730583+0000", 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595278582000001, 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595505553000001, 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141612 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730583+0000", 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730583+0000", 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730582+0000", 2024-08-29T15:17:52.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730589+0000", 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5889999999999999e-06 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730595+0000", 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.246e-06 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730600+0000", 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5329999999999996e-06 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730698+0000", 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8617999999999997e-05 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730710+0000", 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1807999999999999e-05 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705889+0000", 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97517906300000001 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705894+0000", 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2390000000000002e-06 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141623 6.a 6.7ce8af7a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730603+0000", 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595257870999999, 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595524731999999, 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141623 2024-08-29T15:17:52.072 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730603+0000", 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730603+0000", 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.073 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730603+0000", 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730605+0000", 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.95e-06 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730610+0000", 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4750000000000001e-06 2024-08-29T15:17:52.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730615+0000", 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4360000000000004e-06 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730741+0000", 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000126284 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730747+0000", 2024-08-29T15:17:52.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5380000000000002e-06 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705914+0000", 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97516711199999995 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705920+0000", 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3090000000000001e-06 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.076 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141633 6.a 6.abc6f2da (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730627+0000", 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595233886000001, 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595542263, 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141633 2024-08-29T15:17:52.077 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730627+0000", 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730627+0000", 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730627+0000", 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730630+0000", 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7499999999999999e-06 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730636+0000", 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3890000000000002e-06 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730641+0000", 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8420000000000004e-06 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730777+0000", 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136033 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730783+0000", 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4300000000000003e-06 2024-08-29T15:17:52.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705939+0000", 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97515581200000001 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705945+0000", 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8780000000000003e-06 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.081 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141652 6.a 6.fcaf487a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730660+0000", 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595201333, 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595549397999999, 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141652 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.082 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730660+0000", 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730660+0000", 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730659+0000", 2024-08-29T15:17:52.083 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730668+0000", 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2409999999999995e-06 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730674+0000", 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8189999999999997e-06 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.084 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730678+0000", 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7439999999999998e-06 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730812+0000", 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013413100000000001 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730818+0000", 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5470000000000004e-06 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.085 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705965+0000", 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97514675299999998 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705971+0000", 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9059999999999999e-06 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.086 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141670 6.a 6.db4590fa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730682+0000", 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595179104, 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595567562999999, 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141670 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.087 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730682+0000", 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730682+0000", 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730682+0000", 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.088 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730684+0000", 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2239999999999998e-06 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730689+0000", 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7500000000000003e-06 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730691+0000", 2024-08-29T15:17:52.089 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4959999999999999e-06 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730840+0000", 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014849000000000001 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730843+0000", 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3950000000000001e-06 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.090 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.705994+0000", 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97515056099999997 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706000+0000", 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.144e-06 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141679 6.a 6.7f0062ba (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730694+0000", 2024-08-29T15:17:52.091 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.59516762, 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595595883, 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141679 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730694+0000", 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.092 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730694+0000", 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730693+0000", 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730695+0000", 2024-08-29T15:17:52.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2249999999999999e-06 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730697+0000", 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.886e-06 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730700+0000", 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4540000000000001e-06 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730868+0000", 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016767400000000001 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730873+0000", 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6330000000000001e-06 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706034+0000", 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.975160888 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706040+0000", 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6210000000000001e-06 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141680 6.a 6.782fe13a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730702+0000", 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595159231, 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595633304, 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.096 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141680 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730702+0000", 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730702+0000", 2024-08-29T15:17:52.097 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730702+0000", 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730715+0000", 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2830000000000001e-05 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730721+0000", 2024-08-29T15:17:52.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2149999999999998e-06 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730726+0000", 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0849999999999996e-06 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730894+0000", 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016777099999999999 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730897+0000", 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8959999999999999e-06 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706063+0000", 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97516643400000003 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706069+0000", 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9190000000000004e-06 2024-08-29T15:17:52.100 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141681 6.a 6.4fee163a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730739+0000", 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595122444999999, 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595635953, 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.101 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141681 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730739+0000", 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730739+0000", 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730738+0000", 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730740+0000", 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.052e-06 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730745+0000", 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1359999999999999e-06 2024-08-29T15:17:52.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730747+0000", 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.751e-06 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730920+0000", 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017279200000000001 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730926+0000", 2024-08-29T15:17:52.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4859999999999997e-06 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706092+0000", 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97516602900000005 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706099+0000", 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0450000000000003e-06 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141694 6.a 6.7b13fc8a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730750+0000", 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595111206, 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595681497999999, 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141694 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730750+0000", 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730750+0000", 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730750+0000", 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730751+0000", 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1489999999999999e-06 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730757+0000", 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7640000000000002e-06 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730761+0000", 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2119999999999997e-06 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730946+0000", 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018471500000000001 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730948+0000", 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8619999999999999e-06 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706123+0000", 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97517482899999997 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706130+0000", 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.241e-06 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141697 6.a 6.3e9848aa (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730773+0000", 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595087979000001, 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595703463, 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.110 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141697 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730773+0000", 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730773+0000", 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.111 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730773+0000", 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730775+0000", 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0250000000000001e-06 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730781+0000", 2024-08-29T15:17:52.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7309999999999998e-06 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730794+0000", 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3842999999999999e-05 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730967+0000", 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000172602 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730972+0000", 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.378e-06 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706187+0000", 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97521421500000005 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706193+0000", 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8469999999999998e-06 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141704 6.a 6.67bf98a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e201)", 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:39.730810+0000", 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 11.595051414, 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 11.595708302, 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for peered", 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141704 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:17:52.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730810+0000", 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730810+0000", 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730809+0000", 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:17:52.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730812+0000", 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3099999999999999e-06 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730816+0000", 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3769999999999998e-06 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730819+0000", 2024-08-29T15:17:52.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5679999999999998e-06 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730990+0000", 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017139299999999999 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.730993+0000", 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4010000000000001e-06 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706226+0000", 2024-08-29T15:17:52.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97523212199999998 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:40.706232+0000", 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2400000000000004e-06 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 16 2024-08-29T15:17:52.119 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:17:52.120 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:52.120 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:51 smithi027 ceph-mon[93921]: pgmap v608: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:17:52.121 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:51 smithi027 ceph-mon[98970]: pgmap v608: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 10 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:17:52.144 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:52.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:52.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:52.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:52.268 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:52.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:52.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:52.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:52.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:52.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:52.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:52.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:52.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:52.618 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:52.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:52.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:52.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:52.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:52.747 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:17:52.767 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:52.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:52.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:52.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:52.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:52.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:52.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 81069/327507 objects degraded (24.753%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:52.924 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4077803240' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:52.944 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:17:52.944 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:17:52.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:53.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:53.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:53.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 81069/327507 objects degraded (24.753%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:53.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4077803240' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:53.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 81069/327507 objects degraded (24.753%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:53.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4077803240' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:17:53.120 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:53.144 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:53.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:53.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:53.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:53.373 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:53.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:53.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:53.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:53.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:53.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:53.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:17:53.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:53.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:53.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:53.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:53.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:53.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:53.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:53.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:53.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:53.920 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:17:54.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:53 smithi171 ceph-mon[90031]: pgmap v609: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 4.2 MiB/s, 66 objects/s recovering 2024-08-29T15:17:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:53 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:17:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4077803240' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:53 smithi171 ceph-mon[90031]: osdmap e205: 8 total, 7 up, 8 in 2024-08-29T15:17:54.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[98970]: pgmap v609: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 4.2 MiB/s, 66 objects/s recovering 2024-08-29T15:17:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:17:54.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4077803240' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:54.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[98970]: osdmap e205: 8 total, 7 up, 8 in 2024-08-29T15:17:54.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[93921]: pgmap v609: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 4.2 MiB/s, 66 objects/s recovering 2024-08-29T15:17:54.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:17:54.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4077803240' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:17:54.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:53 smithi027 ceph-mon[93921]: osdmap e205: 8 total, 7 up, 8 in 2024-08-29T15:17:54.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:54.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:54.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:54.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:54.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:54.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:54.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:54.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:54.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:54.652 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:54.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:54.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:54.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:54.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:54.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:54.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:55.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:55.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:55.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:55.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:55.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:55.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:55.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:55.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:55.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:55.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:17:55.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:55.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:55.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:55.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:55.659 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:55.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:55.677 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:55.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:55.783 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:55.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:55.907 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:55.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:56.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:56.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:55 smithi171 ceph-mon[90031]: pgmap v611: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 2.6 MiB/s, 41 objects/s recovering 2024-08-29T15:17:56.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:55 smithi027 ceph-mon[98970]: pgmap v611: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 2.6 MiB/s, 41 objects/s recovering 2024-08-29T15:17:56.031 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:55 smithi027 ceph-mon[93921]: pgmap v611: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 81069/327507 objects degraded (24.753%); 21401/327507 objects misplaced (6.535%); 2.6 MiB/s, 41 objects/s recovering 2024-08-29T15:17:56.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:56.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:56.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:56.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:56.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:56.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:56.283 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:56.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:56.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:56.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:56.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:56.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:56.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:56.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:56.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:56.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:56.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:56.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:56.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:56.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:57.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:57.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:57.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:57.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:57.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:57.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:57.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:57.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:57.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:57.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:57.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:57.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:57.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:57.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:57.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:57.682 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 6, 7, 5] 2024-08-29T15:17:57.682 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:17:57.682 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.18726715803844518 2024-08-29T15:17:57.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 4 0.18726715803844518 2024-08-29T15:17:57.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:57.773 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:57.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:57 smithi027 ceph-mon[98970]: pgmap v612: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80739/327507 objects degraded (24.653%); 21401/327507 objects misplaced (6.535%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:17:57.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 80739/327507 objects degraded (24.653%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:57.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/686689721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:57.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:57 smithi027 ceph-mon[93921]: pgmap v612: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80739/327507 objects degraded (24.653%); 21401/327507 objects misplaced (6.535%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:17:57.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 80739/327507 objects degraded (24.653%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:57.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/686689721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:57.847 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:17:57.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:57.897 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:57.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:57.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:17:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:57 smithi171 ceph-mon[90031]: pgmap v612: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80739/327507 objects degraded (24.653%); 21401/327507 objects misplaced (6.535%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:17:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 80739/327507 objects degraded (24.653%), 20 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2024-08-29T15:17:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/686689721' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:17:58.020 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:58.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:17:58.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:58.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:58.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:58.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:58.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:58.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:58.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:58.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:17:58.520 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:17:58.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:58.632 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:17:58.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:58.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:58.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:17:58.772 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:58.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:17:58.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:58 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:58.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/686689721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:17:58.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:58 smithi171 ceph-mon[90031]: osdmap e206: 8 total, 7 up, 8 in 2024-08-29T15:17:58.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/686689721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:17:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[93921]: osdmap e206: 8 total, 7 up, 8 in 2024-08-29T15:17:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:17:58.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/686689721' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:17:58.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[98970]: osdmap e206: 8 total, 7 up, 8 in 2024-08-29T15:17:58.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:17:58.824 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:17:58.824 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:17:58.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:58.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:58.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:58.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:59.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:59.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:59.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:59.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:59.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:59.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:17:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:17:59] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.51.0" 2024-08-29T15:17:59.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:17:59.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:17:59.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:17:59.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:17:59.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:17:59.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:17:59.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:59.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:17:59.747 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:17:59.752 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:59.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:59.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:59.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:17:59.944 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:17:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:17:59.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:17:59.995 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:17:59.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:17:59 smithi027 ceph-mon[98970]: pgmap v614: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80739/327507 objects degraded (24.653%); 21401/327507 objects misplaced (6.535%); 2.6 MiB/s, 41 objects/s recovering 2024-08-29T15:17:59.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:17:59 smithi027 ceph-mon[93921]: pgmap v614: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80739/327507 objects degraded (24.653%); 21401/327507 objects misplaced (6.535%); 2.6 MiB/s, 41 objects/s recovering 2024-08-29T15:18:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:17:59 smithi171 ceph-mon[90031]: pgmap v614: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80739/327507 objects degraded (24.653%); 21401/327507 objects misplaced (6.535%); 2.6 MiB/s, 41 objects/s recovering 2024-08-29T15:18:00.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:00.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:00.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:00.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:00.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:00.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:00.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:00.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:00.455 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:00.478 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:00.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:00.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:18:00.589 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:18:00.589 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:18:00.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:00.605 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:18:00.605 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:18:00.623 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:18:00.623 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:18:00.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:00.728 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:00.830 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:00.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:00.857 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.4 to 0.187267 (2ff0) 2024-08-29T15:18:00.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:00.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:00.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:01.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:01.103 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:01.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:01.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:01.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2339402454' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 0.18726715803844518} : dispatch 2024-08-29T15:18:01.286 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2339402454' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 0.18726715803844518} : dispatch 2024-08-29T15:18:01.287 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2339402454' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 0.18726715803844518} : dispatch 2024-08-29T15:18:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:18:01.306 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:18:01.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:01.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:01.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:01.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:01.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:01.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:01.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:01.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:01.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:01.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:01.851 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:01.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:01.899 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[98970]: pgmap v615: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80401/327507 objects degraded (24.549%); 21401/327507 objects misplaced (6.535%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:18:01.899 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2339402454' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.18726715803844518}]': finished 2024-08-29T15:18:01.899 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[98970]: osdmap e207: 8 total, 7 up, 8 in 2024-08-29T15:18:01.899 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 80401/327507 objects degraded (24.549%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:01.900 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[98970]: osdmap e208: 8 total, 7 up, 8 in 2024-08-29T15:18:01.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[93921]: pgmap v615: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80401/327507 objects degraded (24.549%); 21401/327507 objects misplaced (6.535%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:18:01.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2339402454' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.18726715803844518}]': finished 2024-08-29T15:18:01.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[93921]: osdmap e207: 8 total, 7 up, 8 in 2024-08-29T15:18:01.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 80401/327507 objects degraded (24.549%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:01.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:01 smithi027 ceph-mon[93921]: osdmap e208: 8 total, 7 up, 8 in 2024-08-29T15:18:01.919 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:18:01.919 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:18:01.920 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:01 smithi171 ceph-mon[90031]: pgmap v615: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded, 1 undersized+degraded+peered, 42 active+clean, 2 undersized+peered, 12 active+undersized+degraded, 13 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 80401/327507 objects degraded (24.549%); 21401/327507 objects misplaced (6.535%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:18:01.920 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2339402454' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.18726715803844518}]': finished 2024-08-29T15:18:01.920 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:01 smithi171 ceph-mon[90031]: osdmap e207: 8 total, 7 up, 8 in 2024-08-29T15:18:01.920 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:01 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 80401/327507 objects degraded (24.549%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:01.920 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:01 smithi171 ceph-mon[90031]: osdmap e208: 8 total, 7 up, 8 in 2024-08-29T15:18:01.953 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:01.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:01.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:02.010 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:02.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:02.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:02.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:02.227 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:02.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:02.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.352 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:02.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:02.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:02.475 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:02.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:02.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:02.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:02.700 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:02.715 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:18:02.723 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:02.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:02.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:02.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:02.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:02.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:02.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:02.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:02.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:03.021 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:03.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:03.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:03.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:03.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:03.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:03.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:03.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:03.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:18:03.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:03.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:03.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:03.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:03.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:03.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:03.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:03.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:03.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:03.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:03.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:03.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:03.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:03.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:03.761 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:03.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:03.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:03.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:03.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:03.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:04.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:04.043 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[98970]: pgmap v618: 89 pgs: 1 active+recovery_wait+degraded+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 47 active+clean, 6 active+undersized+degraded, 11 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 54346/327381 objects degraded (16.600%); 21401/327381 objects misplaced (6.537%); 3.8 MiB/s, 65 objects/s recovering 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[98970]: osdmap e209: 8 total, 7 up, 8 in 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[98970]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 203'8863 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[98970]: 6.2 starting backfill to osd.6 from (0'0,0'0] MAX to 203'8765 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[98970]: 6.f starting backfill to osd.6 from (0'0,0'0] MAX to 203'8934 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[98970]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 203'8909 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[98970]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[93921]: pgmap v618: 89 pgs: 1 active+recovery_wait+degraded+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 47 active+clean, 6 active+undersized+degraded, 11 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 54346/327381 objects degraded (16.600%); 21401/327381 objects misplaced (6.537%); 3.8 MiB/s, 65 objects/s recovering 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[93921]: osdmap e209: 8 total, 7 up, 8 in 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[93921]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 203'8863 2024-08-29T15:18:04.044 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[93921]: 6.2 starting backfill to osd.6 from (0'0,0'0] MAX to 203'8765 2024-08-29T15:18:04.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[93921]: 6.f starting backfill to osd.6 from (0'0,0'0] MAX to 203'8934 2024-08-29T15:18:04.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[93921]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 203'8909 2024-08-29T15:18:04.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:03 smithi027 ceph-mon[93921]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:18:04.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:04.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:04.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:04.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:03 smithi171 ceph-mon[90031]: pgmap v618: 89 pgs: 1 active+recovery_wait+degraded+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 47 active+clean, 6 active+undersized+degraded, 11 active+undersized, 10 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 54346/327381 objects degraded (16.600%); 21401/327381 objects misplaced (6.537%); 3.8 MiB/s, 65 objects/s recovering 2024-08-29T15:18:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:03 smithi171 ceph-mon[90031]: osdmap e209: 8 total, 7 up, 8 in 2024-08-29T15:18:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:03 smithi171 ceph-mon[90031]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 203'8863 2024-08-29T15:18:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:03 smithi171 ceph-mon[90031]: 6.2 starting backfill to osd.6 from (0'0,0'0] MAX to 203'8765 2024-08-29T15:18:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:03 smithi171 ceph-mon[90031]: 6.f starting backfill to osd.6 from (0'0,0'0] MAX to 203'8934 2024-08-29T15:18:04.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:03 smithi171 ceph-mon[90031]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 203'8909 2024-08-29T15:18:04.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:03 smithi171 ceph-mon[90031]: Health check update: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-29T15:18:04.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:04.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:04.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:04.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:04.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:04.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:04.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:04.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:04.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:04.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:04.776 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:18:04.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:04.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:04.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:04.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:04.931 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1359089430' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:04.931 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:04 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:04.947 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:18:04.947 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:18:04.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:05.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1359089430' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:05.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:04 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1359089430' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:05.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:04 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:05.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:05.140 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:05.242 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:05.263 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:05.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:05.365 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:05.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:05.490 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:05.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:05.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:05.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:05.638 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:05.740 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:05.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:05.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:05 smithi171 ceph-mon[90031]: pgmap v620: 89 pgs: 1 active+recovery_wait+degraded+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 48 active+clean, 6 active+undersized+degraded, 11 active+undersized, 9 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 54289/327135 objects degraded (16.595%); 21401/327135 objects misplaced (6.542%); 3.8 MiB/s, 65 objects/s recovering 2024-08-29T15:18:05.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:05 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:05.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:05.812 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:05 smithi171 ceph-mon[90031]: osdmap e210: 8 total, 7 up, 8 in 2024-08-29T15:18:05.838 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:18:05.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:05.879 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:18:05.879 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:18:05.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:05.888 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:05.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:05.896 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:18:05.897 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:18:05.897 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:18:05.913 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:18:05.913 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:18:05.930 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:18:05.930 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:18:05.949 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:18:05.949 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:18:05.967 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:18:05.967 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:18:05.983 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:18:05.984 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:18:05.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:06.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:06.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:06.058 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[93921]: pgmap v620: 89 pgs: 1 active+recovery_wait+degraded+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 48 active+clean, 6 active+undersized+degraded, 11 active+undersized, 9 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 54289/327135 objects degraded (16.595%); 21401/327135 objects misplaced (6.542%); 3.8 MiB/s, 65 objects/s recovering 2024-08-29T15:18:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[93921]: osdmap e210: 8 total, 7 up, 8 in 2024-08-29T15:18:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[98970]: pgmap v620: 89 pgs: 1 active+recovery_wait+degraded+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 48 active+clean, 6 active+undersized+degraded, 11 active+undersized, 9 active+clean+remapped; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 54289/327135 objects degraded (16.595%); 21401/327135 objects misplaced (6.542%); 3.8 MiB/s, 65 objects/s recovering 2024-08-29T15:18:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:05 smithi027 ceph-mon[98970]: osdmap e210: 8 total, 7 up, 8 in 2024-08-29T15:18:06.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:06.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:06.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:06.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:06.263 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:06.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:06.386 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:06.488 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:06.511 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:06.613 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:06.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:06.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:06.759 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:06.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:06.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:06.882 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:06.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:06.920 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 6, 7, 5] 2024-08-29T15:18:06.920 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:18:06.920 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-29T15:18:06.920 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.056162 2024-08-29T15:18:06.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 7 0.0561620354469462 2024-08-29T15:18:06.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:07.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:07.029 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2282753760' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:07.029 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:06 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 54289/327135 objects degraded (16.595%), 14 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:07.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2282753760' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:07.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:06 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 54289/327135 objects degraded (16.595%), 14 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:18:07.050 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:18:07.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:07.133 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:07.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:07.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:07.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2282753760' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:06 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 54289/327135 objects degraded (16.595%), 14 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:07.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:07.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:07.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:07.512 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:07.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:07.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:07.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:07.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:07.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:07.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:07 smithi027 ceph-mon[93921]: pgmap v622: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 5 remapped+peering, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 52873/326106 objects degraded (16.213%); 28203/326106 objects misplaced (8.648%); 1.7 MiB/s, 39 objects/s recovering 2024-08-29T15:18:07.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:07 smithi027 ceph-mon[98970]: pgmap v622: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 5 remapped+peering, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 52873/326106 objects degraded (16.213%); 28203/326106 objects misplaced (8.648%); 1.7 MiB/s, 39 objects/s recovering 2024-08-29T15:18:07.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:07.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:07.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:07.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:07.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:07 smithi171 ceph-mon[90031]: pgmap v622: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 5 remapped+peering, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 52873/326106 objects degraded (16.213%); 28203/326106 objects misplaced (8.648%); 1.7 MiB/s, 39 objects/s recovering 2024-08-29T15:18:07.972 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:18:07.972 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:18:07.972 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:18:07.972 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:18:07.972 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:18:07.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:08.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:08.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:08.111 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:08.133 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:08.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:08.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:08.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.258 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:08.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:08.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:18:08.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:08.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:08.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:08.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:08.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:08.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:08.631 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:08.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:18:08.732 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:08.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:08.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:08.877 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:08.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:08.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:08.926 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:08 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:18:08.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:08 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:18:08.951 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:09.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:09.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:08 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-29T15:18:09.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:09.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:18:09.060 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:09.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.156 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:09.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:09.182 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:18:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:18:09] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.51.0" 2024-08-29T15:18:09.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:09.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:09.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:09.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:09.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:09.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:09.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:09.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:09.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:09.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:09.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:09.838 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.7 primary-affinity to 0.056162 (836802) 2024-08-29T15:18:09.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:09.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:09.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:10.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:09 smithi171 ceph-mon[90031]: pgmap v623: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 66344/325296 objects degraded (20.395%); 48404/325296 objects misplaced (14.880%); 1.4 MiB/s, 33 objects/s recovering 2024-08-29T15:18:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1918785383' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462} : dispatch 2024-08-29T15:18:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462} : dispatch 2024-08-29T15:18:10.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:09 smithi027 ceph-mon[93921]: pgmap v623: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 66344/325296 objects degraded (20.395%); 48404/325296 objects misplaced (14.880%); 1.4 MiB/s, 33 objects/s recovering 2024-08-29T15:18:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1918785383' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462} : dispatch 2024-08-29T15:18:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462} : dispatch 2024-08-29T15:18:10.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:09 smithi027 ceph-mon[98970]: pgmap v623: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 6 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 66344/325296 objects degraded (20.395%); 48404/325296 objects misplaced (14.880%); 1.4 MiB/s, 33 objects/s recovering 2024-08-29T15:18:10.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1918785383' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462} : dispatch 2024-08-29T15:18:10.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462} : dispatch 2024-08-29T15:18:10.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:10.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:10.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:10.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:10.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:10.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:10.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:10.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:10.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:10.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:10.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:10.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:10.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:10.711 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:10.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:10.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:10.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:10.822 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:18:10.822 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239386 5.2 5:4b9b9b8e:::benchmark_data_smithi027_135490_object105634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.721669+0000", 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 245.09777417199999, 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2226445109999999, 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:18:10.823 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239386 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721669+0000", 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.824 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721669+0000", 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721667+0000", 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721675+0000", 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1810000000000001e-06 2024-08-29T15:18:10.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721694+0000", 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9380000000000001e-05 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721702+0000", 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8679999999999999e-06 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721761+0000", 2024-08-29T15:18:10.826 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8610999999999998e-05 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721866+0000", 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010515500000000001 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.721970+0000", 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000103955 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728347+0000", 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0063774390000000004 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.744581+0000", 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016233282000000002 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751292+0000", 2024-08-29T15:18:10.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067116160000000001 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.751309+0000", 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7039000000000001e-05 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.694440+0000", 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94313092200000004 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.694453+0000", 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2892e-05 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944216+0000", 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2497627339999999 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944313+0000", 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7345999999999999e-05 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.830 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140312 6.a 6:51f4a16d:::benchmark_data_smithi027_169445_object14885:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.774102+0000", 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.045340357000001, 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9911050069999998, 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140312 2024-08-29T15:18:10.831 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774102+0000", 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774102+0000", 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774102+0000", 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774107+0000", 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1549999999999998e-06 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774117+0000", 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0028e-05 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774124+0000", 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9110000000000004e-06 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774272+0000", 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000147718 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774277+0000", 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3510000000000001e-06 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:22.984215+0000", 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2099375379999999 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.003502+0000", 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019287878000000001 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.224173+0000", 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.22067041900000001 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.240203+0000", 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016029700000000001 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.626322+0000", 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3861194649999999 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.635549+0000", 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0092270570000000003 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.651810+0000", 2024-08-29T15:18:10.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016261191000000001 2024-08-29T15:18:10.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.729055+0000", 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.077244853000000002 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.735570+0000", 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0065148719999999997 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.741780+0000", 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0062104559999999996 2024-08-29T15:18:10.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.765207+0000", 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023427011000000001 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140319 6.a 6:57e3975a:::benchmark_data_smithi027_169445_object14892:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.774128+0000", 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.045314322999999, 2024-08-29T15:18:10.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.022320321, 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140319 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774128+0000", 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774128+0000", 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774128+0000", 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774130+0000", 2024-08-29T15:18:10.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3470000000000001e-06 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774136+0000", 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2480000000000004e-06 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774140+0000", 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4039999999999997e-06 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774294+0000", 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015350699999999999 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774299+0000", 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0660000000000003e-06 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.005299+0000", 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.231000002 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.020248+0000", 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014949471000000001 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.200030+0000", 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17978234100000001 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.222269+0000", 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022238478999999998 2024-08-29T15:18:10.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.686887+0000", 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.464617751 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.696688+0000", 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0098014820000000002 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.713649+0000", 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016960952000000001 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.734249+0000", 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020600040999999999 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.766873+0000", 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032624122999999998 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.773957+0000", 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070838710000000003 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.796449+0000", 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022491629999999999 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140323 6.a 6:5c676f7b:::benchmark_data_smithi027_169445_object14896:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.774144+0000", 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.045299090999997, 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1477737129999999, 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140323 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774144+0000", 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774144+0000", 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774143+0000", 2024-08-29T15:18:10.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774145+0000", 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.748e-06 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774151+0000", 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3990000000000003e-06 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774155+0000", 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7199999999999997e-06 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774316+0000", 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016105500000000001 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774321+0000", 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9119999999999997e-06 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.241233+0000", 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.466912193 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.258218+0000", 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016985031000000001 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.334286+0000", 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.076067873999999994 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.356753+0000", 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022467006000000001 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.826358+0000", 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.469604712 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.839840+0000", 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013481854999999999 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.860076+0000", 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020235690000000001 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.891480+0000", 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031404316000000002 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.892665+0000", 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0011854490000000001 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.899889+0000", 2024-08-29T15:18:10.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072232599999999996 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.921917+0000", 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022028790999999999 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140331 6.a 6:57284f37:::benchmark_data_smithi027_169445_object14904:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.857 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.774159+0000", 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.045284154000001, 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3297802810000001, 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140331 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774159+0000", 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774159+0000", 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774158+0000", 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774164+0000", 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.037e-06 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774170+0000", 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2820000000000004e-06 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774174+0000", 2024-08-29T15:18:10.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.048e-06 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774339+0000", 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016554499999999999 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774344+0000", 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0950000000000003e-06 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.263258+0000", 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.488913283 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.279072+0000", 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015814215999999999 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.411993+0000", 2024-08-29T15:18:10.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.132921242 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.428123+0000", 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016129496 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.974675+0000", 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5465523079999999 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.985413+0000", 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010737976 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.003084+0000", 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017671540999999999 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.066945+0000", 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.063860613999999996 2024-08-29T15:18:10.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.075039+0000", 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.008093533 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.081553+0000", 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0065143220000000003 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.103939+0000", 2024-08-29T15:18:10.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022386087999999998 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140333 6.a 6:557fe0ee:::benchmark_data_smithi027_169445_object14906:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.774177+0000", 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.045265714999999, 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5134638119999999, 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.866 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140333 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774177+0000", 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774177+0000", 2024-08-29T15:18:10.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774177+0000", 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774178+0000", 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6530000000000001e-06 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774183+0000", 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.527e-06 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774187+0000", 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8480000000000003e-06 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774361+0000", 2024-08-29T15:18:10.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017404000000000001 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774366+0000", 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4879999999999998e-06 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.433222+0000", 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.658855628 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.449662+0000", 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016440231 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.521495+0000", 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.071833051999999994 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.536518+0000", 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015022353 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.153369+0000", 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.616851032 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.162336+0000", 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0089677030000000005 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.181226+0000", 2024-08-29T15:18:10.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018889388 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.252392+0000", 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.071165939999999997 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.255612+0000", 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032201220000000002 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.262161+0000", 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0065490970000000003 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.287641+0000", 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025480027999999998 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.874 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140336 6.a 6:57969e77:::benchmark_data_smithi027_169445_object14909:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.774190+0000", 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.045253027999998, 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6797074059999999, 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140336 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774190+0000", 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774190+0000", 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774189+0000", 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774191+0000", 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6190000000000001e-06 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774196+0000", 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8420000000000004e-06 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774200+0000", 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1069999999999998e-06 2024-08-29T15:18:10.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774384+0000", 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018384099999999999 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.774389+0000", 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8389999999999997e-06 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.456875+0000", 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6824864210000001 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.472068+0000", 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015192959000000001 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.582390+0000", 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11032194300000001 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.599489+0000", 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017099224 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.327920+0000", 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7284305259999999 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.338730+0000", 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010810118000000001 2024-08-29T15:18:10.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.356115+0000", 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017385266999999999 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.424073+0000", 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067957711000000004 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.425335+0000", 2024-08-29T15:18:10.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012622880000000001 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.432044+0000", 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067086929999999999 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.453897+0000", 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021853299999999999 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140353 6.a 6:5e588f8a:::benchmark_data_smithi027_169445_object14926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.815310+0000", 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.004132466999998, 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8329529349999998, 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140353 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815310+0000", 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815310+0000", 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815308+0000", 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815316+0000", 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4080000000000001e-06 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815331+0000", 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5095e-05 2024-08-29T15:18:10.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815339+0000", 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4370000000000007e-06 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815388+0000", 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8909999999999998e-05 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.815399+0000", 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0462000000000001e-05 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.603626+0000", 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.788227207 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.625928+0000", 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022302191999999998 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.859396+0000", 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.23346792499999999 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.873897+0000", 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014501366 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.505578+0000", 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6316810159999999 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.522841+0000", 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017262551000000001 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.543776+0000", 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020935496000000001 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.615990+0000", 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072213553 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.617470+0000", 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014799990000000001 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.624209+0000", 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067388739999999997 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.648263+0000", 2024-08-29T15:18:10.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024054379000000001 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140365 6.a 6:5e338404:::benchmark_data_smithi027_169445_object14938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.825343+0000", 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 48.994099724000002, 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.990244943, 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.891 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140365 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825343+0000", 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825343+0000", 2024-08-29T15:18:10.892 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825342+0000", 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825350+0000", 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4880000000000007e-06 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825361+0000", 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0541e-05 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825367+0000", 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1659999999999999e-06 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825420+0000", 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.342e-05 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.825430+0000", 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2860000000000005e-06 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.628146+0000", 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.802716897 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.648527+0000", 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020380459 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.896 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.843080+0000", 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19455330900000001 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.857733+0000", 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014652909 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.692859+0000", 2024-08-29T15:18:10.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8351262880000001 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.701365+0000", 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0085051780000000004 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.717363+0000", 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015998089 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.784943+0000", 2024-08-29T15:18:10.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067580464000000007 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.786299+0000", 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013555710000000001 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.793151+0000", 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0068523639999999997 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.815588+0000", 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.02243699 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140375 6.a 6:5a13e021:::benchmark_data_smithi027_169445_object14948:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.842638+0000", 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 48.976804454000003, 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.148323489, 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.901 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140375 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842638+0000", 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.902 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842638+0000", 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842637+0000", 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842644+0000", 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8179999999999998e-06 2024-08-29T15:18:10.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842655+0000", 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1362999999999999e-05 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842663+0000", 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2470000000000008e-06 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842725+0000", 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1493000000000002e-05 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.842735+0000", 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9669999999999996e-06 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.650760+0000", 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8080255489999999 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.670297+0000", 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019536365 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.828496+0000", 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15819894100000001 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.841249+0000", 2024-08-29T15:18:10.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012753469 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.860996+0000", 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0197467630000001 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.876465+0000", 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015469158 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.898840+0000", 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022374578999999999 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.960992+0000", 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062152111000000003 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.964795+0000", 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038032740000000002 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.908 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.969441+0000", 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046456839999999998 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:25.990962+0000", 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021521115 2024-08-29T15:18:10.909 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140392 6.a 6:574fe876:::benchmark_data_smithi027_169445_object14965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.873601+0000", 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 48.945841438000002, 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3267129100000004, 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.910 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140392 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873601+0000", 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873601+0000", 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873600+0000", 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873608+0000", 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1599999999999998e-06 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873623+0000", 2024-08-29T15:18:10.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5145e-05 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873631+0000", 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8439999999999998e-06 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873675+0000", 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4229e-05 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.873686+0000", 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0691e-05 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.880740+0000", 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0070545480000002 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.896130+0000", 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015389415 2024-08-29T15:18:10.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.975083+0000", 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078953089000000004 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.992282+0000", 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017199196 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.050047+0000", 2024-08-29T15:18:10.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0577650539999999 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.064561+0000", 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014514134 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.089870+0000", 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025308824000000001 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.163669+0000", 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.073798701999999994 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.166837+0000", 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031678919999999998 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.171927+0000", 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050905389999999998 2024-08-29T15:18:10.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.200314+0000", 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028387040999999998 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:140406 6.a 6:54ac6f92:::benchmark_data_smithi027_169445_object14979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:21.886951+0000", 2024-08-29T15:18:10.918 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 48.932492080999999, 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.535110553, 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 140406 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.886951+0000", 2024-08-29T15:18:10.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.886951+0000", 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.886948+0000", 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.886959+0000", 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0540000000000001e-05 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.886982+0000", 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3411e-05 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.886991+0000", 2024-08-29T15:18:10.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9730000000000003e-06 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.887051+0000", 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0306999999999997e-05 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:21.887064+0000", 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2308e-05 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.907259+0000", 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0201951550000001 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:23.925314+0000", 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018055044999999999 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.067296+0000", 2024-08-29T15:18:10.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14198186700000001 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:24.085935+0000", 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018639082000000001 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.261960+0000", 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1760251180000001 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.272894+0000", 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010934131 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.295084+0000", 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022190168999999999 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.367841+0000", 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072756361000000006 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.387468+0000", 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019627067000000002 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.394747+0000", 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072790789999999999 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:26.422061+0000", 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027314560000000002 2024-08-29T15:18:10.926 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141529 6.a 6:5c7bae2a:::benchmark_data_smithi027_169445_object16102:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:35.810938+0000", 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 35.008504467999998, 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2212575069999998, 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.927 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141529 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.810938+0000", 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.810938+0000", 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.810937+0000", 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.810944+0000", 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5310000000000004e-06 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.810957+0000", 2024-08-29T15:18:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2398000000000001e-05 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.810963+0000", 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3720000000000001e-06 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.889809+0000", 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078845542000000005 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:35.909669+0000", 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019860705999999999 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.078870+0000", 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16920070600000001 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.091005+0000", 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012134558 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:38.886255+0000", 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7952507400000002 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:38.899860+0000", 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013604125 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:38.920606+0000", 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020745955999999999 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:38.997439+0000", 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.076833000999999998 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.000064+0000", 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026258150000000001 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.006026+0000", 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0059620439999999997 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.032196+0000", 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026169464999999999 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141552 6.a 6:59200826:::benchmark_data_smithi027_169445_object16125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:36.030089+0000", 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 34.789353959000003, 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.183469477, 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141552 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.030089+0000", 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.030089+0000", 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.030087+0000", 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.030096+0000", 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0100000000000001e-06 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.030111+0000", 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.517e-05 2024-08-29T15:18:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.030117+0000", 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3249999999999996e-06 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.094559+0000", 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.064442002999999998 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.106341+0000", 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011781448 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.283344+0000", 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17700283999999999 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.296491+0000", 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013147156 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.079263+0000", 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7827719320000002 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.091277+0000", 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012013909999999999 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.109639+0000", 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018362725999999999 2024-08-29T15:18:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.182509+0000", 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072869269 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.188110+0000", 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056010239999999996 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.192432+0000", 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043224359999999998 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.213558+0000", 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021126147000000001 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141555 6.a 6:5099db28:::benchmark_data_smithi027_169445_object16128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:36.120278+0000", 2024-08-29T15:18:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 34.699165118000003, 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3423114190000001, 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141555 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.120278+0000", 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.120278+0000", 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.120275+0000", 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.120287+0000", 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.269e-05 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.120305+0000", 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7923000000000001e-05 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.120313+0000", 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6260000000000003e-06 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.345253+0000", 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.22493949999999999 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.368149+0000", 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022896300000000001 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.506555+0000", 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13840655199999999 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.525870+0000", 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019315082000000001 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.265625+0000", 2024-08-29T15:18:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7397544489999999 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.276395+0000", 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010769933000000001 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.295133+0000", 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018738386999999999 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.424854+0000", 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12972102199999999 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.429523+0000", 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046687849999999999 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.437719+0000", 2024-08-29T15:18:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081963420000000006 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.462589+0000", 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024869737999999999 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141568 6.a 6:565f2f3b:::benchmark_data_smithi027_169445_object16141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:36.387846+0000", 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 34.431597277999998, 2024-08-29T15:18:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3009094889999999, 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141568 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.387846+0000", 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.387846+0000", 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.387843+0000", 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.387855+0000", 2024-08-29T15:18:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1365999999999999e-05 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.387872+0000", 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7813999999999999e-05 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.387883+0000", 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0481e-05 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.594100+0000", 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.20621709999999999 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.609296+0000", 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015196177 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.744198+0000", 2024-08-29T15:18:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.134902256 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.758178+0000", 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013979194 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.498050+0000", 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.739871875 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.509189+0000", 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011139605 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.527388+0000", 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018198551 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.556555+0000", 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029167149 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.661184+0000", 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10462901600000001 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.666232+0000", 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050485219999999997 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.688755+0000", 2024-08-29T15:18:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022522665000000001 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141581 6.a 6:564186fd:::benchmark_data_smithi027_169445_object16154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:36.602494+0000", 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 34.216948828, 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1618765039999999, 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141581 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.602494+0000", 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.602494+0000", 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.602492+0000", 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.602505+0000", 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3436e-05 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.602524+0000", 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9069999999999999e-05 2024-08-29T15:18:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.602536+0000", 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2199e-05 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.806062+0000", 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.20352564500000001 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.824445+0000", 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018383137000000001 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.972596+0000", 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.148150895 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.992964+0000", 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020368209000000002 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.581557+0000", 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5885931690000001 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.591081+0000", 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0095231889999999996 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0", 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.608782+0000", 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017701095 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.719011+0000", 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11022957 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.764370+0000", 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.045359258999999999 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141604 6.a 6:5e84e55e:::benchmark_data_smithi027_169445_object16177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:36.912127+0000", 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 33.907316213000001, 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.852870378, 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141604 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.912127+0000", 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.912127+0000", 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.912124+0000", 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:18:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.912136+0000", 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2499e-05 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.912155+0000", 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8538000000000001e-05 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.912166+0000", 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1117999999999999e-05 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:36.999018+0000", 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.086852396999999998 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.016264+0000", 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017246365999999999 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.179428+0000", 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16316365599999999 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.190895+0000", 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011467323999999999 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.764984+0000", 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5740887990000001 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.764997+0000", 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.273e-05 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141608 6.a 6:5f0d4814:::benchmark_data_smithi027_169445_object16181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:37.086347+0000", 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 33.733095923999997, 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6781576339999997, 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141608 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.086347+0000", 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.086347+0000", 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.086345+0000", 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.086356+0000", 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0895e-05 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.086373+0000", 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7689e-05 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.086383+0000", 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7799999999999995e-06 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.222884+0000", 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.136500856 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.239006+0000", 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016122068999999999 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.312767+0000", 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.073760609000000005 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.326915+0000", 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01414815 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.764486+0000", 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4375716130000002 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.764505+0000", 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8068999999999999e-05 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:141610 6.a 6:56485c9c:::benchmark_data_smithi027_169445_object16183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:37.206973+0000", 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 33.612469918000002, 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5582747000000001, 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 141610 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.206973+0000", 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.206973+0000", 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.206971+0000", 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.206980+0000", 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8279999999999992e-06 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.206994+0000", 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4017e-05 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.207001+0000", 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.306e-06 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.245367+0000", 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038365585000000001 2024-08-29T15:18:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.261090+0000", 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015723471999999999 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.374926+0000", 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.113835851 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:18:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:37.386511+0000", 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011585121 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.765241+0000", 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3787301799999998 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:39.765248+0000", 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2269999999999998e-06 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:10.983 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:18:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:10.984 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:10.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:11.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:10 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462}]': finished 2024-08-29T15:18:11.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:10 smithi027 ceph-mon[98970]: osdmap e211: 8 total, 7 up, 8 in 2024-08-29T15:18:11.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:10 smithi027 ceph-mon[98970]: osdmap e212: 8 total, 7 up, 8 in 2024-08-29T15:18:11.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:10 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462}]': finished 2024-08-29T15:18:11.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:10 smithi027 ceph-mon[93921]: osdmap e211: 8 total, 7 up, 8 in 2024-08-29T15:18:11.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:10 smithi027 ceph-mon[93921]: osdmap e212: 8 total, 7 up, 8 in 2024-08-29T15:18:11.085 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:11.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:11.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:11.235 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:11.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:10 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0561620354469462}]': finished 2024-08-29T15:18:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:10 smithi171 ceph-mon[90031]: osdmap e211: 8 total, 7 up, 8 in 2024-08-29T15:18:11.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:10 smithi171 ceph-mon[90031]: osdmap e212: 8 total, 7 up, 8 in 2024-08-29T15:18:11.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:11.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:11.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:11.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:11.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:11.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:11.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:11.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:11.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:11.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:18:11.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:11.838 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:18:11.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:11.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:11.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:11.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:12.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[93921]: pgmap v625: 89 pgs: 5 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 54570/324693 objects degraded (16.807%); 40655/324693 objects misplaced (12.521%); 1.2 MiB/s, 28 objects/s recovering 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/831180588' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 54570/324693 objects degraded (16.807%), 19 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[93921]: osdmap e213: 8 total, 7 up, 8 in 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[98970]: pgmap v625: 89 pgs: 5 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 54570/324693 objects degraded (16.807%); 40655/324693 objects misplaced (12.521%); 1.2 MiB/s, 28 objects/s recovering 2024-08-29T15:18:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/831180588' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:18:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:18:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 54570/324693 objects degraded (16.807%), 19 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:18:12.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:11 smithi027 ceph-mon[98970]: osdmap e213: 8 total, 7 up, 8 in 2024-08-29T15:18:12.072 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:12.086 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:12.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:12.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:12.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:12.198 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:11 smithi171 ceph-mon[90031]: pgmap v625: 89 pgs: 5 remapped+peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 9 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 54570/324693 objects degraded (16.807%); 40655/324693 objects misplaced (12.521%); 1.2 MiB/s, 28 objects/s recovering 2024-08-29T15:18:12.198 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/831180588' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:18:12.198 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:18:12.198 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:11 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 54570/324693 objects degraded (16.807%), 19 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:12.198 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:11 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:12.198 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:18:12.199 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:11 smithi171 ceph-mon[90031]: osdmap e213: 8 total, 7 up, 8 in 2024-08-29T15:18:12.216 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:12.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:12.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:12.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:12.343 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:12.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:12.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:12.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:12.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:12.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:12.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:12.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:12.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:12.621 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:12.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:12.722 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:12.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:12.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:12.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:12.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:12.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:12.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:12.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:12.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:13.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:13.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:13.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:13.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:13.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:13.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:13.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:12 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-29T15:18:13.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:12 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-29T15:18:13.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:12 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-29T15:18:13.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:13.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:13.483 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:13.507 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:13.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:13.608 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:13.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:13.732 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:13.757 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:13.858 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:13.882 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:13.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:13.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:14.007 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:13 smithi027 ceph-mon[93921]: pgmap v628: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 41 active+clean, 5 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 54357/323619 objects degraded (16.797%); 40475/323619 objects misplaced (12.507%); 480 KiB/s, 7 objects/s recovering 2024-08-29T15:18:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:13 smithi027 ceph-mon[98970]: pgmap v628: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 41 active+clean, 5 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 54357/323619 objects degraded (16.797%); 40475/323619 objects misplaced (12.507%); 480 KiB/s, 7 objects/s recovering 2024-08-29T15:18:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:14.077 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:18:14.077 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:18:14.077 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:18:14.077 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:18:14.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:18:14.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:18:14.086 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:18:14.086 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:18:14.086 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:18:14.098 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:18:14.098 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:18:14.109 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:14.115 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:18:14.115 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:18:14.132 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.132 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:18:14.133 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:18:14.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:14.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:18:14.150 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:18:14.167 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:18:14.167 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:18:14.234 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:14.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:13 smithi171 ceph-mon[90031]: pgmap v628: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 41 active+clean, 5 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 54357/323619 objects degraded (16.797%); 40475/323619 objects misplaced (12.507%); 480 KiB/s, 7 objects/s recovering 2024-08-29T15:18:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:14.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:14.383 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:14.485 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:14.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:14.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:14.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:14.759 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:18:14.831 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:18:14.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:14.886 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:14.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:14.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:15.012 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1477690753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1477690753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:15.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:15.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:15.171 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1477690753' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:15.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:18:15.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:15.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:15.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:15.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:15.435 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:15.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:15.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:15.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:15.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:18:15.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:15.636 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:15.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:15.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:15.761 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:15.776 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:15.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:15.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:15.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:15.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:15.988 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:16.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:16.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:16.123 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 6, 7, 5] 2024-08-29T15:18:16.123 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2024-08-29T15:18:16.124 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2024-08-29T15:18:16.124 DEBUG:teuthology.orchestra.run.smithi027:osd.2> sudo journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service 2024-08-29T15:18:16.136 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:16.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl start ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2 2024-08-29T15:18:16.151 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:15 smithi027 ceph-mon[93921]: pgmap v629: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 54317/323445 objects degraded (16.793%); 40475/323445 objects misplaced (12.514%); 288 KiB/s, 6 objects/s recovering 2024-08-29T15:18:16.152 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:15 smithi027 ceph-mon[98970]: pgmap v629: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 54317/323445 objects degraded (16.793%); 40475/323445 objects misplaced (12.514%); 288 KiB/s, 6 objects/s recovering 2024-08-29T15:18:16.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:16.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:15 smithi171 ceph-mon[90031]: pgmap v629: 89 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 4 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 42 active+clean, 5 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 54317/323445 objects degraded (16.793%); 40475/323445 objects misplaced (12.514%); 288 KiB/s, 6 objects/s recovering 2024-08-29T15:18:16.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:16.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:16.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:16.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:16.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:16.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:16 smithi027 systemd[1]: Starting Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:18:16.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:16.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:16.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:16.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:16.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:18:16.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:16.897 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:16.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:16.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:17.001 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:16 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 54317/323445 objects degraded (16.793%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:17.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:17.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:17.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:17.145 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:16 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 54317/323445 objects degraded (16.793%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:17.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:16 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 54317/323445 objects degraded (16.793%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:17.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:17.230 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:17.250 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:17.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:17.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:17.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:17.471 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:18:17.471 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:18:17.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:17.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:17.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:17.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:17.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:17.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:17.800 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:17 smithi027 podman[187737]: 2024-08-29 15:18:17.486964084 +0000 UTC m=+0.021680620 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:18:17.878 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:17.902 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:17.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:17.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:17.922 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:18:18.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:18.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:17 smithi027 ceph-mon[93921]: pgmap v630: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 65905/323004 objects degraded (20.404%); 48170/323004 objects misplaced (14.913%); 642 KiB/s, 12 objects/s recovering 2024-08-29T15:18:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:17 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-08-29T15:18:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1596134339' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:17 smithi027 ceph-mon[98970]: pgmap v630: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 65905/323004 objects degraded (20.404%); 48170/323004 objects misplaced (14.913%); 642 KiB/s, 12 objects/s recovering 2024-08-29T15:18:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:17 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-08-29T15:18:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1596134339' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:18.111 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:17 smithi171 ceph-mon[90031]: pgmap v630: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 65905/323004 objects degraded (20.404%); 48170/323004 objects misplaced (14.913%); 642 KiB/s, 12 objects/s recovering 2024-08-29T15:18:18.112 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:17 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-08-29T15:18:18.112 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1596134339' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:18.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:18.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:18:18.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:18.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:18.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:18.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:18.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:18.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:18.393 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:18.406 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:18.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:18.508 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:18.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:18.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:18.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:18.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:18.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:18.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:18.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:18.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:18.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:18.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:18.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:19.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:19.039 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:19.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:19.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:19.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:19.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:19.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:18 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:18:19.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:18 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1596134339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:19.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:18 smithi027 ceph-mon[98970]: osdmap e214: 8 total, 7 up, 8 in 2024-08-29T15:18:19.193 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:18:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:18:19] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-08-29T15:18:19.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:18 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:18:19.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:18 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1596134339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:19.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:18 smithi027 ceph-mon[93921]: osdmap e214: 8 total, 7 up, 8 in 2024-08-29T15:18:19.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:19.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:18 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:18:19.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:18 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1596134339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:19.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:18 smithi171 ceph-mon[90031]: osdmap e214: 8 total, 7 up, 8 in 2024-08-29T15:18:19.289 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:19.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:19.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:19.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:19.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:19.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:19.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:19.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:19.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:19.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:19.889 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:19.914 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:19.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:19.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:19.924 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:19 smithi027 podman[187737]: 2024-08-29 15:18:19.635714077 +0000 UTC m=+2.170430600 container create 5b49405dc660055e56c1249484aebc1fa5bf784fef2a8f80b97256c6cbacf684 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-29T15:18:20.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:19 smithi171 ceph-mon[90031]: pgmap v632: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 65846/322620 objects degraded (20.410%); 48091/322620 objects misplaced (14.906%); 183 KiB/s, 4 objects/s recovering 2024-08-29T15:18:20.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:20.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:20.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:20.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:20.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:20.195 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:19 smithi027 ceph-mon[98970]: pgmap v632: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 65846/322620 objects degraded (20.410%); 48091/322620 objects misplaced (14.906%); 183 KiB/s, 4 objects/s recovering 2024-08-29T15:18:20.195 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:19 smithi027 ceph-mon[93921]: pgmap v632: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 65846/322620 objects degraded (20.410%); 48091/322620 objects misplaced (14.906%); 183 KiB/s, 4 objects/s recovering 2024-08-29T15:18:20.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:20.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:20.352 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:18:20.353 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:18:20.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.416 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:20.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:20.506 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 podman[187737]: 2024-08-29 15:18:20.401191573 +0000 UTC m=+2.935908114 container init 5b49405dc660055e56c1249484aebc1fa5bf784fef2a8f80b97256c6cbacf684 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-08-29T15:18:20.507 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 podman[187737]: 2024-08-29 15:18:20.406054672 +0000 UTC m=+2.940771202 container start 5b49405dc660055e56c1249484aebc1fa5bf784fef2a8f80b97256c6cbacf684 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:18:20.507 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 podman[187737]: 2024-08-29 15:18:20.506073969 +0000 UTC m=+3.040790509 container attach 5b49405dc660055e56c1249484aebc1fa5bf784fef2a8f80b97256c6cbacf684 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-08-29T15:18:20.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:20.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:20.646 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:20.670 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:20.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:20.764 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.764 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.764 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.764 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.764 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:18:20.764 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-29T15:18:20.764 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.765 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.765 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.765 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-29T15:18:20.765 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:18:20.765 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:18:20.765 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-29T15:18:20.765 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-29T15:18:20.772 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:20.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:20.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:20.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:20.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:20.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:21.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:18:21.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:21.022 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:18:21.022 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:18:21.023 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-29T15:18:21.024 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate[187911]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-29T15:18:21.024 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 bash[187737]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-29T15:18:21.024 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:20 smithi027 podman[187737]: 2024-08-29 15:18:20.994655014 +0000 UTC m=+3.529371538 container died 5b49405dc660055e56c1249484aebc1fa5bf784fef2a8f80b97256c6cbacf684 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2) 2024-08-29T15:18:21.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:18:21.040 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:18:21.040 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:18:21.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:21.057 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:18:21.057 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:18:21.075 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:18:21.075 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:18:21.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:18:21.093 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:18:21.093 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:18:21.110 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:18:21.110 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:18:21.127 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:18:21.128 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:18:21.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:21.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:21.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:21.230 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:21.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:21.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:21.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:21.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:21.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:21.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:21.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:21.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:21.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:21.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:21.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:21.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:21.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:21.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:21.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:22.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:22.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:22.068 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:21 smithi027 podman[188148]: 2024-08-29 15:18:21.745533372 +0000 UTC m=+0.743909484 container remove 5b49405dc660055e56c1249484aebc1fa5bf784fef2a8f80b97256c6cbacf684 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731) 2024-08-29T15:18:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[98970]: pgmap v633: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 48 op/s; 65797/322353 objects degraded (20.411%); 48068/322353 objects misplaced (14.912%); 295 KiB/s, 6 objects/s recovering 2024-08-29T15:18:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/328837951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:18:22.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 65797/322353 objects degraded (20.411%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[93921]: pgmap v633: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 48 op/s; 65797/322353 objects degraded (20.411%); 48068/322353 objects misplaced (14.912%); 295 KiB/s, 6 objects/s recovering 2024-08-29T15:18:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/328837951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:18:22.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:21 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 65797/322353 objects degraded (20.411%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:22.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:22.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:21 smithi171 ceph-mon[90031]: pgmap v633: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 48 op/s; 65797/322353 objects degraded (20.411%); 48068/322353 objects misplaced (14.912%); 295 KiB/s, 6 objects/s recovering 2024-08-29T15:18:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/328837951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:18:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:21 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 65797/322353 objects degraded (20.411%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:22.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:22.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:22.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:22.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:22.431 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:18:22.474 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:18:22.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:22.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:22.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:22.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:22.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:22.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:22.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:22.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:22.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:22.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:22.934 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:22.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:22.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:22.991 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:22 smithi027 podman[188507]: 2024-08-29 15:18:22.578145688 +0000 UTC m=+0.016629793 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:18:22.991 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:22 smithi027 podman[188507]: 2024-08-29 15:18:22.698032757 +0000 UTC m=+0.136516851 container create 61ba5f8296d5ccbb0506a2576c8daa8f01e7e9753c7a2127d8c934f28fcecac1 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-29T15:18:22.991 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:22 smithi027 podman[188507]: 2024-08-29 15:18:22.990244902 +0000 UTC m=+0.428729004 container init 61ba5f8296d5ccbb0506a2576c8daa8f01e7e9753c7a2127d8c934f28fcecac1 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, org.label-schema.build-date=20240731, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-29T15:18:23.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:23.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:23.070 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:18:23.071 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:18:23.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.071 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:143696 6.a 6:57fd3b9a:::benchmark_data_smithi027_169445_object18269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-08-29T15:18:23.071 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:22.994401+0000", 2024-08-29T15:18:23.072 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.075283102000000005, 2024-08-29T15:18:23.072 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.075325372000000002, 2024-08-29T15:18:23.072 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:23.072 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for degraded object", 2024-08-29T15:18:23.073 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:18:23.073 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:18:23.073 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:18:23.073 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 143696 2024-08-29T15:18:23.073 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.073 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:23.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:23.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.994401+0000", 2024-08-29T15:18:23.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:23.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.074 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:23.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.994401+0000", 2024-08-29T15:18:23.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:23.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:23.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.994399+0000", 2024-08-29T15:18:23.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:18:23.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:23.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.994409+0000", 2024-08-29T15:18:23.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7850000000000003e-06 2024-08-29T15:18:23.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:23.077 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.994426+0000", 2024-08-29T15:18:23.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7345000000000001e-05 2024-08-29T15:18:23.077 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.077 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.077 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:23.077 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.994435+0000", 2024-08-29T15:18:23.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8950000000000004e-06 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.031348+0000", 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036913171000000002 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.062015+0000", 2024-08-29T15:18:23.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030666992000000001 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPushReply(6.a 214/202 [PushReplyOp(6:57fd3b9a:::benchmark_data_smithi027_169445_object18269:head)])", 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:23.064918+0000", 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0047653549999999998, 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0048542159999999997, 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:18:23.079 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.064918+0000", 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.064918+0000", 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.064917+0000", 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.064926+0000", 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6759999999999996e-06 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:18:23.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.064942+0000", 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5696e-05 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:23.064954+0000", 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2628e-05 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 2 2024-08-29T15:18:23.082 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:18:23.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:18:23.107 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:23.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:23.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.191 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:23.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:23.245 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:22 smithi027 podman[188507]: 2024-08-29 15:18:22.995267143 +0000 UTC m=+0.433751237 container start 61ba5f8296d5ccbb0506a2576c8daa8f01e7e9753c7a2127d8c934f28fcecac1 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-29T15:18:23.245 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:23 smithi027 ceph-osd[188574]: -- 172.21.15.27:0/46681602 <== mon.2 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561263ae7a40 con 0x561263ac5c00 2024-08-29T15:18:23.245 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:23 smithi027 bash[188507]: 61ba5f8296d5ccbb0506a2576c8daa8f01e7e9753c7a2127d8c934f28fcecac1 2024-08-29T15:18:23.245 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:23 smithi027 systemd[1]: Started Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:18:23.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:23.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:23.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:23.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:23.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:23.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:23.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:23.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:23.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:23.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:18:23.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:23.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:23.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:23.946 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:23.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:23.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:23 smithi171 ceph-mon[90031]: pgmap v634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 55 op/s; 65712/321960 objects degraded (20.410%); 47991/321960 objects misplaced (14.906%); 333 KiB/s, 12 objects/s recovering 2024-08-29T15:18:24.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:24.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:24.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.202 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:24.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:24.253 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:23 smithi027 ceph-mon[93921]: pgmap v634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 55 op/s; 65712/321960 objects degraded (20.410%); 47991/321960 objects misplaced (14.906%); 333 KiB/s, 12 objects/s recovering 2024-08-29T15:18:24.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:23 smithi027 ceph-mon[98970]: pgmap v634: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 55 op/s; 65712/321960 objects degraded (20.410%); 47991/321960 objects misplaced (14.906%); 333 KiB/s, 12 objects/s recovering 2024-08-29T15:18:24.303 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:24.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:24.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:24.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:24.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:24.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:24.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:18:24.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:24.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:24.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:24.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:24.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:24.717 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:24.747 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:24.779 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:24 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[188540]: 2024-08-29T15:18:24.625+0000 7fe5ef79a640 -1 Falling back to public interface 2024-08-29T15:18:24.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:24.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:24.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:24.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:24.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:24.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:25.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:25.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:25.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:25.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:25.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:25.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:25.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:25.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:25.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:25.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:25.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:25.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:25.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:25.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:25.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:25.839 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:25.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:25.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:25.964 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:25.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:26.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:26.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:25 smithi027 ceph-mon[98970]: pgmap v635: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 65679/321819 objects degraded (20.409%); 47991/321819 objects misplaced (14.912%); 448 KiB/s, 12 objects/s recovering 2024-08-29T15:18:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:25 smithi027 ceph-mon[93921]: pgmap v635: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 65679/321819 objects degraded (20.409%); 47991/321819 objects misplaced (14.912%); 448 KiB/s, 12 objects/s recovering 2024-08-29T15:18:26.089 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:26.149 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:25 smithi171 ceph-mon[90031]: pgmap v635: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 65679/321819 objects degraded (20.409%); 47991/321819 objects misplaced (14.912%); 448 KiB/s, 12 objects/s recovering 2024-08-29T15:18:26.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:26.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:26.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:26.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:26.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:26.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:26.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:26.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:26.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:26.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:26.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:26.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:26.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:26.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:26.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:26.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:26.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:27.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:27.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:27.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:27.128 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:18:27.128 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:18:27.128 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:18:27.128 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:18:27.128 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:18:27.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:26 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 65679/321819 objects degraded (20.409%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:27.179 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:26 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 65679/321819 objects degraded (20.409%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:27.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:27.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:27.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:26 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 65679/321819 objects degraded (20.409%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:config diff 2024-08-29T15:18:27.278 INFO:teuthology.orchestra.run.smithi027.stderr:config diff get 2024-08-29T15:18:27.279 INFO:teuthology.orchestra.run.smithi027.stderr:config get 2024-08-29T15:18:27.279 INFO:teuthology.orchestra.run.smithi027.stderr:config help [] 2024-08-29T15:18:27.279 INFO:teuthology.orchestra.run.smithi027.stderr:config set ... 2024-08-29T15:18:27.279 INFO:teuthology.orchestra.run.smithi027.stderr:admin_socket: invalid command 2024-08-29T15:18:27.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:27.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:27.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:27.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:27.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:27.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:27.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:27.620 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:27.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:27.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:27.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:27.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:27.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:27.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:18:27.938 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:18:27.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:27.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:27.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:27.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:28.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:28.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:28.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:28.183 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:18:28.184 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-2, ['dump_ops_in_flight'] 2024-08-29T15:18:28.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:28.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:28.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:27 smithi171 ceph-mon[90031]: pgmap v636: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 55 op/s; 65438/321339 objects degraded (20.364%); 47917/321339 objects misplaced (14.912%); 1.4 MiB/s, 29 objects/s recovering 2024-08-29T15:18:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2019669666' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:18:28.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:27 smithi027 ceph-mon[98970]: pgmap v636: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 55 op/s; 65438/321339 objects degraded (20.364%); 47917/321339 objects misplaced (14.912%); 1.4 MiB/s, 29 objects/s recovering 2024-08-29T15:18:28.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2019669666' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:18:28.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:27 smithi027 ceph-mon[93921]: pgmap v636: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 55 op/s; 65438/321339 objects degraded (20.364%); 47917/321339 objects misplaced (14.912%); 1.4 MiB/s, 29 objects/s recovering 2024-08-29T15:18:28.286 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2019669666' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:18:28.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:28.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:28.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:28.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:28.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:28.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:28.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:28.627 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:28.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:28.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:28.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:28.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:28.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:28.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:28.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:28.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:28.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:28.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:29.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:29.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:29.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:29.126 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:29.228 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:29.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:28 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2019669666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:18:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:28 smithi171 ceph-mon[90031]: osdmap e215: 8 total, 7 up, 8 in 2024-08-29T15:18:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2019669666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:18:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[98970]: osdmap e215: 8 total, 7 up, 8 in 2024-08-29T15:18:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:29.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:29.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2019669666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:18:29.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[93921]: osdmap e215: 8 total, 7 up, 8 in 2024-08-29T15:18:29.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:29.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:29.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:18:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:18:29] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-08-29T15:18:29.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:29.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:29.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:29.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:29.531 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:18:29.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:29.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:29.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:29.751 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:29.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:29.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:29.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:29.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:30.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:30.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:30.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:30.231 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:30.254 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:29 smithi171 ceph-mon[90031]: pgmap v638: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 58 op/s; 65357/320874 objects degraded (20.368%); 47817/320874 objects misplaced (14.902%); 1.4 MiB/s, 29 objects/s recovering 2024-08-29T15:18:30.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:29 smithi027 ceph-mon[98970]: pgmap v638: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 58 op/s; 65357/320874 objects degraded (20.368%); 47817/320874 objects misplaced (14.902%); 1.4 MiB/s, 29 objects/s recovering 2024-08-29T15:18:30.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:29 smithi027 ceph-mon[93921]: pgmap v638: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 58 op/s; 65357/320874 objects degraded (20.368%); 47817/320874 objects misplaced (14.902%); 1.4 MiB/s, 29 objects/s recovering 2024-08-29T15:18:30.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:30.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:30.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:30.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:30.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:30.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:30.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:30.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.643 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:30.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:30.744 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:30.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:30.880 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:30.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:30.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:31.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:31.035 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:31.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:31.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:31.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:31.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:31.316 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[188540]: 2024-08-29T15:18:31.209+0000 7fe5ef79a640 -1 osd.2 181 log_to_monitors true 2024-08-29T15:18:31.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:31.372 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:18:31.372 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:18:31.389 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:18:31.389 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:18:31.390 INFO:teuthology.orchestra.run.smithi027.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-29T15:18:31.406 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:18:31.406 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:18:31.424 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:18:31.424 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:18:31.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:31.441 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:18:31.441 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:18:31.458 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:18:31.459 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:18:31.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:31.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:18:31.476 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:18:31.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:31.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:31.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:31.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:31.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:18:31.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:31.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:31.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:31.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:31.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:31.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:31.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:32.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:32.097 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:32.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:32.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:31 smithi171 ceph-mon[90031]: pgmap v639: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 59 op/s; 65293/320571 objects degraded (20.368%); 47793/320571 objects misplaced (14.909%); 1.4 MiB/s, 30 objects/s recovering 2024-08-29T15:18:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:31 smithi171 ceph-mon[90031]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:18:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/33093629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:32.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:31 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 65293/320571 objects degraded (20.368%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:32.304 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[188540]: 2024-08-29T15:18:31.996+0000 7fe5e353c640 -1 osd.2 181 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-29T15:18:32.304 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[98970]: pgmap v639: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 59 op/s; 65293/320571 objects degraded (20.368%); 47793/320571 objects misplaced (14.909%); 1.4 MiB/s, 30 objects/s recovering 2024-08-29T15:18:32.304 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[98970]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:18:32.304 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/33093629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:32.304 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 65293/320571 objects degraded (20.368%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:32.304 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[93921]: pgmap v639: 89 pgs: 1 active+recovering+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 59 op/s; 65293/320571 objects degraded (20.368%); 47793/320571 objects misplaced (14.909%); 1.4 MiB/s, 30 objects/s recovering 2024-08-29T15:18:32.305 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[93921]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-29T15:18:32.305 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/33093629' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:32.305 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:31 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 65293/320571 objects degraded (20.368%), 21 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:32.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:32.346 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:32.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:32.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:32.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:32.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:32.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:32.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:32.708 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:18:32.709 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:18:32.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:32.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-29T15:18:32.812 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-29T15:18:32.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:32.848 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:32.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:32.856 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:32.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:32.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:32.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:32.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:32 smithi027 ceph-mon[93921]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T15:18:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:32 smithi027 ceph-mon[93921]: osdmap e216: 8 total, 7 up, 8 in 2024-08-29T15:18:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:32 smithi027 ceph-mon[93921]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:18:33.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:32 smithi027 ceph-mon[98970]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T15:18:33.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:32 smithi027 ceph-mon[98970]: osdmap e216: 8 total, 7 up, 8 in 2024-08-29T15:18:33.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:32 smithi027 ceph-mon[98970]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:18:33.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:33.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:33.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:33.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:33.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:18:33.207 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:33.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:33.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:33.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:32 smithi171 ceph-mon[90031]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-29T15:18:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:32 smithi171 ceph-mon[90031]: osdmap e216: 8 total, 7 up, 8 in 2024-08-29T15:18:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:32 smithi171 ceph-mon[90031]: from='osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]} : dispatch 2024-08-29T15:18:33.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:33.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:33.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:33.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:18:33.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:33.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:33.505 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:33.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:33.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:33.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:33.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:33.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:33.754 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:33.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:33.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:33.882 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:33.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:33.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:34.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:34.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:34.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:34.118 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:33 smithi171 ceph-mon[90031]: pgmap v641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 79 op/s; 64914/319920 objects degraded (20.291%); 47682/319920 objects misplaced (14.904%); 3.9 MiB/s, 64 objects/s recovering 2024-08-29T15:18:34.118 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:33 smithi171 ceph-mon[90031]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:18:34.118 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:33 smithi171 ceph-mon[90031]: osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804] boot 2024-08-29T15:18:34.118 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:33 smithi171 ceph-mon[90031]: osdmap e217: 8 total, 8 up, 8 in 2024-08-29T15:18:34.118 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:33 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-29T15:18:34.119 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:33 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:34.119 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:33 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:34.135 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:34.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[98970]: pgmap v641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 79 op/s; 64914/319920 objects degraded (20.291%); 47682/319920 objects misplaced (14.904%); 3.9 MiB/s, 64 objects/s recovering 2024-08-29T15:18:34.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[98970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:18:34.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[98970]: osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804] boot 2024-08-29T15:18:34.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[98970]: osdmap e217: 8 total, 8 up, 8 in 2024-08-29T15:18:34.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-29T15:18:34.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:34.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:34.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[93921]: pgmap v641: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 5 active+undersized+degraded, 9 active+undersized, 15 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 79 op/s; 64914/319920 objects degraded (20.291%); 47682/319920 objects misplaced (14.904%); 3.9 MiB/s, 64 objects/s recovering 2024-08-29T15:18:34.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[93921]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-29T15:18:34.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[93921]: osd.2 [v2:172.21.15.27:6818/829268804,v1:172.21.15.27:6819/829268804] boot 2024-08-29T15:18:34.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[93921]: osdmap e217: 8 total, 8 up, 8 in 2024-08-29T15:18:34.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-29T15:18:34.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:34.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:33 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:34.237 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:34.260 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:34.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:34.384 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:34.477 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:34.478 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:34.485 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:34.506 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:34.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:34.608 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:34.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:34.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:34.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:34.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:18:34.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:34.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:34.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:34.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:35.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:35.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:35.120 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:35.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:35.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:35.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: osdmap e218: 8 total, 8 up, 8 in 2024-08-29T15:18:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: 6.2 continuing backfill to osd.6 from (176'2199,216'8918] MIN to 216'8918 2024-08-29T15:18:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: 6.1 continuing backfill to osd.6 from (176'1980,216'9018] 6:801e79a9:::benchmark_data_smithi027_169445_object115041:head to 216'9018 2024-08-29T15:18:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: 6.f continuing backfill to osd.6 from (177'2368,216'9096] MIN to 216'9096 2024-08-29T15:18:35.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: 6.e continuing backfill to osd.5 from (176'2033,216'9090] MIN to 216'9090 2024-08-29T15:18:35.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:35.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:35.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:35 smithi171 ceph-mon[90031]: osdmap e219: 8 total, 8 up, 8 in 2024-08-29T15:18:35.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:35.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: osdmap e218: 8 total, 8 up, 8 in 2024-08-29T15:18:35.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: 6.2 continuing backfill to osd.6 from (176'2199,216'8918] MIN to 216'8918 2024-08-29T15:18:35.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: 6.1 continuing backfill to osd.6 from (176'1980,216'9018] 6:801e79a9:::benchmark_data_smithi027_169445_object115041:head to 216'9018 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: 6.f continuing backfill to osd.6 from (177'2368,216'9096] MIN to 216'9096 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: 6.e continuing backfill to osd.5 from (176'2033,216'9090] MIN to 216'9090 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[93921]: osdmap e219: 8 total, 8 up, 8 in 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: osdmap e218: 8 total, 8 up, 8 in 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: 6.2 continuing backfill to osd.6 from (176'2199,216'8918] MIN to 216'8918 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: 6.1 continuing backfill to osd.6 from (176'1980,216'9018] 6:801e79a9:::benchmark_data_smithi027_169445_object115041:head to 216'9018 2024-08-29T15:18:35.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: 6.f continuing backfill to osd.6 from (177'2368,216'9096] MIN to 216'9096 2024-08-29T15:18:35.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: 6.e continuing backfill to osd.5 from (176'2033,216'9090] MIN to 216'9090 2024-08-29T15:18:35.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:35.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:35.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:35 smithi027 ceph-mon[98970]: osdmap e219: 8 total, 8 up, 8 in 2024-08-29T15:18:35.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:35.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:35.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:35.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:35.519 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:35.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:35.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:35.643 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:35.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:35.767 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:35.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:35.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:35.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:35.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:36.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:36.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[93921]: pgmap v644: 89 pgs: 1 active+undersized+remapped, 4 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 2 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 60 op/s; 51363/319791 objects degraded (16.061%); 47681/319791 objects misplaced (14.910%); 3.2 MiB/s, 68 objects/s recovering 2024-08-29T15:18:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[93921]: osdmap e220: 8 total, 8 up, 8 in 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[98970]: pgmap v644: 89 pgs: 1 active+undersized+remapped, 4 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 2 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 60 op/s; 51363/319791 objects degraded (16.061%); 47681/319791 objects misplaced (14.910%); 3.2 MiB/s, 68 objects/s recovering 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:36.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:36 smithi027 ceph-mon[98970]: osdmap e220: 8 total, 8 up, 8 in 2024-08-29T15:18:36.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:36.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:36.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:36 smithi171 ceph-mon[90031]: pgmap v644: 89 pgs: 1 active+undersized+remapped, 4 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 2 active+undersized+degraded, 8 active+undersized, 13 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 60 op/s; 51363/319791 objects degraded (16.061%); 47681/319791 objects misplaced (14.910%); 3.2 MiB/s, 68 objects/s recovering 2024-08-29T15:18:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:18:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:18:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:36.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:36 smithi171 ceph-mon[90031]: osdmap e220: 8 total, 8 up, 8 in 2024-08-29T15:18:36.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.269 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:36.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:36.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:36.397 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:36.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:36.523 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:36.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:36.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:36.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:36.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:36.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:36.768 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:36.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:36.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:36.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:36.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:36.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:36.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:37.015 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:18:37.015 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:18:37.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:37.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:37.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:37.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:37.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 51363/319791 objects degraded (16.061%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:37 smithi171 ceph-mon[90031]: osdmap e221: 8 total, 8 up, 8 in 2024-08-29T15:18:37.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:37.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 51363/319791 objects degraded (16.061%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:37.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:37 smithi027 ceph-mon[98970]: osdmap e221: 8 total, 8 up, 8 in 2024-08-29T15:18:37.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 51363/319791 objects degraded (16.061%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:18:37.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:37 smithi027 ceph-mon[93921]: osdmap e221: 8 total, 8 up, 8 in 2024-08-29T15:18:37.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:37.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:37.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:37.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:37.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:37.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:37.751 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:37.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:37.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:37.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:37.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:37.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:38.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:38.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:38.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:38.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:38.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:38.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:38 smithi171 ceph-mon[90031]: pgmap v647: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded, 58 active+clean, 1 active+undersized+degraded, 3 active+undersized, 7 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 44590/319677 objects degraded (13.948%); 43256/319677 objects misplaced (13.531%); 5.5 MiB/s, 125 objects/s recovering 2024-08-29T15:18:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:38 smithi171 ceph-mon[90031]: osdmap e222: 8 total, 8 up, 8 in 2024-08-29T15:18:38.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:38 smithi027 ceph-mon[98970]: pgmap v647: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded, 58 active+clean, 1 active+undersized+degraded, 3 active+undersized, 7 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 44590/319677 objects degraded (13.948%); 43256/319677 objects misplaced (13.531%); 5.5 MiB/s, 125 objects/s recovering 2024-08-29T15:18:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:38 smithi027 ceph-mon[98970]: osdmap e222: 8 total, 8 up, 8 in 2024-08-29T15:18:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:38 smithi027 ceph-mon[93921]: pgmap v647: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+degraded, 58 active+clean, 1 active+undersized+degraded, 3 active+undersized, 7 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 20 op/s; 44590/319677 objects degraded (13.948%); 43256/319677 objects misplaced (13.531%); 5.5 MiB/s, 125 objects/s recovering 2024-08-29T15:18:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:38 smithi027 ceph-mon[93921]: osdmap e222: 8 total, 8 up, 8 in 2024-08-29T15:18:38.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.407 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:38.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:38.508 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.530 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:38.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:38.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:38.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:38.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:38.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:38.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:38.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:38.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:38.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:39.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:39.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:18:39.040 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:39.051 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:39.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:39.168 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:39.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:39.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:39.298 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:18:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:18:39] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-08-29T15:18:39.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:39.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:39.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:39.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:39.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:39.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:39.557 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:39.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:39.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:39.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:39.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:39.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:39.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:39.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:39.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:40.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:40.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:40.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:40.165 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:40 smithi027 ceph-mon[98970]: pgmap v650: 89 pgs: 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 29133/319560 objects degraded (9.117%); 38979/319560 objects misplaced (12.198%); 6.1 MiB/s, 133 objects/s recovering 2024-08-29T15:18:40.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:40 smithi027 ceph-mon[93921]: pgmap v650: 89 pgs: 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 29133/319560 objects degraded (9.117%); 38979/319560 objects misplaced (12.198%); 6.1 MiB/s, 133 objects/s recovering 2024-08-29T15:18:40.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:40 smithi171 ceph-mon[90031]: pgmap v650: 89 pgs: 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 29133/319560 objects degraded (9.117%); 38979/319560 objects misplaced (12.198%); 6.1 MiB/s, 133 objects/s recovering 2024-08-29T15:18:40.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:40.304 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:18:40.304 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:18:40.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.304 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(6.9 222/218 [PushOp(6:9f88ee85:::benchmark_data_smithi027_169445_object92700:head, version: 186'5842, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:9f88ee85:::benchmark_data_smithi027_169445_object92700:head@186'5842, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-29T15:18:40.304 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:40.301207+0000", 2024-08-29T15:18:40.304 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00025928399999999998, 2024-08-29T15:18:40.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029133400000000003, 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301207+0000", 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:18:40.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301207+0000", 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301208+0000", 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.001e-06 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:18:40.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301236+0000", 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8027000000000001e-05 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301255+0000", 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9089e-05 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301268+0000", 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2661e-05 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:18:40.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301375+0000", 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010677499999999999 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:40.301389+0000", 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3793e-05 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:18:40.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:18:40.309 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:18:40.309 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:18:40.309 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2024-08-29T15:18:40.309 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:18:40.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:40.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:40.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:40.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:40.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:40.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:40.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:40.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:40.687 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:40.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:40.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:40.912 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:40.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:40.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:41.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:41.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:41.166 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:41.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:41.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:41.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:41.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:41.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:41.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:41.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:41.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:41.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:41.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:41.664 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:41.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:41.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:41.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:41.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:41.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:41.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:41.936 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:41.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:42.025 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:18:42.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:42.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:42.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:42.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:42.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:42.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:42 smithi171 ceph-mon[90031]: pgmap v651: 89 pgs: 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 34895/319560 objects degraded (10.920%); 38979/319560 objects misplaced (12.198%); 6.5 MiB/s, 140 objects/s recovering 2024-08-29T15:18:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3378542947' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:42 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:42.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 34895/319560 objects degraded (10.920%), 10 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:42.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:42.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:42.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[98970]: pgmap v651: 89 pgs: 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 34895/319560 objects degraded (10.920%); 38979/319560 objects misplaced (12.198%); 6.5 MiB/s, 140 objects/s recovering 2024-08-29T15:18:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3378542947' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 34895/319560 objects degraded (10.920%), 10 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[93921]: pgmap v651: 89 pgs: 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped+peering, 3 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 34895/319560 objects degraded (10.920%); 38979/319560 objects misplaced (12.198%); 6.5 MiB/s, 140 objects/s recovering 2024-08-29T15:18:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3378542947' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:18:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 34895/319560 objects degraded (10.920%), 10 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:42.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:42.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:42.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:42.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:42.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:42.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:42.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:42.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:42.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:42.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:42.816 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:42.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:42.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 0 1 2024-08-29T15:18:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:42.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:42.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:43.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:43.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:43.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:43.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:43 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:43 smithi171 ceph-mon[90031]: osdmap e223: 8 total, 8 up, 8 in 2024-08-29T15:18:43.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:43 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:43.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:43.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:43 smithi027 ceph-mon[98970]: osdmap e223: 8 total, 8 up, 8 in 2024-08-29T15:18:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:43 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:18:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:18:43.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:43 smithi027 ceph-mon[93921]: osdmap e223: 8 total, 8 up, 8 in 2024-08-29T15:18:43.322 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:43.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:43.425 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:43.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:43.553 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:43.575 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:43.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:43.698 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:43.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:43.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:43.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:43.948 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:43.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:44.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:44.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:44.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:44.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:44.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:44 smithi027 ceph-mon[93921]: pgmap v653: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 40483/319560 objects degraded (12.668%); 42862/319560 objects misplaced (13.413%); 9.0 MiB/s, 159 objects/s recovering 2024-08-29T15:18:44.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:44.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:44.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:44 smithi027 ceph-mon[98970]: pgmap v653: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 40483/319560 objects degraded (12.668%); 42862/319560 objects misplaced (13.413%); 9.0 MiB/s, 159 objects/s recovering 2024-08-29T15:18:44.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:44.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:44.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:44.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.330 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:44.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:44.426 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:18:44.427 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:18:44.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:44.435 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:44.448 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:18:44.449 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:18:44.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:44.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:44.481 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:18:44.482 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-08-29T15:18:44.501 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:18:44.501 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:18:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:44 smithi171 ceph-mon[90031]: pgmap v653: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 40483/319560 objects degraded (12.668%); 42862/319560 objects misplaced (13.413%); 9.0 MiB/s, 159 objects/s recovering 2024-08-29T15:18:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:18:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:44.524 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:18:44.525 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:18:44.542 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:18:44.542 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:18:44.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:44.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:44.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:44.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:44.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:44.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:44.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:44.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:44.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:44.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:44.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:45.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:45.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:45.153 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:18:45.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:45.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:45.280 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3254603867' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:45.280 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3254603867' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:45.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:45.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:45.450 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:45.472 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:45.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3254603867' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:18:45.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:45.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:45.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:45.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:18:45.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:45.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:45.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:45.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:45.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:45.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:45.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:45.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:46.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:46.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:46.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:46.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:46.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:46 smithi027 ceph-mon[98970]: pgmap v654: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 39893/319560 objects degraded (12.484%); 42862/319560 objects misplaced (13.413%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:18:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:46 smithi027 ceph-mon[93921]: pgmap v654: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 39893/319560 objects degraded (12.484%); 42862/319560 objects misplaced (13.413%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:18:46.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:46.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:46.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:46.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:46.468 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:46.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:46 smithi171 ceph-mon[90031]: pgmap v654: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 5 op/s; 39893/319560 objects degraded (12.484%); 42862/319560 objects misplaced (13.413%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:18:46.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:46.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:46.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:46.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:46.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:46.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:46.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:46.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:46.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:46.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:46.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:47.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:47.095 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:47.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:47.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:47.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:47.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:47.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 39893/319560 objects degraded (12.484%), 11 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:47.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 39893/319560 objects degraded (12.484%), 11 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:47.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:47.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:47.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:47.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:47.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 39893/319560 objects degraded (12.484%), 11 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:47.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:47.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:47.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:47.605 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:18:47.605 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:18:47.626 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:18:47.626 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:18:47.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:47.723 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:47.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:47.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:47.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:47.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:47.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:47.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:47.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:48.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:48.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:48.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:48.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:48.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:48 smithi027 ceph-mon[98970]: pgmap v655: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 39893/319560 objects degraded (12.484%); 42862/319560 objects misplaced (13.413%); 10 MiB/s, 161 objects/s recovering 2024-08-29T15:18:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:48 smithi027 ceph-mon[93921]: pgmap v655: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 39893/319560 objects degraded (12.484%); 42862/319560 objects misplaced (13.413%); 10 MiB/s, 161 objects/s recovering 2024-08-29T15:18:48.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:48.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:48.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:48.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:48.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:48 smithi171 ceph-mon[90031]: pgmap v655: 89 pgs: 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean, 4 active+clean+remapped; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 39893/319560 objects degraded (12.484%); 42862/319560 objects misplaced (13.413%); 10 MiB/s, 161 objects/s recovering 2024-08-29T15:18:48.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:48.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:48.708 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:48.730 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:48.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:48.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:48.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:48.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:48.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:49.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:49.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:49.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:49.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:49.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:18:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:18:49] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.51.0" 2024-08-29T15:18:49.336 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:49.357 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:49.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:49.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:49.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:49.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:49.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:49.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:49.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:49.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:49.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:49.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:49.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:49.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:49.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:49.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:49.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:50.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:50.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:50.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:50.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:50.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:50 smithi027 ceph-mon[93921]: pgmap v656: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 43924/319524 objects degraded (13.747%); 34644/319524 objects misplaced (10.842%); 12 MiB/s, 197 objects/s recovering 2024-08-29T15:18:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:50 smithi027 ceph-mon[98970]: pgmap v656: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 43924/319524 objects degraded (13.747%); 34644/319524 objects misplaced (10.842%); 12 MiB/s, 197 objects/s recovering 2024-08-29T15:18:50.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:50.365 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:50.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:50.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:50.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:50 smithi171 ceph-mon[90031]: pgmap v656: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 43924/319524 objects degraded (13.747%); 34644/319524 objects misplaced (10.842%); 12 MiB/s, 197 objects/s recovering 2024-08-29T15:18:50.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:50.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:50.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:50.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:50.740 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:50.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:50.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:50.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:50.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:50.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:50.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:50.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:51.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:51.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:51.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:51.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:51.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:51.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:51.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:51.368 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:51.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:51.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:51.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:51.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:51.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:51.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:51.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:51.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:51.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:51.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:51.749 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:51.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:51.851 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:51.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:51.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:51.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:52.000 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:52.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:52.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:52.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:52.227 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:52.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:52 smithi027 ceph-mon[98970]: pgmap v657: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 43082/319524 objects degraded (13.483%); 34644/319524 objects misplaced (10.842%); 16 MiB/s, 256 objects/s recovering 2024-08-29T15:18:52.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 43082/319524 objects degraded (13.483%), 10 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:52 smithi027 ceph-mon[93921]: pgmap v657: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 43082/319524 objects degraded (13.483%); 34644/319524 objects misplaced (10.842%); 16 MiB/s, 256 objects/s recovering 2024-08-29T15:18:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 43082/319524 objects degraded (13.483%), 10 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:52.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:52.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:52.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:52.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:52 smithi171 ceph-mon[90031]: pgmap v657: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 43082/319524 objects degraded (13.483%); 34644/319524 objects misplaced (10.842%); 16 MiB/s, 256 objects/s recovering 2024-08-29T15:18:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 43082/319524 objects degraded (13.483%), 10 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:52.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:52.626 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:52.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:52.728 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:52.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:52.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:52.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:52.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:52.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:53.005 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:53.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:53.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:53.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.131 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:53.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:53.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:53.254 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:53.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:53.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:53.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:53.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:53.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:53.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:53.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:53.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:53.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:53.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:53.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:53.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:53.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:53.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:53.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:53.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:53.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:54.011 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:54.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:54.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:54.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:54.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:54.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:54 smithi027 ceph-mon[98970]: pgmap v658: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 42419/319524 objects degraded (13.276%); 33981/319524 objects misplaced (10.635%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:18:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:54 smithi027 ceph-mon[93921]: pgmap v658: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 42419/319524 objects degraded (13.276%); 33981/319524 objects misplaced (10.635%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:18:54.367 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:54.389 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:54.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:54.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:54.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:54 smithi171 ceph-mon[90031]: pgmap v658: 89 pgs: 1 activating+undersized+degraded+remapped, 2 activating, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 75 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 42419/319524 objects degraded (13.276%); 33981/319524 objects misplaced (10.635%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:18:54.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:54.638 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:54.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:54.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:54.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:54.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:54.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:54.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:54.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:54.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:55.022 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:55.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:55.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:55.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:55.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:55.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:55.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:55.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:55.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:55.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:55.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:55.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:55.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:55.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:55.752 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:55.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:55.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:55.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:55.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:55.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:56.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:56.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:56.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:56.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:56.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:56.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:56.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:56 smithi027 ceph-mon[93921]: pgmap v659: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 41680/319524 objects degraded (13.044%); 33981/319524 objects misplaced (10.635%); 18 MiB/s, 289 objects/s recovering 2024-08-29T15:18:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:56 smithi027 ceph-mon[98970]: pgmap v659: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 41680/319524 objects degraded (13.044%); 33981/319524 objects misplaced (10.635%); 18 MiB/s, 289 objects/s recovering 2024-08-29T15:18:56.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:56.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:56.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:56 smithi171 ceph-mon[90031]: pgmap v659: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 41680/319524 objects degraded (13.044%); 33981/319524 objects misplaced (10.635%); 18 MiB/s, 289 objects/s recovering 2024-08-29T15:18:56.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:56.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:56.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:56.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:56.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:56.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:56.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:56.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:56.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:56.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:57.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:57.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:57.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:57.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:57.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:57.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 41680/319524 objects degraded (13.044%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 41680/319524 objects degraded (13.044%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:57.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:57.408 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:57.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:57.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 41680/319524 objects degraded (13.044%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:18:57.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:57.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:57.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:57.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:57.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:57.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:57.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:18:57.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:57.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:57.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:18:58.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:58.044 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:58.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:58.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:58.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.171 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:58.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:58.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:58.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:58 smithi027 ceph-mon[93921]: pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 41680/319248 objects degraded (13.056%); 33964/319248 objects misplaced (10.639%); 15 MiB/s, 240 objects/s recovering 2024-08-29T15:18:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:58 smithi027 ceph-mon[98970]: pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 41680/319248 objects degraded (13.056%); 33964/319248 objects misplaced (10.639%); 15 MiB/s, 240 objects/s recovering 2024-08-29T15:18:58.396 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:18:58.419 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:58.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:18:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:58 smithi171 ceph-mon[90031]: pgmap v660: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 41680/319248 objects degraded (13.056%); 33964/319248 objects misplaced (10.639%); 15 MiB/s, 240 objects/s recovering 2024-08-29T15:18:58.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:58.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:58.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:58.646 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:58.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:58.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:58.793 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:58.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:58.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:58.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:58.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:59.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:59.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:59.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:18:59.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:18:59.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:59.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:18:59.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:59.290 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:18:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:18:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:59.320 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:18:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:18:59] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-08-29T15:18:59.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:18:59.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:59.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:18:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:18:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:18:59.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:18:59.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:59.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:18:59.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:18:59.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:18:59.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:18:59.799 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:18:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:18:59.901 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:18:59.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:18:59.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:18:59.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:00.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:00.048 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:00.150 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:00.171 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:00.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:00.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:00.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:00 smithi027 ceph-mon[93921]: pgmap v661: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 41122/318915 objects degraded (12.894%); 33359/318915 objects misplaced (10.460%); 18 MiB/s, 319 objects/s recovering 2024-08-29T15:19:00.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:00 smithi027 ceph-mon[98970]: pgmap v661: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 41122/318915 objects degraded (12.894%); 33359/318915 objects misplaced (10.460%); 18 MiB/s, 319 objects/s recovering 2024-08-29T15:19:00.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:00.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:00 smithi171 ceph-mon[90031]: pgmap v661: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 41122/318915 objects degraded (12.894%); 33359/318915 objects misplaced (10.460%); 18 MiB/s, 319 objects/s recovering 2024-08-29T15:19:00.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:00.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:00.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:00.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:00.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:00.674 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:00.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:00.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:00.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:00.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:00.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:00.926 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:00.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:00.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:01.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.054 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:01.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:01.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:01.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:01.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:01.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:01.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:01 smithi027 ceph-mon[93921]: osdmap e224: 8 total, 8 up, 8 in 2024-08-29T15:19:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:01 smithi027 ceph-mon[98970]: osdmap e224: 8 total, 8 up, 8 in 2024-08-29T15:19:01.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:01.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:01.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:01 smithi171 ceph-mon[90031]: osdmap e224: 8 total, 8 up, 8 in 2024-08-29T15:19:01.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:01.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:01.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:01.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:01.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:01.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:01.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:01.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:01.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:01.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:01.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:02.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:02.059 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:02.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:02.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:02.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:02.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[93921]: pgmap v663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 33252/318558 objects degraded (10.438%); 39931/318558 objects misplaced (12.535%); 12 MiB/s, 256 objects/s recovering 2024-08-29T15:19:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[93921]: 6.2 continuing backfill to osd.6 from (176'2199,223'8960] MIN to 223'8960 2024-08-29T15:19:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[93921]: osdmap e225: 8 total, 8 up, 8 in 2024-08-29T15:19:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 33252/318558 objects degraded (10.438%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[98970]: pgmap v663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 33252/318558 objects degraded (10.438%); 39931/318558 objects misplaced (12.535%); 12 MiB/s, 256 objects/s recovering 2024-08-29T15:19:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[98970]: 6.2 continuing backfill to osd.6 from (176'2199,223'8960] MIN to 223'8960 2024-08-29T15:19:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[98970]: osdmap e225: 8 total, 8 up, 8 in 2024-08-29T15:19:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 33252/318558 objects degraded (10.438%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:02.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:02.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:02.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:02 smithi171 ceph-mon[90031]: pgmap v663: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 32 op/s; 33252/318558 objects degraded (10.438%); 39931/318558 objects misplaced (12.535%); 12 MiB/s, 256 objects/s recovering 2024-08-29T15:19:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:02 smithi171 ceph-mon[90031]: 6.2 continuing backfill to osd.6 from (176'2199,223'8960] MIN to 223'8960 2024-08-29T15:19:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:02 smithi171 ceph-mon[90031]: osdmap e225: 8 total, 8 up, 8 in 2024-08-29T15:19:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 33252/318558 objects degraded (10.438%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:02.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:02.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:02.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:02.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:02.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:02.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:02.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:02.931 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:19:02.931 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:19:02.931 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:19:02.931 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:19:02.931 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:19:02.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:02.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:02.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:03.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:03.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:03.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:03.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:03.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:03.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:03.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:03.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:03.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:03.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:03.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:19:03.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:03.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:03.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:03.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:03.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:03.845 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:03.947 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:03.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:03.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:04.020 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:04.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:04.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:04.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:04.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:04.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:04.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:04.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:04 smithi027 ceph-mon[98970]: pgmap v665: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 28633/317205 objects degraded (9.027%); 49793/317205 objects misplaced (15.697%); 5.2 MiB/s, 247 objects/s recovering 2024-08-29T15:19:04.373 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:04 smithi027 ceph-mon[93921]: pgmap v665: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 28633/317205 objects degraded (9.027%); 49793/317205 objects misplaced (15.697%); 5.2 MiB/s, 247 objects/s recovering 2024-08-29T15:19:04.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:04.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:04 smithi171 ceph-mon[90031]: pgmap v665: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 28633/317205 objects degraded (9.027%); 49793/317205 objects misplaced (15.697%); 5.2 MiB/s, 247 objects/s recovering 2024-08-29T15:19:04.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:04.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:04.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:04.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:04.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:19:04.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:04.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:04.822 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:04.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:04.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:04.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:04.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:04.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:05.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:05.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:05.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:05.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:05.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:05.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:05.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:05.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:05.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:05.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:05.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:05.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:05.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:05.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:05.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:05.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:05.842 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:05.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:05.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:05.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:05.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:06.072 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.0 to 1 (10000) 2024-08-29T15:19:06.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:06.092 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:06 smithi027 ceph-mon[93921]: pgmap v666: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 28633/316977 objects degraded (9.033%); 49793/316977 objects misplaced (15.709%); 5.2 MiB/s, 247 objects/s recovering 2024-08-29T15:19:06.092 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2764184580' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:19:06.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:06 smithi027 ceph-mon[98970]: pgmap v666: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 28633/316977 objects degraded (9.033%); 49793/316977 objects misplaced (15.709%); 5.2 MiB/s, 247 objects/s recovering 2024-08-29T15:19:06.092 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2764184580' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:19:06.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:06.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:06.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:06.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:06.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:06.362 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:06.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:06.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:06.486 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:06.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:06 smithi171 ceph-mon[90031]: pgmap v666: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 28633/316977 objects degraded (9.033%); 49793/316977 objects misplaced (15.709%); 5.2 MiB/s, 247 objects/s recovering 2024-08-29T15:19:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2764184580' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:19:06.554 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:06.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:06.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:06.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:06.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:06.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:06.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:06.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:06.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:06.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:06.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:06.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:07.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:07.091 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:19:07.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:07.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:07.235 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:07.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2764184580' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:19:07.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[98970]: osdmap e226: 8 total, 8 up, 8 in 2024-08-29T15:19:07.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4082733834' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:19:07.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:19:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/316977 objects degraded (9.033%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2764184580' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:19:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[93921]: osdmap e226: 8 total, 8 up, 8 in 2024-08-29T15:19:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4082733834' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:19:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:19:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/316977 objects degraded (9.033%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:07.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:07.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:07.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:07.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:07.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2764184580' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:19:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:07 smithi171 ceph-mon[90031]: osdmap e226: 8 total, 8 up, 8 in 2024-08-29T15:19:07.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4082733834' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:19:07.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:07 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:19:07.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/316977 objects degraded (9.033%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:07.588 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:07.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:07.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:07.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:07.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:07.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:07.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:07.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:07.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:07.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:08.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:08.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:08.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:08.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:08.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:08.363 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:08.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:08.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:08.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:08 smithi171 ceph-mon[90031]: pgmap v668: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 28633/316548 objects degraded (9.045%); 49432/316548 objects misplaced (15.616%); 3.9 MiB/s, 187 objects/s recovering 2024-08-29T15:19:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:08 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:08 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:19:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:08 smithi171 ceph-mon[90031]: osdmap e227: 8 total, 8 up, 8 in 2024-08-29T15:19:08.564 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[93921]: pgmap v668: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 28633/316548 objects degraded (9.045%); 49432/316548 objects misplaced (15.616%); 3.9 MiB/s, 187 objects/s recovering 2024-08-29T15:19:08.564 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:08.564 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:19:08.565 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[93921]: osdmap e227: 8 total, 8 up, 8 in 2024-08-29T15:19:08.565 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[98970]: pgmap v668: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 28633/316548 objects degraded (9.045%); 49432/316548 objects misplaced (15.616%); 3.9 MiB/s, 187 objects/s recovering 2024-08-29T15:19:08.565 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:08.565 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:19:08.565 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:08 smithi027 ceph-mon[98970]: osdmap e227: 8 total, 8 up, 8 in 2024-08-29T15:19:08.590 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:08.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:08.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:08.715 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:08.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:08.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:08.864 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:08.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:08.988 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:08.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:08.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:09.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:09.112 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:09.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:09.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:09.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:09.239 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:09.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:09.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:09.267 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:19:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:19:09] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-08-29T15:19:09.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:09.364 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:09.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:09.466 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:09.490 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:09.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:09.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:09.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:09.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:09.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:09.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:09.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:09.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:09.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:09.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:09.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:09.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:09.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62785 5.3 5:c3da9dee:::benchmark_data_smithi027_135490_object62784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 489.23894473399997, 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.23727498499999999, 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62785 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698061+0000", 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698064+0000", 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2879999999999999e-06 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698106+0000", 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2039999999999997e-05 2024-08-29T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698128+0000", 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2064000000000001e-05 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.698136+0000", 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3790000000000006e-06 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.700871+0000", 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002734972 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.701013+0000", 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141607 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.701141+0000", 2024-08-29T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012853499999999999 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.704979+0000", 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038374239999999999 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.716497+0000", 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011518188 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935036+0000", 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.218538912 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935095+0000", 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8857000000000002e-05 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.935336+0000", 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024171899999999999 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62814 5.3 5:c9d52f91:::benchmark_data_smithi027_135490_object62813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 489.21052133900002, 2024-08-29T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.210827087, 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62814 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726485+0000", 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726486+0000", 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5e-06 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726507+0000", 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0245e-05 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726519+0000", 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2074e-05 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726526+0000", 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7980000000000006e-06 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726593+0000", 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6962e-05 2024-08-29T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726740+0000", 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014641200000000001 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.726858+0000", 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011843 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.728600+0000", 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017419340000000001 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.728790+0000", 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018932899999999999 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937147+0000", 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20835720599999999 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937263+0000", 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116499 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.937312+0000", 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8698e-05 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62826 5.3 5:c75bdcb4:::benchmark_data_smithi027_135490_object62825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 489.185703918, 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.188340805, 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62826 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751302+0000", 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751303+0000", 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.201e-06 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751324+0000", 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0271e-05 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751356+0000", 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2428999999999997e-05 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751364+0000", 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9039999999999992e-06 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751427+0000", 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3304000000000002e-05 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751579+0000", 2024-08-29T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015205000000000001 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.751757+0000", 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017760700000000001 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.754471+0000", 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027138589999999999 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.756577+0000", 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021064460000000001 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939494+0000", 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18291691700000001 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939598+0000", 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010330700000000001 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939643+0000", 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5510000000000003e-05 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62843 5.3 5:c10518e0:::benchmark_data_smithi027_135490_object62842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 489.17473350699998, 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17756435800000001, 2024-08-29T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62843 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762273+0000", 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762274+0000", 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.116e-06 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762294+0000", 2024-08-29T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9743000000000001e-05 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762308+0000", 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4959e-05 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762316+0000", 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1829999999999997e-06 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762372+0000", 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6172999999999999e-05 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762497+0000", 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012541700000000001 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.762631+0000", 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013369200000000001 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.765559+0000", 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029280959999999998 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.765658+0000", 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8919999999999995e-05 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939688+0000", 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17403020299999999 2024-08-29T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939785+0000", 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6971000000000004e-05 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939837+0000", 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1885000000000002e-05 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:62883 5.3 5:c29effb9:::benchmark_data_smithi027_135490_object62882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 489.14826461899997, 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15126415900000001, 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 62883 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788742+0000", 2024-08-29T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788744+0000", 2024-08-29T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5919999999999999e-06 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788770+0000", 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5687999999999999e-05 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788788+0000", 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8099000000000001e-05 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788798+0000", 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7659999999999993e-06 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.788890+0000", 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2724000000000002e-05 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.789064+0000", 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000174048 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.789215+0000", 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015093600000000001 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.791500+0000", 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00228502 2024-08-29T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.791885+0000", 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038479299999999997 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939879+0000", 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14799395700000001 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.939965+0000", 2024-08-29T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6304999999999994e-05 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:00.940006+0000", 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0231000000000002e-05 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68893 5.5 5:a47e561b:::benchmark_data_smithi027_135490_object68892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 485.03545349400002, 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14711858999999999, 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68893 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901553+0000", 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901554+0000", 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.24e-06 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901576+0000", 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2155999999999999e-05 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901587+0000", 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1258e-05 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.901596+0000", 2024-08-29T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3930000000000008e-06 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904408+0000", 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028119130000000001 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904518+0000", 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010994500000000001 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904644+0000", 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012621299999999999 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907190+0000", 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025458640000000001 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048542+0000", 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.141352177 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048577+0000", 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4686e-05 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048622+0000", 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5382e-05 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.048671+0000", 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9363000000000002e-05 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68895 5.3 5:c72cbd73:::benchmark_data_smithi027_135490_object68894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 485.03482851199999, 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.151393162, 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68895 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902178+0000", 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902179+0000", 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8700000000000004e-07 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902204+0000", 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5474999999999999e-05 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902216+0000", 2024-08-29T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2199e-05 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902224+0000", 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1289999999999996e-06 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904177+0000", 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00195218 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904303+0000", 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012648 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.904431+0000", 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012815400000000001 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909174+0000", 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047423450000000002 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.909243+0000", 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9497000000000002e-05 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053476+0000", 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.144233477 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053526+0000", 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9477000000000001e-05 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.053571+0000", 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4861999999999998e-05 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68897 5.5 5:adfd8055:::benchmark_data_smithi027_135490_object68896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 485.03472657899999, 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.147127652, 2024-08-29T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68897 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902280+0000", 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7599999999999997e-07 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902303+0000", 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2676000000000001e-05 2024-08-29T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902313+0000", 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7529999999999996e-06 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.902319+0000", 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6290000000000004e-06 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907215+0000", 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048957189999999998 2024-08-29T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907355+0000", 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139775 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.907500+0000", 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014554800000000001 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049212+0000", 2024-08-29T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14171125700000001 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049280+0000", 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8319999999999996e-05 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049292+0000", 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1849e-05 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049363+0000", 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1616e-05 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.049407+0000", 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3934000000000003e-05 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239396 5.3 5:c28abb26:::benchmark_data_smithi027_135490_object105644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 304.19403023299998, 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4671054730000002, 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239396 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742978+0000", 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.674e-06 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742984+0000", 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7530000000000004e-06 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742995+0000", 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0967e-05 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.743005+0000", 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4690000000000003e-06 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.751912+0000", 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008907491 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.751924+0000", 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1246999999999999e-05 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692438+0000", 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94051403300000003 2024-08-29T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692450+0000", 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.269e-05 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.176340+0000", 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.483889434 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.176472+0000", 2024-08-29T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013255499999999999 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.177116+0000", 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064412200000000001 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.201804+0000", 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024687298999999999 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.201859+0000", 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5201000000000002e-05 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.206234+0000", 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043748629999999997 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.209955+0000", 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037214819999999999 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.209993+0000", 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7898999999999997e-05 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210081+0000", 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8294000000000005e-05 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239400 5.3 5:c91c85c5:::benchmark_data_smithi027_135490_object105648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 304.17865977100001, 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4617042850000002, 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239400 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758348+0000", 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4130000000000001e-06 2024-08-29T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758354+0000", 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0510000000000002e-06 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758382+0000", 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7864999999999998e-05 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758391+0000", 2024-08-29T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2390000000000008e-06 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758468+0000", 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7152999999999998e-05 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758481+0000", 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2927e-05 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692477+0000", 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93399591900000001 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692485+0000", 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6340000000000001e-06 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210333+0000", 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5178479149999999 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210442+0000", 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108992 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210625+0000", 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018322499999999999 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219736+0000", 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0091110099999999992 2024-08-29T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219805+0000", 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8861000000000004e-05 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219816+0000", 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1392999999999999e-05 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219858+0000", 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2416000000000002e-05 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219940+0000", 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1883000000000002e-05 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220051+0000", 2024-08-29T15:19:10.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110389 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239422 5.3 5:cc7358ac:::benchmark_data_smithi027_135490_object105670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 301.91723508699999, 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21943114, 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.012 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239422 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:19:10.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019773+0000", 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2479999999999999e-06 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019782+0000", 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0650000000000005e-06 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019802+0000", 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9653999999999999e-05 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019811+0000", 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4369999999999996e-06 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019893+0000", 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1891999999999995e-05 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019903+0000", 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0003999999999999e-05 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220071+0000", 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20016803999999999 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220151+0000", 2024-08-29T15:19:10.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9785999999999998e-05 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220220+0000", 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8520000000000001e-05 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228770+0000", 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085506139999999998 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228784+0000", 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3207000000000001e-05 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228816+0000", 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2806999999999999e-05 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239075+0000", 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010258751 2024-08-29T15:19:10.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239111+0000", 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5917999999999997e-05 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239202+0000", 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1197000000000001e-05 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.019 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239427 5.3 5:cac43ec6:::benchmark_data_smithi027_135490_object105675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 301.89428103900002, 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.197921239, 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239427 2024-08-29T15:19:10.020 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042726+0000", 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.215e-06 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042730+0000", 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9469999999999996e-06 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042741+0000", 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0995999999999999e-05 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042748+0000", 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.652e-06 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042842+0000", 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4467000000000002e-05 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042852+0000", 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9080000000000007e-06 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.223936+0000", 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18108415 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.224043+0000", 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106591 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.224167+0000", 2024-08-29T15:19:10.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012385599999999999 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228861+0000", 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046937619999999998 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228899+0000", 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8325000000000003e-05 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239219+0000", 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010320424999999999 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240479+0000", 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001259825 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240537+0000", 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7386000000000002e-05 2024-08-29T15:19:10.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240646+0000", 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109734 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239430 5.3 5:c7a58ecd:::benchmark_data_smithi027_135490_object105678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 301.88391400099999, 2024-08-29T15:19:10.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18776626399999999, 2024-08-29T15:19:10.028 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239430 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053094+0000", 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4100000000000001e-06 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053100+0000", 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2979999999999997e-06 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053111+0000", 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0664e-05 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053117+0000", 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7479999999999996e-06 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053200+0000", 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2795999999999998e-05 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053210+0000", 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0399e-05 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228935+0000", 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.175724134 2024-08-29T15:19:10.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.229054+0000", 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119704 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.229207+0000", 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015269400000000001 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.033 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240172+0000", 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010965088 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240433+0000", 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026095699999999998 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240446+0000", 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3414000000000001e-05 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240699+0000", 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025234500000000001 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240748+0000", 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9118e-05 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240858+0000", 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011049500000000001 2024-08-29T15:19:10.035 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140284 6.a 6.98e9132a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.763440887, 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59515916599999996, 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:10.036 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140284 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173567+0000", 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4729999999999999e-06 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173572+0000", 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7270000000000004e-06 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173590+0000", 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7442999999999999e-05 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173598+0000", 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2830000000000001e-06 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173700+0000", 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101889 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173709+0000", 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4749999999999993e-06 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768713+0000", 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59500392899999999 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768724+0000", 2024-08-29T15:19:10.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1946999999999999e-05 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140312 6.a 6.b6852f8a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.751922588, 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58367630699999995, 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:10.041 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140312 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:19:10.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5400000000000002e-07 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185087+0000", 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1209999999999998e-06 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185095+0000", 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0530000000000007e-06 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185101+0000", 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0349999999999998e-06 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185153+0000", 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1468999999999998e-05 2024-08-29T15:19:10.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185161+0000", 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9549999999999992e-06 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768751+0000", 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58359058799999997 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768760+0000", 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6319999999999997e-06 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140319 6.a 6.5ae9c7ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.739534435, 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57131815699999999, 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.046 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140319 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197473+0000", 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2270000000000001e-06 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197513+0000", 2024-08-29T15:19:10.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9731000000000003e-05 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197529+0000", 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6337e-05 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197537+0000", 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1890000000000007e-06 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.049 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197704+0000", 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016718599999999999 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197716+0000", 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1282e-05 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768782+0000", 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57106648599999998 2024-08-29T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768790+0000", 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7190000000000001e-06 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140323 6.a 6.def6e63a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.73841355499999, 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57023369499999998, 2024-08-29T15:19:10.051 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140323 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7599999999999996e-07 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198597+0000", 2024-08-29T15:19:10.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9940000000000002e-06 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198611+0000", 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3522e-05 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198617+0000", 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4880000000000004e-06 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198665+0000", 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8040000000000001e-05 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198673+0000", 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1400000000000002e-06 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768811+0000", 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57013830200000004 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768826+0000", 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5432999999999999e-05 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140331 6.a 6.ecf214ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.72350480999999, 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56336548399999997, 2024-08-29T15:19:10.056 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140331 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213503+0000", 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.091e-06 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.058 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213510+0000", 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.861e-06 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213531+0000", 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0811000000000001e-05 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213538+0000", 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2869999999999999e-06 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213591+0000", 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2735999999999998e-05 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213598+0000", 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4330000000000001e-06 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776857+0000", 2024-08-29T15:19:10.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56325978499999996 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776867+0000", 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4800000000000007e-06 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140333 6.a 6.7707feaa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:19:10.061 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.721630937, 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56152404199999995, 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140333 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215377+0000", 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7320000000000001e-06 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215385+0000", 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6839999999999996e-06 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215402+0000", 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7489000000000001e-05 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215411+0000", 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7450000000000001e-06 2024-08-29T15:19:10.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215492+0000", 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0819e-05 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215501+0000", 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6369999999999994e-06 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.065 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776893+0000", 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56139153100000005 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776899+0000", 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4049999999999997e-06 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140336 6.a 6.ee7969ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:10.066 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.718566322, 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.55869965600000004, 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140336 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:10.067 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218442+0000", 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8250000000000001e-06 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218450+0000", 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0690000000000002e-06 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218472+0000", 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2178e-05 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218482+0000", 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0071999999999999e-05 2024-08-29T15:19:10.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218555+0000", 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3465000000000002e-05 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218568+0000", 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2772000000000001e-05 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:10.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.777129+0000", 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.55856027900000005 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.777140+0000", 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0995999999999999e-05 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:10.071 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:19:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:10.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:10.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:10.197 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:10.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:10.292 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:10 smithi027 ceph-mon[98970]: pgmap v670: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 28633/316014 objects degraded (9.061%); 49326/316014 objects misplaced (15.609%); 3.4 MiB/s, 162 objects/s recovering 2024-08-29T15:19:10.292 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:10 smithi027 ceph-mon[93921]: pgmap v670: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 28633/316014 objects degraded (9.061%); 49326/316014 objects misplaced (15.609%); 3.4 MiB/s, 162 objects/s recovering 2024-08-29T15:19:10.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:10.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:10.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:10.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:10.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:10 smithi171 ceph-mon[90031]: pgmap v670: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 28633/316014 objects degraded (9.061%); 49326/316014 objects misplaced (15.609%); 3.4 MiB/s, 162 objects/s recovering 2024-08-29T15:19:10.549 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:10.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:10.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:19:10.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.696 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:10.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:10.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:10.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:10.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:10.947 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:10.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:11.041 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 1 1 2024-08-29T15:19:11.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:11.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:11.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:11.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:11.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:11.285 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:19:11.285 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:19:11.303 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:19:11.303 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:19:11.324 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:19:11.325 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:19:11.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:11.342 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:19:11.343 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:19:11.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:11.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:11.364 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:19:11.364 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:19:11.383 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:19:11.383 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:19:11.409 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:19:11.409 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:19:11.428 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:19:11.428 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:19:11.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:11.484 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:11.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:11.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:11.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:11.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:11.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:11.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:11.858 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:11.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:11.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:11.982 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:11.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:12.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:19:12.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:12.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:12.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:12.240 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:12.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:12.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:12 smithi027 ceph-mon[93921]: pgmap v671: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 57 op/s; 28633/315813 objects degraded (9.066%); 49300/315813 objects misplaced (15.611%); 2.7 MiB/s, 43 objects/s recovering 2024-08-29T15:19:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/26208193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/315813 objects degraded (9.066%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:12 smithi027 ceph-mon[98970]: pgmap v671: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 57 op/s; 28633/315813 objects degraded (9.066%); 49300/315813 objects misplaced (15.611%); 2.7 MiB/s, 43 objects/s recovering 2024-08-29T15:19:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/26208193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:12.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/315813 objects degraded (9.066%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:12.347 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:12.353 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:12.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:12.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:12.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:12.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:12.399 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:12.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:12.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:12.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:12 smithi171 ceph-mon[90031]: pgmap v671: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 57 op/s; 28633/315813 objects degraded (9.066%); 49300/315813 objects misplaced (15.611%); 2.7 MiB/s, 43 objects/s recovering 2024-08-29T15:19:12.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/26208193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:12.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/315813 objects degraded (9.066%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:12.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:12.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:12.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:12.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:12.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:12.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:12.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:12.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:12.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:12.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:12.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:13.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.020 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:13.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:13.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:13.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:13.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:13.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:13.267 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:13.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:13.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:13.391 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:13.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:13.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:13.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.518 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:13.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:13.619 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:13.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:13.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:13.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:13.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:13.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:13.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:13.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:13.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:13.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:13.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:14.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:14.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:14.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:14.244 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:14.266 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:14.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:14.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:14.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:14.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:14.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:14 smithi027 ceph-mon[93921]: pgmap v672: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 28633/314985 objects degraded (9.090%); 48874/314985 objects misplaced (15.516%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:19:14.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:14.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:14 smithi027 ceph-mon[98970]: pgmap v672: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 28633/314985 objects degraded (9.090%); 48874/314985 objects misplaced (15.516%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:19:14.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:14 smithi171 ceph-mon[90031]: pgmap v672: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 28633/314985 objects degraded (9.090%); 48874/314985 objects misplaced (15.516%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:19:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:14.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:14.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:14.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:14.642 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:14.744 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:14.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:14.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:14.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:14.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:14.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:14.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:15.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:15.122 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:15.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:15.246 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:15.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:15.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:15.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:15.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:15.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:15.518 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:15.620 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:15.642 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:15.744 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:15.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:15.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:15.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:15.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:15.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:15.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:16.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:16.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:16.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:16.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:16.241 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:16.264 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:16.351 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:16 smithi027 ceph-mon[93921]: pgmap v673: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 28633/314853 objects degraded (9.094%); 48874/314853 objects misplaced (15.523%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:19:16.352 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:16 smithi027 ceph-mon[98970]: pgmap v673: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 28633/314853 objects degraded (9.094%); 48874/314853 objects misplaced (15.523%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:19:16.365 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:16.388 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:16.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:16.515 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:16.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:16 smithi171 ceph-mon[90031]: pgmap v673: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 28633/314853 objects degraded (9.094%); 48874/314853 objects misplaced (15.523%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:19:16.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:16.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:16.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:16.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:16.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:16.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:16.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:16.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:16.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:16.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:16.917 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:19:16.917 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:19:16.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:17.018 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:17.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:17.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:17.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:17.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:17.266 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:17.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:17.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:17.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:17.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:17.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/314853 objects degraded (9.094%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:17.546 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/314853 objects degraded (9.094%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:17.547 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/314853 objects degraded (9.094%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:17.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:19:17.618 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:17.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:17.742 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:17.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:19:17.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:17.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:17.888 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:17.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:17.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:18.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:18.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:18.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:18.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:18.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:18.263 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:18.365 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:18.379 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:18 smithi027 ceph-mon[93921]: pgmap v674: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 28633/314259 objects degraded (9.111%); 48485/314259 objects misplaced (15.428%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:19:18.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:18 smithi027 ceph-mon[98970]: pgmap v674: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 28633/314259 objects degraded (9.111%); 48485/314259 objects misplaced (15.428%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:19:18.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:18.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:18.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:18.514 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:18.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:18 smithi171 ceph-mon[90031]: pgmap v674: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 76 op/s; 28633/314259 objects degraded (9.111%); 48485/314259 objects misplaced (15.428%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:19:18.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:18.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:18.722 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:18.734 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:18.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:18.772 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:18.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:18.874 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:18.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:18.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:18.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:18.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:19.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:19.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:19.111 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:19:19.111 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.1 to 1 (10000) 2024-08-29T15:19:19.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:19.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:19.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:19.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:19.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:19.315 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3360261963' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:19.315 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:19.315 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1876363375' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:19:19.315 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:19:19.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3360261963' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:19.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:19.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1876363375' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:19:19.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:19:19.316 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:19:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:19:19] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.51.0" 2024-08-29T15:19:19.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:19.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:19.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:19.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3360261963' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:19 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1876363375' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:19:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:19 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:19:19.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:19.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:19.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:19.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:19.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:19.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:19.896 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:19.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:19.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:19.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:20.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:20.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:20.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:20.150 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:20.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:20.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:20.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:20.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:20.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:20.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:20 smithi171 ceph-mon[90031]: pgmap v675: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/313779 objects degraded (9.125%); 48401/313779 objects misplaced (15.425%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:19:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:20 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:19:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:20 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:19:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:20 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:19:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:20 smithi171 ceph-mon[90031]: osdmap e228: 8 total, 8 up, 8 in 2024-08-29T15:19:20.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:20.558 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[93921]: pgmap v675: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/313779 objects degraded (9.125%); 48401/313779 objects misplaced (15.425%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:19:20.558 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:19:20.558 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:19:20.558 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:19:20.558 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[93921]: osdmap e228: 8 total, 8 up, 8 in 2024-08-29T15:19:20.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[98970]: pgmap v675: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/313779 objects degraded (9.125%); 48401/313779 objects misplaced (15.425%); 3.6 MiB/s, 57 objects/s recovering 2024-08-29T15:19:20.559 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:19:20.559 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:19:20.559 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:19:20.559 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:20 smithi027 ceph-mon[98970]: osdmap e228: 8 total, 8 up, 8 in 2024-08-29T15:19:20.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:20.660 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:20.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:20.762 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:20.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:20.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:20.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:20.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:20.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:21.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:21.033 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:21.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:21.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:21.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:21.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:21.283 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:21.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:21.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:21.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:21.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:21.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 2 1 2024-08-29T15:19:21.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:21.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:21.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:21.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:21.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:21.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:21.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:21.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:21.908 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:21.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:21.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:21.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:19:22.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:22.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:22.156 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:22.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:22.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:22.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:22.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:22 smithi027 ceph-mon[98970]: pgmap v677: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 28633/313587 objects degraded (9.131%); 48377/313587 objects misplaced (15.427%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:19:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/313587 objects degraded (9.131%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:22 smithi027 ceph-mon[93921]: pgmap v677: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 28633/313587 objects degraded (9.131%); 48377/313587 objects misplaced (15.427%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:19:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/313587 objects degraded (9.131%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:22.404 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:22.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:22 smithi171 ceph-mon[90031]: pgmap v677: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 28633/313587 objects degraded (9.131%); 48377/313587 objects misplaced (15.427%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:19:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/313587 objects degraded (9.131%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:22.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:22.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:22.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:22.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:22.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:22.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:22.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:22.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:22.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:23.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:23.054 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:23.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:23.094 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:19:23.094 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:19:23.108 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:19:23.109 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:19:23.128 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:19:23.128 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:19:23.146 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:19:23.146 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:19:23.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:23.164 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:19:23.164 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:19:23.181 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:19:23.182 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:19:23.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:23.204 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:19:23.205 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:19:23.223 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:19:23.223 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:19:23.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:23.305 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/33611755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:23.306 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/33611755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:23.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:23.328 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:19:23.328 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:19:23.329 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:19:23.329 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:19:23.329 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:19:23.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:23.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/33611755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:23.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:23.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:23.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:23.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:23.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.706 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:23.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:23.807 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:23.831 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:23.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:23.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:23.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:24.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:24.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:24.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:24.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:24.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:24.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:24.288 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:24 smithi027 ceph-mon[93921]: pgmap v678: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 28633/312498 objects degraded (9.163%); 47894/312498 objects misplaced (15.326%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:19:24.288 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:24 smithi027 ceph-mon[98970]: pgmap v678: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 28633/312498 objects degraded (9.163%); 47894/312498 objects misplaced (15.326%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:19:24.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:24.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:24.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:24.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:24.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:24.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:24 smithi171 ceph-mon[90031]: pgmap v678: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 28633/312498 objects degraded (9.163%); 47894/312498 objects misplaced (15.326%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:19:24.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:24.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:24.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:24.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:24.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:24.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:19:24.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:24.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:24.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:24.873 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:24.876 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:24.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:24.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:24.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:25.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:25.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:25.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:25.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:25.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:25.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:25.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:25.337 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:25.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:25.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:25.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:25.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:25.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:25.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:25.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:25.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:25.815 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:25.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:25.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:25.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:25.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:25.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:25.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:19:26.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:26.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:26.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:26.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:26.283 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:26 smithi027 ceph-mon[93921]: pgmap v679: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 28633/312300 objects degraded (9.168%); 47894/312300 objects misplaced (15.336%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:19:26.284 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:26 smithi027 ceph-mon[98970]: pgmap v679: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 28633/312300 objects degraded (9.168%); 47894/312300 objects misplaced (15.336%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:19:26.316 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:26.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:26.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:26.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:26 smithi171 ceph-mon[90031]: pgmap v679: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 85 op/s; 28633/312300 objects degraded (9.168%); 47894/312300 objects misplaced (15.336%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:19:26.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:26.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:26.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:26.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:26.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:26.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:26.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:26.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:26.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:26.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:26.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:27.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:27.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:27.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:27.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:27.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:27.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:27.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/312300 objects degraded (9.168%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/312300 objects degraded (9.168%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:27.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:27.388 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:27.394 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:27.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:27.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:27.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/312300 objects degraded (9.168%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:27.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:27.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:27.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:27.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:27.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:27.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:27.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:27.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:28.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:28.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:28.139 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:19:28.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:28.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:28.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:28.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:28.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:28.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:28.492 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:28 smithi027 ceph-mon[98970]: pgmap v680: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 28633/311943 objects degraded (9.179%); 47504/311943 objects misplaced (15.228%); 4.4 MiB/s, 70 objects/s recovering 2024-08-29T15:19:28.492 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2763905490' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:19:28.492 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:28 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:19:28.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:28 smithi027 ceph-mon[93921]: pgmap v680: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 28633/311943 objects degraded (9.179%); 47504/311943 objects misplaced (15.228%); 4.4 MiB/s, 70 objects/s recovering 2024-08-29T15:19:28.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2763905490' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:19:28.492 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:28 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:19:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:28 smithi171 ceph-mon[90031]: pgmap v680: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 28633/311943 objects degraded (9.179%); 47504/311943 objects misplaced (15.228%); 4.4 MiB/s, 70 objects/s recovering 2024-08-29T15:19:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2763905490' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:19:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:28 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:19:28.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:28.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:28.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:28.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:28.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:28.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:28.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:28.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:28.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:28.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:28.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:29.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:29.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:29.125 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:19:29] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-08-29T15:19:29.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:29.214 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:29.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:29.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:29.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:29.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:19:29.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[98970]: osdmap e229: 8 total, 8 up, 8 in 2024-08-29T15:19:29.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:29.403 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:29.403 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:19:29.403 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[93921]: osdmap e229: 8 total, 8 up, 8 in 2024-08-29T15:19:29.404 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:29.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:29.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:29.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:29 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:29 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:19:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:29 smithi171 ceph-mon[90031]: osdmap e229: 8 total, 8 up, 8 in 2024-08-29T15:19:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:29.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:29.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:29.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:29.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:29.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:29.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:29.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:29.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:29.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:29.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:29.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:30.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:30.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:30.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:30.142 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.2 to 1 (10000) 2024-08-29T15:19:30.167 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:30 smithi027 ceph-mon[93921]: pgmap v682: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 28633/311472 objects degraded (9.193%); 47417/311472 objects misplaced (15.224%); 4.7 MiB/s, 74 objects/s recovering 2024-08-29T15:19:30.167 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2146979293' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:19:30.167 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:19:30.167 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:30 smithi027 ceph-mon[98970]: pgmap v682: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 28633/311472 objects degraded (9.193%); 47417/311472 objects misplaced (15.224%); 4.7 MiB/s, 74 objects/s recovering 2024-08-29T15:19:30.167 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2146979293' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:19:30.167 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:19:30.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:30.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:30.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:30.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:30.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:30.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:30 smithi171 ceph-mon[90031]: pgmap v682: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 28633/311472 objects degraded (9.193%); 47417/311472 objects misplaced (15.224%); 4.7 MiB/s, 74 objects/s recovering 2024-08-29T15:19:30.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2146979293' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:19:30.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:19:30.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:30.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:30.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:30.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:30.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:30.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:30.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:30.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:30.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:30.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:30.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:30.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:30.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:30.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:31.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 3 1 2024-08-29T15:19:31.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:31.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:31.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:31.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:31.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:31.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:19:31.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:31 smithi027 ceph-mon[98970]: osdmap e230: 8 total, 8 up, 8 in 2024-08-29T15:19:31.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:19:31.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:31 smithi027 ceph-mon[93921]: osdmap e230: 8 total, 8 up, 8 in 2024-08-29T15:19:31.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:31.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:31.460 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:19:31.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:31 smithi171 ceph-mon[90031]: osdmap e230: 8 total, 8 up, 8 in 2024-08-29T15:19:31.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:31.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:31.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:31.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:31.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:31.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:31.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:31.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:31.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:31.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:32.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:32.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:32.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:32.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:32.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:32.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:32.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:32.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:32 smithi027 ceph-mon[93921]: pgmap v684: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 48 op/s; 28633/311325 objects degraded (9.197%); 47417/311325 objects misplaced (15.231%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:19:32.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/311325 objects degraded (9.197%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:32 smithi027 ceph-mon[98970]: pgmap v684: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 48 op/s; 28633/311325 objects degraded (9.197%); 47417/311325 objects misplaced (15.231%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:19:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/311325 objects degraded (9.197%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:32.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:32.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:32.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:32.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:32.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:32.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:32 smithi171 ceph-mon[90031]: pgmap v684: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 48 op/s; 28633/311325 objects degraded (9.197%); 47417/311325 objects misplaced (15.231%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:19:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/311325 objects degraded (9.197%), 9 pgs degraded (PG_DEGRADED) 2024-08-29T15:19:32.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:32.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:32.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:32.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:32.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:32.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:32.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:32.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:32.845 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:32.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68914 5.c 5:3b301fb0:::benchmark_data_smithi027_135490_object68913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.916458+0000", 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 507.945973735, 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13917643499999999, 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.866 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68914 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916458+0000", 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916458+0000", 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916459+0000", 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.333e-06 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916479+0000", 2024-08-29T15:19:32.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0097000000000001e-05 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916490+0000", 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0786e-05 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916496+0000", 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5690000000000004e-06 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916540+0000", 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3767999999999998e-05 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916626+0000", 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6602999999999997e-05 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:19:32.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.916730+0000", 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104308 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.918642+0000", 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019117649999999999 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.918979+0000", 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033689100000000002 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055434+0000", 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13645533700000001 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055589+0000", 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015502100000000001 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055634+0000", 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4956999999999997e-05 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68920 5.c 5:320cd6cb:::benchmark_data_smithi027_135490_object68919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.921040+0000", 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 507.94139205099998, 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134813617, 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.873 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68920 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921040+0000", 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921040+0000", 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921041+0000", 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3740000000000001e-06 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921061+0000", 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9623000000000002e-05 2024-08-29T15:19:32.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921071+0000", 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0229e-05 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921076+0000", 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4779999999999999e-06 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921121+0000", 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4415000000000001e-05 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921247+0000", 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012585 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.921345+0000", 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8319999999999994e-05 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.923113+0000", 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001768071 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.924140+0000", 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010273820000000001 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055681+0000", 2024-08-29T15:19:32.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.131540405 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055808+0000", 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127222 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.055853+0000", 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5247999999999998e-05 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.879 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:68927 5.c 5:3864dffc:::benchmark_data_smithi027_135490_object68926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:11:04.929290+0000", 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 507.93314217900001, 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.127866969, 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.880 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68927 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929290+0000", 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929290+0000", 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929291+0000", 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5149999999999999e-06 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929311+0000", 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9524000000000001e-05 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929320+0000", 2024-08-29T15:19:32.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2070000000000001e-06 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929325+0000", 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2360000000000004e-06 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929368+0000", 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3155999999999998e-05 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929457+0000", 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8553000000000004e-05 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.929549+0000", 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2368999999999996e-05 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.931187+0000", 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001637388 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:04.931438+0000", 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025143000000000002 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.057018+0000", 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12558042799999999 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.057071+0000", 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3007000000000003e-05 2024-08-29T15:19:32.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:11:05.057157+0000", 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5155999999999996e-05 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:237081 5.0 e137/111 5:0dda9191:::benchmark_data_smithi027_135490_object103329:head v 137'14454, mlcod=137'14452)", 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.798340+0000", 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 330.06409125200003, 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11104399, 2024-08-29T15:19:32.886 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798340+0000", 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798340+0000", 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798341+0000", 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5899999999999995e-07 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798348+0000", 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4749999999999998e-06 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798358+0000", 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.063e-05 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798365+0000", 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5440000000000004e-06 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798459+0000", 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4293e-05 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.798477+0000", 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7669999999999999e-05 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909340+0000", 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.110863032 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909384+0000", 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4487000000000003e-05 2024-08-29T15:19:32.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:237080 5.8 e137/111 5:144f62ea:::benchmark_data_smithi027_135490_object103328:head v 137'14601, mlcod=137'14597)", 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.802875+0000", 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 330.059556714, 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.106458555, 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802875+0000", 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802875+0000", 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802877+0000", 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5039999999999999e-06 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802884+0000", 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9440000000000005e-06 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802898+0000", 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3244999999999999e-05 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802906+0000", 2024-08-29T15:19:32.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3149999999999992e-06 2024-08-29T15:19:32.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802966+0000", 2024-08-29T15:19:32.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0229000000000002e-05 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.802988+0000", 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1809000000000001e-05 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.813444+0000", 2024-08-29T15:19:32.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010456045000000001 2024-08-29T15:19:32.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909334+0000", 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.095889463999999994 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44573.0:237085 5.0 e137/111 5:02701778:::benchmark_data_smithi027_135490_object103333:head v 137'14455, mlcod=137'14452)", 2024-08-29T15:19:32.897 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.804867+0000", 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 330.057564458, 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10456966199999999, 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804867+0000", 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804867+0000", 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804868+0000", 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8299999999999995e-07 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804877+0000", 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3100000000000001e-06 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804889+0000", 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2812e-05 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.804897+0000", 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7039999999999994e-06 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.809114+0000", 2024-08-29T15:19:32.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042168689999999998 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.809133+0000", 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8963000000000001e-05 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909387+0000", 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.100254073 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909437+0000", 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9947999999999997e-05 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:237090 5.c 5:357a930e:::benchmark_data_smithi027_135490_object103338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e137)", 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:02.807682+0000", 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 330.05475022399997, 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.106071554, 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.902 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 237090 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807682+0000", 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807682+0000", 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807684+0000", 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0109999999999999e-06 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807691+0000", 2024-08-29T15:19:32.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3710000000000002e-06 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807709+0000", 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7995e-05 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807718+0000", 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1200000000000008e-06 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807787+0000", 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9233999999999995e-05 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.807924+0000", 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013661200000000001 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.808040+0000", 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115635 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.812900+0000", 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048600780000000003 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.909344+0000", 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.096444385999999993 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.913576+0000", 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042316749999999998 2024-08-29T15:19:32.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.913686+0000", 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011013100000000001 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:02.913753+0000", 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7305999999999996e-05 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.908 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239392 5.7 5:e745e56f:::benchmark_data_smithi027_135490_object105640:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.749988+0000", 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 327.11244378499998, 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1819669820000001, 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.909 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239392 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749988+0000", 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749988+0000", 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749989+0000", 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3939999999999999e-06 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.749994+0000", 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1670000000000001e-06 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750008+0000", 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4151999999999999e-05 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750018+0000", 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0678e-05 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754784+0000", 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047659549999999997 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754794+0000", 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0009e-05 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700248+0000", 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94545362099999997 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700260+0000", 2024-08-29T15:19:32.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1853e-05 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.928959+0000", 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2286991270000001 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.931955+0000", 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029960260000000002 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.914 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239398 5.7 5:ed13b1ac:::benchmark_data_smithi027_135490_object105646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.750029+0000", 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 327.112402374, 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2354920040000001, 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239398 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750029+0000", 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750029+0000", 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750030+0000", 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6400000000000002e-07 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750033+0000", 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6850000000000001e-06 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750040+0000", 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.750045+0000", 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2939999999999996e-06 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754817+0000", 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047715550000000002 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.754839+0000", 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2296000000000001e-05 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700288+0000", 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94544935299999999 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.700297+0000", 2024-08-29T15:19:32.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1990000000000006e-06 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.932308+0000", 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.232011145 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933364+0000", 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001056476 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4,5", 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.933939+0000", 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00057509599999999998 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.961786+0000", 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027846619 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.965789+0000", 2024-08-29T15:19:32.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040033589999999997 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.983390+0000", 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017600437999999999 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.985244+0000", 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001854198 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.985391+0000", 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014689600000000001 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:07.985521+0000", 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013052300000000001 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:242791 5.7 5:e93ed9d7:::benchmark_data_smithi027_135490_object109039:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-29T15:19:32.923 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:12.856201+0000", 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 320.006230561, 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13081316600000001, 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 242791 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856201+0000", 2024-08-29T15:19:32.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856201+0000", 2024-08-29T15:19:32.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856203+0000", 2024-08-29T15:19:32.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7549999999999999e-06 2024-08-29T15:19:32.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856207+0000", 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5349999999999998e-06 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856218+0000", 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0399e-05 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.856224+0000", 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4910000000000002e-06 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865071+0000", 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088467279999999999 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865215+0000", 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014368000000000001 2024-08-29T15:19:32.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4,5", 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865360+0000", 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144938 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.869315+0000", 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039552889999999999 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.876209+0000", 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068940850000000003 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.877487+0000", 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001277891 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.986727+0000", 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.109240306 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.986887+0000", 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015986400000000001 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.987014+0000", 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127205 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:242793 5.c 5:30095f22:::benchmark_data_smithi027_135490_object109041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:12.865875+0000", 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 319.99655660600001, 2024-08-29T15:19:32.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.102709334, 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 242793 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865875+0000", 2024-08-29T15:19:32.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865875+0000", 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865876+0000", 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3039999999999999e-06 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865882+0000", 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2109999999999997e-06 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865898+0000", 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6033e-05 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.865905+0000", 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5809999999999999e-06 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.866112+0000", 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000206405 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.866211+0000", 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9601999999999999e-05 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.866315+0000", 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103751 2024-08-29T15:19:32.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.871304+0000", 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049892490000000003 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.879733+0000", 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084289050000000004 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.968476+0000", 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.088742677000000006 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.968538+0000", 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2026999999999994e-05 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:12.968584+0000", 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6588999999999997e-05 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.940 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:68822 6.1 6:86e313fb:::benchmark_data_smithi027_169445_object68821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.49602382099999, 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.107834163, 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68822 2024-08-29T15:19:32.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366410+0000", 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8899999999999999e-06 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366438+0000", 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8589e-05 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366453+0000", 2024-08-29T15:19:32.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5085e-05 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366463+0000", 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1179999999999998e-06 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366622+0000", 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000159746 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366765+0000", 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000142237 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366907+0000", 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014195399999999999 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.369648+0000", 2024-08-29T15:19:32.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002740957 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.370870+0000", 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001222835 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474094+0000", 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10322338 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474180+0000", 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6434999999999995e-05 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474242+0000", 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1937e-05 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.947 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124391 6.1 6:80fa3010:::benchmark_data_smithi027_169445_object124390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 154.85123698800001, 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1538895120000001, 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124391 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011196+0000", 2024-08-29T15:19:32.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3300000000000001e-07 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011216+0000", 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0068000000000002e-05 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011228+0000", 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.243e-05 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011234+0000", 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2890000000000003e-06 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.018714+0000", 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007479388 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.020057+0000", 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001342768 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.020202+0000", 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014581300000000001 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.035616+0000", 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015413736000000001 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.035626+0000", 2024-08-29T15:19:32.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0301e-05 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025526+0000", 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98989923499999999 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025540+0000", 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4188e-05 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.164991+0000", 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139451225 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165084+0000", 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3238000000000004e-05 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.954 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124405 6.1 6:81409361:::benchmark_data_smithi027_169445_object124404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 154.83464362999999, 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1459181940000001, 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124405 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027789+0000", 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.24e-06 2024-08-29T15:19:32.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027810+0000", 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0891999999999999e-05 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027833+0000", 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3217000000000001e-05 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027843+0000", 2024-08-29T15:19:32.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1419999999999999e-06 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027929+0000", 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6318999999999998e-05 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.028098+0000", 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016911500000000001 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.028250+0000", 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015169000000000001 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.036231+0000", 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079816360000000003 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.036241+0000", 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9119999999999993e-06 2024-08-29T15:19:32.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025573+0000", 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98933212100000001 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025582+0000", 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2900000000000002e-06 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165270+0000", 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1396881759999999 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165781+0000", 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00051096900000000001 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173600+0000", 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0078194070000000004 2024-08-29T15:19:32.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173706+0000", 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106068 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124438 6.1 6:8a3680bc:::benchmark_data_smithi027_169445_object124437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 154.792729952, 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.104159272, 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.963 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124438 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:19:32.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069704+0000", 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.719e-06 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069728+0000", 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4142999999999999e-05 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069747+0000", 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9177e-05 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069755+0000", 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4859999999999997e-06 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069844+0000", 2024-08-29T15:19:32.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8729999999999999e-05 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069853+0000", 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0489999999999993e-06 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025603+0000", 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.955749607 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025609+0000", 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5390000000000004e-06 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165847+0000", 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1402372869999999 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165953+0000", 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106016 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.166064+0000", 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111908 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.168561+0000", 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024969829999999999 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.169970+0000", 2024-08-29T15:19:32.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014081149999999999 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173730+0000", 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037604190000000001 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173779+0000", 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9508000000000001e-05 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173861+0000", 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1586000000000001e-05 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140304 6.4 6:2a4ae72d:::benchmark_data_smithi027_169445_object14877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.68273000600001, 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63546416900000002, 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.971 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140304 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179704+0000", 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0020000000000001e-06 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179709+0000", 2024-08-29T15:19:32.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6659999999999996e-06 2024-08-29T15:19:32.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179727+0000", 2024-08-29T15:19:32.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7693000000000002e-05 2024-08-29T15:19:32.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:32.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179735+0000", 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7819999999999994e-06 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179797+0000", 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1838e-05 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179805+0000", 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3289999999999994e-06 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810640+0000", 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63083506700000003 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810759+0000", 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011892699999999999 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3", 2024-08-29T15:19:32.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810883+0000", 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124098 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814964+0000", 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040809540000000004 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815029+0000", 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4528000000000003e-05 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815081+0000", 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2151999999999998e-05 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815166+0000", 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5133e-05 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140324 6.d 6:b4e34602:::benchmark_data_smithi027_169445_object14897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.66141934199999, 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.61402909999999999, 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.980 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140324 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201014+0000", 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6649999999999999e-06 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201019+0000", 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0340000000000004e-06 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201033+0000", 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4331000000000001e-05 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201041+0000", 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2200000000000003e-06 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201098+0000", 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7484000000000001e-05 2024-08-29T15:19:32.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201106+0000", 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8949999999999998e-06 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.786966+0000", 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58585960400000003 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.790090+0000", 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003124258 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.790556+0000", 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046574499999999999 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806640+0000", 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016084317000000001 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814905+0000", 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082646869999999997 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814955+0000", 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9997999999999998e-05 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815042+0000", 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6861999999999996e-05 2024-08-29T15:19:32.986 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140335 6.d 6:b3ea1342:::benchmark_data_smithi027_169445_object14908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.64487045999999, 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59763891800000002, 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140335 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217563+0000", 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.88e-06 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217570+0000", 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0049999999999998e-06 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217589+0000", 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8686e-05 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217598+0000", 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6419999999999996e-06 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217665+0000", 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7447000000000007e-05 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217675+0000", 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7799999999999995e-06 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802631+0000", 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58495612900000005 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802735+0000", 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104397 2024-08-29T15:19:32.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802810+0000", 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5044999999999997e-05 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814892+0000", 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01208188 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:32.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815054+0000", 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016216000000000001 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815094+0000", 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.015e-05 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815200+0000", 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105717 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.993 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140337 6.d 6:bf19c6d4:::benchmark_data_smithi027_169445_object14910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.64154477700001, 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59452165300000004, 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:32.994 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140337 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220889+0000", 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.708e-06 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:32.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220896+0000", 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8249999999999999e-06 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220914+0000", 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8476000000000001e-05 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220923+0000", 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1209999999999996e-06 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.221258+0000", 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033503 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.221267+0000", 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7930000000000003e-06 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806696+0000", 2024-08-29T15:19:32.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58542917100000003 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806843+0000", 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000146623 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806962+0000", 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011940200000000001 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:32.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815264+0000", 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083018880000000003 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815277+0000", 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2835e-05 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:33.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815315+0000", 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7991000000000003e-05 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815409+0000", 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3789999999999995e-05 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140341 6.d 6:ba4d184c:::benchmark_data_smithi027_169445_object14914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:19:33.001 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.62736706499999, 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58419336799999999, 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140341 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:19:33.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235067+0000", 2024-08-29T15:19:33.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.984e-06 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235074+0000", 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0840000000000003e-06 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235095+0000", 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1307000000000001e-05 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235106+0000", 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0665999999999999e-05 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235173+0000", 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7220999999999999e-05 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235185+0000", 2024-08-29T15:19:33.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2189e-05 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810910+0000", 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57572489900000001 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.811020+0000", 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110179 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.811153+0000", 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013316200000000001 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815416+0000", 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042625670000000001 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819099+0000", 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003683151 2024-08-29T15:19:33.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819162+0000", 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2558000000000007e-05 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819258+0000", 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6401000000000001e-05 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:33.008 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:19:33.009 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:19:33.009 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:19:33.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:33.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:33.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:33.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:33.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/982720173' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:33.172 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/982720173' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:33.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:33.283 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:33.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:19:33.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:19:33.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:19:33.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:19:33.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:19:33.289 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:19:33.289 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:19:33.289 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:19:33.289 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:19:33.289 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:19:33.290 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:19:33.290 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:19:33.291 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:19:33.291 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:19:33.291 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:19:33.291 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:19:33.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:33.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:33.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:33.532 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/982720173' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:33.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:33.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:33.678 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:33.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:33.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:33.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:33.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:33.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:33.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:33.959 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:19:33.972 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:34.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:34.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:34.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:34.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:34.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:34.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:34.303 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:34.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:34 smithi027 ceph-mon[93921]: pgmap v685: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 28633/310581 objects degraded (9.219%); 46927/310581 objects misplaced (15.109%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:19:34.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:34 smithi027 ceph-mon[98970]: pgmap v685: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 28633/310581 objects degraded (9.219%); 46927/310581 objects misplaced (15.109%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:19:34.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:34.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:34 smithi171 ceph-mon[90031]: pgmap v685: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 71 op/s; 28633/310581 objects degraded (9.219%); 46927/310581 objects misplaced (15.109%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:19:34.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:34.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:19:34.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:34.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:34.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:34.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:34.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:34.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:34.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:34.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:34.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:34.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:34.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:35.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:35.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:35.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:35.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:35.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:35.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:35.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:35.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:35.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:35.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:35.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:35.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:35.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:35.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:35.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:35.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:35.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:35.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:36.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:36.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:36.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:36.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:36.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:36.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:36.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:36.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:36.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:36.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:36 smithi171 ceph-mon[90031]: pgmap v686: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 28633/310422 objects degraded (9.224%); 46927/310422 objects misplaced (15.117%); 3.0 MiB/s, 47 objects/s recovering 2024-08-29T15:19:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:19:36.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:36.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:36.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:36 smithi027 ceph-mon[98970]: pgmap v686: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 28633/310422 objects degraded (9.224%); 46927/310422 objects misplaced (15.117%); 3.0 MiB/s, 47 objects/s recovering 2024-08-29T15:19:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:19:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:36 smithi027 ceph-mon[93921]: pgmap v686: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 63 op/s; 28633/310422 objects degraded (9.224%); 46927/310422 objects misplaced (15.117%); 3.0 MiB/s, 47 objects/s recovering 2024-08-29T15:19:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:19:36.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:36.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:36.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:36.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:36.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:36.814 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:36.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:36.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:36.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:37.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:37.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:37.064 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:37.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:37.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:37.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:37.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:37.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:37.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/310422 objects degraded (9.224%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:37.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/310422 objects degraded (9.224%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:37.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:37.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/310422 objects degraded (9.224%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:37.544 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:37.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:37.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:37.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:37.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:37.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:37.751 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:19:37.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:37.816 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:37.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:37.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:37.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:37.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:38.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:38.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:38.158 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.3 to 1 (10000) 2024-08-29T15:19:38.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:38.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:38.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:38.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:38.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:38.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:38.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:38.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:38 smithi171 ceph-mon[90031]: pgmap v687: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 61 op/s; 28633/309918 objects degraded (9.239%); 46493/309918 objects misplaced (15.002%); 5.6 MiB/s, 90 objects/s recovering 2024-08-29T15:19:38.522 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1765750401' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:19:38.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:38.567 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:38 smithi027 ceph-mon[98970]: pgmap v687: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 61 op/s; 28633/309918 objects degraded (9.239%); 46493/309918 objects misplaced (15.002%); 5.6 MiB/s, 90 objects/s recovering 2024-08-29T15:19:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1765750401' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:19:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:38 smithi027 ceph-mon[93921]: pgmap v687: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 61 op/s; 28633/309918 objects degraded (9.239%); 46493/309918 objects misplaced (15.002%); 5.6 MiB/s, 90 objects/s recovering 2024-08-29T15:19:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1765750401' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:19:38.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:38.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:38.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:38.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:38.824 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:38.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:38.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:38.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:38.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:38.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:39.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:39.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:39.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:39.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:39.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:39.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:39.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:39 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1765750401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:19:39.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:39 smithi027 ceph-mon[93921]: osdmap e231: 8 total, 8 up, 8 in 2024-08-29T15:19:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:39 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1765750401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:19:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:39 smithi027 ceph-mon[98970]: osdmap e231: 8 total, 8 up, 8 in 2024-08-29T15:19:39.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:19:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:19:39] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-08-29T15:19:39.323 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:39.425 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:39.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:39.473 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:39 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1765750401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:19:39.473 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:39 smithi171 ceph-mon[90031]: osdmap e231: 8 total, 8 up, 8 in 2024-08-29T15:19:39.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:39.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:39.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:39.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:39.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:39.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:39.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:39.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:39.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:39.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:39.949 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:39.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:39.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:40.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:40.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:40.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 4 1 2024-08-29T15:19:40.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:40.197 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:40.299 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:40.324 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:40.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:40.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:40.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:40 smithi171 ceph-mon[90031]: pgmap v689: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 28633/309450 objects degraded (9.253%); 46394/309450 objects misplaced (14.992%); 5.6 MiB/s, 90 objects/s recovering 2024-08-29T15:19:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:40 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:40 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:40.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:40 smithi027 ceph-mon[93921]: pgmap v689: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 28633/309450 objects degraded (9.253%); 46394/309450 objects misplaced (14.992%); 5.6 MiB/s, 90 objects/s recovering 2024-08-29T15:19:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:40 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:40 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:40.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:40 smithi027 ceph-mon[98970]: pgmap v689: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 28633/309450 objects degraded (9.253%); 46394/309450 objects misplaced (14.992%); 5.6 MiB/s, 90 objects/s recovering 2024-08-29T15:19:40.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:40 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:40.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:40 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:40.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:40.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:40.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:40.744 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:40.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:40.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:40.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:40.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:40.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:41.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:41.083 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:41.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:41.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:41.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:41.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:41.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:41.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:41.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:41.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:41.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:41.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:41.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:41.717 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:41.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:41.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:41.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:41.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:41.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:41.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:42.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:42.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:42.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:42.197 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:42.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:42.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:42.229 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:19:42.229 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:19:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:42 smithi027 ceph-mon[98970]: pgmap v690: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/309351 objects degraded (9.256%); 46394/309351 objects misplaced (14.997%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:19:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/309351 objects degraded (9.256%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:42 smithi027 ceph-mon[93921]: pgmap v690: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/309351 objects degraded (9.256%); 46394/309351 objects misplaced (14.997%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:19:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/309351 objects degraded (9.256%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:42.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:42.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:42.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:42.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:42.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:42 smithi171 ceph-mon[90031]: pgmap v690: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/309351 objects degraded (9.256%); 46394/309351 objects misplaced (14.997%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:19:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/309351 objects degraded (9.256%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:42.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:42.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:42.692 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:42.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:42.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:42.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:42.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:42.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:42.964 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:42.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:43.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:43.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:43.183 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:19:43.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:43.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:43.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:43.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:43.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3514274893' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:43.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3514274893' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:43.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:43.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3514274893' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:43.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:19:43.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:43.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:43.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:43.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:43.812 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:43.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:43.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:43.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:43.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:44.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:44.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:44.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[93921]: pgmap v691: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 28633/308592 objects degraded (9.279%); 45873/308592 objects misplaced (14.865%); 4.9 MiB/s, 77 objects/s recovering 2024-08-29T15:19:44.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:44.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:19:44.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:44.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:44.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:44.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:44.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:44.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:44.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:44.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:44 smithi171 ceph-mon[90031]: pgmap v691: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 28633/308592 objects degraded (9.279%); 45873/308592 objects misplaced (14.865%); 4.9 MiB/s, 77 objects/s recovering 2024-08-29T15:19:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:44 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:44 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:19:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:44 smithi171 ceph-mon[90031]: osdmap e232: 8 total, 8 up, 8 in 2024-08-29T15:19:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:44.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[93921]: osdmap e232: 8 total, 8 up, 8 in 2024-08-29T15:19:44.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[98970]: pgmap v691: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 28633/308592 objects degraded (9.279%); 45873/308592 objects misplaced (14.865%); 4.9 MiB/s, 77 objects/s recovering 2024-08-29T15:19:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:19:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:19:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[98970]: osdmap e232: 8 total, 8 up, 8 in 2024-08-29T15:19:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:44.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:44.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:44.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:44.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:44.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:44.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:19:44.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:44.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:44.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:44.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:44.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:44.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:44.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:45.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:45.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:45.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:45.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:45.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:45.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:45.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:45.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:45.456 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:45.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:45.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:45.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:45.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:45.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:45.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:45.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:45.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:45.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:45.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:45.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:46.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:46.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:46.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:46.186 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.4 to 1 (10000) 2024-08-29T15:19:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:46.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:46.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:46.341 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:46.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:46.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:46.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:46 smithi171 ceph-mon[90031]: pgmap v693: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 61 op/s; 28633/308445 objects degraded (9.283%); 45873/308445 objects misplaced (14.872%); 3.1 MiB/s, 50 objects/s recovering 2024-08-29T15:19:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2199853011' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:19:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:46 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:19:46.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:46.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:46 smithi027 ceph-mon[98970]: pgmap v693: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 61 op/s; 28633/308445 objects degraded (9.283%); 45873/308445 objects misplaced (14.872%); 3.1 MiB/s, 50 objects/s recovering 2024-08-29T15:19:46.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2199853011' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:19:46.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:46 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:19:46.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:46 smithi027 ceph-mon[93921]: pgmap v693: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 61 op/s; 28633/308445 objects degraded (9.283%); 45873/308445 objects misplaced (14.872%); 3.1 MiB/s, 50 objects/s recovering 2024-08-29T15:19:46.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2199853011' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:19:46.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:46 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:19:46.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:46.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:46.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:46.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:46.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:46.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:46.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:46.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:46.965 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:46.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:47.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:47.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:47.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:47.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:47.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:47.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:47.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:47.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:47.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:47.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:47.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:47.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:47.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:47.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:19:47.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[98970]: osdmap e233: 8 total, 8 up, 8 in 2024-08-29T15:19:47.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28633/308445 objects degraded (9.283%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:47.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[98970]: osdmap e234: 8 total, 8 up, 8 in 2024-08-29T15:19:47.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:19:47.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[93921]: osdmap e233: 8 total, 8 up, 8 in 2024-08-29T15:19:47.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28633/308445 objects degraded (9.283%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:47.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:47 smithi027 ceph-mon[93921]: osdmap e234: 8 total, 8 up, 8 in 2024-08-29T15:19:47.490 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:47.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 7 1 2024-08-29T15:19:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:47 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:19:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:47 smithi171 ceph-mon[90031]: osdmap e233: 8 total, 8 up, 8 in 2024-08-29T15:19:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28633/308445 objects degraded (9.283%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:47.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:47 smithi171 ceph-mon[90031]: osdmap e234: 8 total, 8 up, 8 in 2024-08-29T15:19:47.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:47.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:47.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:47.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:47.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:19:47.715 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:47.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:47.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:47.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:47.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:47.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:47.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:47.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:48.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:48.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:48.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:48.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:48.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:48.174 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:19:48.174 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:19:48.174 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:19:48.174 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:19:48.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:19:48.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:19:48.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:19:48.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:19:48.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:19:48.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:19:48.185 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:19:48.185 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:19:48.207 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:19:48.208 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:19:48.224 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:19:48.224 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:19:48.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:48.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:48.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:48.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:48.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:48 smithi027 ceph-mon[98970]: pgmap v695: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/307878 objects degraded (9.300%); 45397/307878 objects misplaced (14.745%); 6.5 MiB/s, 103 objects/s recovering 2024-08-29T15:19:48.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:48 smithi027 ceph-mon[98970]: osdmap e235: 8 total, 8 up, 8 in 2024-08-29T15:19:48.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3807039019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:48.395 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:48 smithi027 ceph-mon[93921]: pgmap v695: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/307878 objects degraded (9.300%); 45397/307878 objects misplaced (14.745%); 6.5 MiB/s, 103 objects/s recovering 2024-08-29T15:19:48.395 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:48 smithi027 ceph-mon[93921]: osdmap e235: 8 total, 8 up, 8 in 2024-08-29T15:19:48.395 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3807039019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:48.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:48.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:48.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:48 smithi171 ceph-mon[90031]: pgmap v695: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 28633/307878 objects degraded (9.300%); 45397/307878 objects misplaced (14.745%); 6.5 MiB/s, 103 objects/s recovering 2024-08-29T15:19:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:48 smithi171 ceph-mon[90031]: osdmap e235: 8 total, 8 up, 8 in 2024-08-29T15:19:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3807039019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:19:48.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:48.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:48.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:48.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:48.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:48.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:48.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:48.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:48.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:49.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:49.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:49.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:49.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:49.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:49.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:49.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:19:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:19:49] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-08-29T15:19:49.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:49.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:49.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:49.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:49.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:49.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:49.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:49.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:49.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:49.877 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:49.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:49.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:49.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:50.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:50.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:50.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:50.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:50.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:50.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:50.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:50 smithi027 ceph-mon[93921]: pgmap v698: 89 pgs: 2 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 24995/307512 objects degraded (8.128%); 32433/307512 objects misplaced (10.547%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:19:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:50 smithi027 ceph-mon[98970]: pgmap v698: 89 pgs: 2 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 24995/307512 objects degraded (8.128%); 32433/307512 objects misplaced (10.547%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:19:50.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:50.402 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:50.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:50 smithi171 ceph-mon[90031]: pgmap v698: 89 pgs: 2 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 24995/307512 objects degraded (8.128%); 32433/307512 objects misplaced (10.547%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:19:50.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:50.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:50.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:50.651 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:50.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:50.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:50.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:50.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:50.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:51.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:51.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:51.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:51.151 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:51.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:51.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:51.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:51.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:51.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:51.490 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:51.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:51.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:51.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:51.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:51.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:51.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:51.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:51.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:51.899 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:51.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:52.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:52.023 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:52.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:52.148 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:52.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:52.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:52.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:52.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:52.399 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:52.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:52.421 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:19:52.421 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:19:52.421 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:19:52.422 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:19:52.422 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:19:52.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:52 smithi171 ceph-mon[90031]: pgmap v699: 89 pgs: 2 peering, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 70 op/s; 25011/307176 objects degraded (8.142%); 19690/307176 objects misplaced (6.410%); 4.5 MiB/s, 70 objects/s recovering 2024-08-29T15:19:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 25011/307176 objects degraded (8.142%), 14 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:52.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:52.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:52 smithi027 ceph-mon[93921]: pgmap v699: 89 pgs: 2 peering, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 70 op/s; 25011/307176 objects degraded (8.142%); 19690/307176 objects misplaced (6.410%); 4.5 MiB/s, 70 objects/s recovering 2024-08-29T15:19:52.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 25011/307176 objects degraded (8.142%), 14 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:52.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:52 smithi027 ceph-mon[98970]: pgmap v699: 89 pgs: 2 peering, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 70 op/s; 25011/307176 objects degraded (8.142%); 19690/307176 objects misplaced (6.410%); 4.5 MiB/s, 70 objects/s recovering 2024-08-29T15:19:52.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 25011/307176 objects degraded (8.142%), 14 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:52.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:52.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:52.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:52.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:52.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:52.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:52.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:52.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:19:53.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:53.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:53.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:53.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:53.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:53.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:53.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:53.411 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:53.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:53.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:53.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:53.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:53.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:53.787 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:53.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:53.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:19:53.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:53.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:53.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:54.016 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:54.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:54.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:54.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:54.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:54.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:54.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:54.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:54.413 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:54.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:54 smithi027 ceph-mon[93921]: pgmap v700: 89 pgs: 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 30102/306114 objects degraded (9.834%); 19064/306114 objects misplaced (6.228%); 908 KiB/s, 13 objects/s recovering 2024-08-29T15:19:54.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:54 smithi027 ceph-mon[98970]: pgmap v700: 89 pgs: 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 30102/306114 objects degraded (9.834%); 19064/306114 objects misplaced (6.228%); 908 KiB/s, 13 objects/s recovering 2024-08-29T15:19:54.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:54 smithi171 ceph-mon[90031]: pgmap v700: 89 pgs: 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 30102/306114 objects degraded (9.834%); 19064/306114 objects misplaced (6.228%); 908 KiB/s, 13 objects/s recovering 2024-08-29T15:19:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.538 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:54.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:54.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:54.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:54.708 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:54.751 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:19:54.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:54.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:54.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:54.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:54.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:55.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:55.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:55.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:55.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:55.160 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:55.215 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.7 to 1 (10000) 2024-08-29T15:19:55.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:55.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:55.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:55.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:55.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3346554013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:19:55.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:55.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:55.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3346554013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:19:55.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3346554013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:19:55.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:55.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:55.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:55.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:55.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:55.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:55.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:56.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:56.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:56.130 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:56.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:56.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:56.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:56.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:56.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:56.473 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:56 smithi027 ceph-mon[93921]: pgmap v701: 89 pgs: 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 30102/305916 objects degraded (9.840%); 19064/305916 objects misplaced (6.232%); 720 KiB/s, 10 objects/s recovering 2024-08-29T15:19:56.474 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3346554013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:19:56.474 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:56 smithi027 ceph-mon[93921]: osdmap e236: 8 total, 8 up, 8 in 2024-08-29T15:19:56.474 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:56 smithi027 ceph-mon[98970]: pgmap v701: 89 pgs: 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 30102/305916 objects degraded (9.840%); 19064/305916 objects misplaced (6.232%); 720 KiB/s, 10 objects/s recovering 2024-08-29T15:19:56.474 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3346554013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:19:56.474 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:56 smithi027 ceph-mon[98970]: osdmap e236: 8 total, 8 up, 8 in 2024-08-29T15:19:56.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 5 1 2024-08-29T15:19:56.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:56 smithi171 ceph-mon[90031]: pgmap v701: 89 pgs: 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 81 op/s; 30102/305916 objects degraded (9.840%); 19064/305916 objects misplaced (6.232%); 720 KiB/s, 10 objects/s recovering 2024-08-29T15:19:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3346554013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:19:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:56 smithi171 ceph-mon[90031]: osdmap e236: 8 total, 8 up, 8 in 2024-08-29T15:19:56.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:56.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:56.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:56.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:56.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:56.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:56.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:56.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:56.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:56.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:56.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:57.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:57.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:57.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:57.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:57.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:57.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:57.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:57.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:57.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:57.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:57.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:57.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/305916 objects degraded (9.840%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:57.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:57.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:57.476 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/305916 objects degraded (9.840%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:57.477 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:57.477 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/305916 objects degraded (9.840%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:19:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:57.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:57.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:57.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:57.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:57.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:57.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:57.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:57.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:57.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:57.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:57.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:57.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:58.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:58.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:58.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:58.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:58.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:58.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[98970]: pgmap v703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 30102/305103 objects degraded (9.866%); 18991/305103 objects misplaced (6.224%); 915 KiB/s, 13 objects/s recovering 2024-08-29T15:19:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:19:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:19:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[93921]: pgmap v703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 30102/305103 objects degraded (9.866%); 18991/305103 objects misplaced (6.224%); 915 KiB/s, 13 objects/s recovering 2024-08-29T15:19:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:19:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:19:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:58.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:58.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:58 smithi171 ceph-mon[90031]: pgmap v703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 30102/305103 objects degraded (9.866%); 18991/305103 objects misplaced (6.224%); 915 KiB/s, 13 objects/s recovering 2024-08-29T15:19:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:19:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:19:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:19:58.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:19:58.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:58.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:19:58.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:58.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:58.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:58.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:58.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:58.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:58.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:58.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:59.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:19:59.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:59.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:19:59.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:19:59.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:59.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:19:59.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:19:59.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:59.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:19:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:19:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:19:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:19:59] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:19:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:19:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:59.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:19:59.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:19:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:19:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:19:59.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:59.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:59.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:19:59.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:19:59.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:19:59.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:19:59.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:19:59.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:19:59.947 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:19:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:19:59.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:19:59.953 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:19:59.953 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:19:59.971 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:19:59.971 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:19:59.988 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:19:59.988 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:00.005 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:00.005 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:00.022 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:00.023 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:00.040 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:00.041 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:00.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:00.057 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:00.057 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:00.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:00.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:00.076 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:00.076 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:00.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:00.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:00.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:00.260 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:20:00.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:00.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:00.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:00.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:00.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pgmap v704: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 30102/304581 objects degraded (9.883%); 18957/304581 objects misplaced (6.224%); 800 KiB/s, 12 objects/s recovering 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2425095559' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1324700981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 30102/304581 objects degraded (9.883%), 17 pgs degraded, 7 pgs undersized 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: [WRN] PG_DEGRADED: Degraded data redundancy: 30102/304581 objects degraded (9.883%), 17 pgs degraded, 7 pgs undersized 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 2.5 is active+recovery_wait+degraded, acting [7,0,4] 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 2.6 is active+recovery_wait+undersized+degraded+remapped, acting [1,6] 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 3.e is active+recovery_wait+degraded, acting [4,7,1] 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 3.f is active+recovery_wait+degraded, acting [4,7,0] 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 3.13 is active+recovery_wait+degraded, acting [4,7,2] 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 3.14 is active+recovery_wait+degraded, acting [4,7,6] 2024-08-29T15:20:00.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 3.15 is active+recovery_wait+degraded, acting [3,7,4] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 3.1c is active+recovery_wait+degraded, acting [5,4,1] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.0 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.1 is stuck undersized for 84s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,7] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.3 is stuck undersized for 83s, current state active+recovering+undersized+degraded+remapped, last acting [6,2] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.5 is stuck undersized for 80s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.8 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.9 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.a is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,0] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.e is active+recovery_wait+degraded, acting [4,1,2] 2024-08-29T15:20:00.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:00 smithi171 ceph-mon[90031]: pg 6.f is active+recovery_wait+degraded, acting [3,2,4] 2024-08-29T15:20:00.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:00.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pgmap v704: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 30102/304581 objects degraded (9.883%); 18957/304581 objects misplaced (6.224%); 800 KiB/s, 12 objects/s recovering 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2425095559' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1324700981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 30102/304581 objects degraded (9.883%), 17 pgs degraded, 7 pgs undersized 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: [WRN] PG_DEGRADED: Degraded data redundancy: 30102/304581 objects degraded (9.883%), 17 pgs degraded, 7 pgs undersized 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 2.5 is active+recovery_wait+degraded, acting [7,0,4] 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 2.6 is active+recovery_wait+undersized+degraded+remapped, acting [1,6] 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 3.e is active+recovery_wait+degraded, acting [4,7,1] 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 3.f is active+recovery_wait+degraded, acting [4,7,0] 2024-08-29T15:20:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 3.13 is active+recovery_wait+degraded, acting [4,7,2] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 3.14 is active+recovery_wait+degraded, acting [4,7,6] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 3.15 is active+recovery_wait+degraded, acting [3,7,4] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 3.1c is active+recovery_wait+degraded, acting [5,4,1] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.0 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.1 is stuck undersized for 84s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,7] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.3 is stuck undersized for 83s, current state active+recovering+undersized+degraded+remapped, last acting [6,2] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.5 is stuck undersized for 80s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.8 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.9 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.a is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,0] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.e is active+recovery_wait+degraded, acting [4,1,2] 2024-08-29T15:20:00.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[98970]: pg 6.f is active+recovery_wait+degraded, acting [3,2,4] 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pgmap v704: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 30102/304581 objects degraded (9.883%); 18957/304581 objects misplaced (6.224%); 800 KiB/s, 12 objects/s recovering 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2425095559' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1324700981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 30102/304581 objects degraded (9.883%), 17 pgs degraded, 7 pgs undersized 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: [WRN] PG_DEGRADED: Degraded data redundancy: 30102/304581 objects degraded (9.883%), 17 pgs degraded, 7 pgs undersized 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 2.5 is active+recovery_wait+degraded, acting [7,0,4] 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 2.6 is active+recovery_wait+undersized+degraded+remapped, acting [1,6] 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 3.e is active+recovery_wait+degraded, acting [4,7,1] 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 3.f is active+recovery_wait+degraded, acting [4,7,0] 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 3.13 is active+recovery_wait+degraded, acting [4,7,2] 2024-08-29T15:20:00.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 3.14 is active+recovery_wait+degraded, acting [4,7,6] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 3.15 is active+recovery_wait+degraded, acting [3,7,4] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 3.1c is active+recovery_wait+degraded, acting [5,4,1] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.0 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,3] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.1 is stuck undersized for 84s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,7] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.3 is stuck undersized for 83s, current state active+recovering+undersized+degraded+remapped, last acting [6,2] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.5 is stuck undersized for 80s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.8 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.9 is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,7] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.a is stuck undersized for 83s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,0] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.e is active+recovery_wait+degraded, acting [4,1,2] 2024-08-29T15:20:00.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:00 smithi027 ceph-mon[93921]: pg 6.f is active+recovery_wait+degraded, acting [3,2,4] 2024-08-29T15:20:00.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:00.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:00.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:00.698 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:00.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:00.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:00.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:00.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:00.946 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:00.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:00.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:01.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:01.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:01.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:01.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:01.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:01.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:01.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:01.347 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:01.347 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:01.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:01.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:01.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:01 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:01.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:01 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:20:01.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:01 smithi171 ceph-mon[90031]: osdmap e237: 8 total, 8 up, 8 in 2024-08-29T15:20:01.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:01.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:01 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:01.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:01 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:20:01.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:01 smithi027 ceph-mon[98970]: osdmap e237: 8 total, 8 up, 8 in 2024-08-29T15:20:01.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:01 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:01.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:01 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:20:01.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:01 smithi027 ceph-mon[93921]: osdmap e237: 8 total, 8 up, 8 in 2024-08-29T15:20:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:01.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:01.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:01.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:01.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:01.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:01.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:01.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:01.920 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:01.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:01.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:01.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:02.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:02.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:20:02.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:02.114 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:02.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:02.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:02.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:02.289 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:02.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:02.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:02 smithi027 ceph-mon[93921]: pgmap v706: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 30102/304326 objects degraded (9.891%); 18957/304326 objects misplaced (6.229%); 280 KiB/s, 4 objects/s recovering 2024-08-29T15:20:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/304326 objects degraded (9.891%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:02 smithi027 ceph-mon[98970]: pgmap v706: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 30102/304326 objects degraded (9.891%); 18957/304326 objects misplaced (6.229%); 280 KiB/s, 4 objects/s recovering 2024-08-29T15:20:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/304326 objects degraded (9.891%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:02.411 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:02 smithi171 ceph-mon[90031]: pgmap v706: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 30102/304326 objects degraded (9.891%); 18957/304326 objects misplaced (6.229%); 280 KiB/s, 4 objects/s recovering 2024-08-29T15:20:02.411 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/304326 objects degraded (9.891%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:02.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:02.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:02.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:02.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:02.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:02.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:02.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:02.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:02.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:02.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:02.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:02.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:02.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:02.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:02.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:03.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:03.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:03.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:20:03.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:03.206 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:03.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.331 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:03.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:03.432 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:03.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:03.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:03.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:03.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:03.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:03.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:03.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.833 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:03.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:03.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:03.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:03.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:04.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:04.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:04.121 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:04.121 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:04.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:04.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:04 smithi171 ceph-mon[90031]: pgmap v707: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30102/303234 objects degraded (9.927%); 18862/303234 objects misplaced (6.220%); 568 KiB/s, 8 objects/s recovering 2024-08-29T15:20:04.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:04.367 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:04.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:04.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:04.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:04.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:04 smithi027 ceph-mon[98970]: pgmap v707: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30102/303234 objects degraded (9.927%); 18862/303234 objects misplaced (6.220%); 568 KiB/s, 8 objects/s recovering 2024-08-29T15:20:04.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:04 smithi027 ceph-mon[93921]: pgmap v707: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30102/303234 objects degraded (9.927%); 18862/303234 objects misplaced (6.220%); 568 KiB/s, 8 objects/s recovering 2024-08-29T15:20:04.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:04.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:04.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:04.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:04.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:04.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:04.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:04.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:20:04.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:04.922 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:04.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:04.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:04.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:04.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:05.075 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:05.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:05.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:05.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:05.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:20:05.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:05.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:05.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:05.254 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.5 to 1 (10000) 2024-08-29T15:20:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4277121039' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:20:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:20:05.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4277121039' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:20:05.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:20:05.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:05.377 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:05.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:05.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:05.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4277121039' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:20:05.499 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:20:05.502 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:05.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:05.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:05.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:05.628 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:05.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:05.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:05.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:05.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:05.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:05.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:05.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:05.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:05.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:05.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:06.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:06.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:06.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:06.128 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:06.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:06.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:06.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:06 smithi027 ceph-mon[93921]: pgmap v708: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 30102/303057 objects degraded (9.933%); 18862/303057 objects misplaced (6.224%); 488 KiB/s, 7 objects/s recovering 2024-08-29T15:20:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:06 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:20:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:06 smithi027 ceph-mon[93921]: osdmap e238: 8 total, 8 up, 8 in 2024-08-29T15:20:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:06 smithi027 ceph-mon[98970]: pgmap v708: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 30102/303057 objects degraded (9.933%); 18862/303057 objects misplaced (6.224%); 488 KiB/s, 7 objects/s recovering 2024-08-29T15:20:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:06 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:20:06.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:06 smithi027 ceph-mon[98970]: osdmap e238: 8 total, 8 up, 8 in 2024-08-29T15:20:06.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:06.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:06.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:06.502 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:06.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:06 smithi171 ceph-mon[90031]: pgmap v708: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 94 op/s; 30102/303057 objects degraded (9.933%); 18862/303057 objects misplaced (6.224%); 488 KiB/s, 7 objects/s recovering 2024-08-29T15:20:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:06 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:20:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:06 smithi171 ceph-mon[90031]: osdmap e238: 8 total, 8 up, 8 in 2024-08-29T15:20:06.604 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:06.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:06.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:06.754 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:06.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:06.882 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:06.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:06.901 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:06.902 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:06.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 6 1 2024-08-29T15:20:06.984 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:07.008 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:07.110 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:07.132 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:07.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:07.234 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:07.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:07.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:07.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:07.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:07.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/303057 objects degraded (9.933%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:07.429 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/303057 objects degraded (9.933%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:07.439 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:20:07.439 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:20:07.439 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:20:07.439 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:20:07.440 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:20:07.440 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:07.447 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:07.449 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:07.449 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:07.468 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:07.469 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:07.482 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:07.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:07.487 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:07.487 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:07.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:07.512 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:07.513 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/303057 objects degraded (9.933%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:07.529 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:07.529 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:07.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:07.635 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:07.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:20:07.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:07.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:07.855 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:07.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:07.886 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:07.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:07.988 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:08.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:08.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:08.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:08.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:08.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:08.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:08.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:08.384 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:08.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:08.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:08.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:08.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:08 smithi171 ceph-mon[90031]: pgmap v710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 93 op/s; 30102/302328 objects degraded (9.957%); 18804/302328 objects misplaced (6.220%); 544 KiB/s, 8 objects/s recovering 2024-08-29T15:20:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2851446408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:08.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:08 smithi027 ceph-mon[98970]: pgmap v710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 93 op/s; 30102/302328 objects degraded (9.957%); 18804/302328 objects misplaced (6.220%); 544 KiB/s, 8 objects/s recovering 2024-08-29T15:20:08.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2851446408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:08.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:08 smithi027 ceph-mon[93921]: pgmap v710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 93 op/s; 30102/302328 objects degraded (9.957%); 18804/302328 objects misplaced (6.220%); 544 KiB/s, 8 objects/s recovering 2024-08-29T15:20:08.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2851446408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:08.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:08.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:08.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:08.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:08.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:08.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:08.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:08.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:08.986 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:09.009 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:09.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:09.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:09.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:09.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:09.260 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:09.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:09.295 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:20:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:20:09] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:20:09.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:09.386 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:09.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:09.488 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:09.510 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:09.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:09.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:09.629 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:23215 5.e 5:7c274c1d:::benchmark_data_smithi027_135490_object23214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 568.665804901, 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13346666099999999, 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.630 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 23215 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.631 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960453+0000", 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960452+0000", 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960487+0000", 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4594000000000001e-05 2024-08-29T15:20:09.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960497+0000", 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0348e-05 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960504+0000", 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6660000000000002e-06 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960556+0000", 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2392999999999998e-05 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960676+0000", 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119975 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.960812+0000", 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013557900000000001 2024-08-29T15:20:09.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.961708+0000", 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00089608400000000001 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.962192+0000", 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00048372900000000002 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093816+0000", 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13162422100000001 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093876+0000", 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9784000000000002e-05 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.093920+0000", 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4129999999999999e-05 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68906 5.e 5:73dcaf78:::benchmark_data_smithi027_135490_object68905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 544.71777936000001, 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14042518300000001, 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.637 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68906 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908479+0000", 2024-08-29T15:20:09.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908477+0000", 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908551+0000", 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3900999999999996e-05 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908563+0000", 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1518999999999999e-05 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908570+0000", 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0559999999999999e-06 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908679+0000", 2024-08-29T15:20:09.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010953700000000001 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908808+0000", 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012906199999999999 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.908966+0000", 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000158018 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913092+0000", 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041260869999999996 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.916369+0000", 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032766510000000002 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048843+0000", 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13247402 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048878+0000", 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5175000000000001e-05 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.048904+0000", 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5653e-05 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.643 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:68910 5.e 5:7bc74f0d:::benchmark_data_smithi027_135490_object68909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:20:09.644 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:20:09.644 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 544.71353243500005, 2024-08-29T15:20:09.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13951414700000001, 2024-08-29T15:20:09.644 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.644 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.644 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.644 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 68910 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912726+0000", 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912724+0000", 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912763+0000", 2024-08-29T15:20:09.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9082999999999998e-05 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912777+0000", 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4174e-05 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.912784+0000", 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0419999999999997e-06 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913119+0000", 2024-08-29T15:20:09.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00033500199999999999 2024-08-29T15:20:09.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913247+0000", 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012835799999999999 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.913411+0000", 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016349600000000001 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.916432+0000", 2024-08-29T15:20:09.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030208549999999998 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:04.917924+0000", 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014924789999999999 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052043+0000", 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13411882899999999 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052180+0000", 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013707500000000001 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:05.052240+0000", 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9765999999999997e-05 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.651 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71204 5.e 5:72845dde:::benchmark_data_smithi027_135490_object71203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 542.78960343599999, 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10090418, 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71204 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.652 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836655+0000", 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836653+0000", 2024-08-29T15:20:09.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836686+0000", 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3639000000000001e-05 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836700+0000", 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3372e-05 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836715+0000", 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5500000000000001e-05 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836792+0000", 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7065999999999997e-05 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.836904+0000", 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111976 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.837024+0000", 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119389 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891058+0000", 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.054034182 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891105+0000", 2024-08-29T15:20:09.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6613999999999998e-05 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937443+0000", 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046338468000000001 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937523+0000", 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0029000000000002e-05 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937559+0000", 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5862999999999999e-05 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71206 5.e 5:74b85a3d:::benchmark_data_smithi027_135490_object71205:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:20:09.658 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 542.78770545700002, 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.099155491999999998, 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71206 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:20:09.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838553+0000", 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838551+0000", 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838623+0000", 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2087000000000003e-05 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838635+0000", 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.164e-05 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.838642+0000", 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7109999999999998e-06 2024-08-29T15:20:09.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839510+0000", 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00086809300000000001 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839722+0000", 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021236700000000001 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839853+0000", 2024-08-29T15:20:09.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013045999999999999 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891096+0000", 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051243155999999998 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891152+0000", 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5998000000000002e-05 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937606+0000", 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046454595000000001 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937672+0000", 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5773999999999997e-05 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.937708+0000", 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5984e-05 2024-08-29T15:20:09.664 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:71207 5.e 5:712e06d6:::benchmark_data_smithi027_135490_object71206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 542.78668323299996, 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.101516334, 2024-08-29T15:20:09.665 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 71207 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839575+0000", 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839574+0000", 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839633+0000", 2024-08-29T15:20:09.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8607999999999999e-05 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839644+0000", 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.187e-05 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.839652+0000", 2024-08-29T15:20:09.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1910000000000003e-06 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844029+0000", 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004377148 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844134+0000", 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010534100000000001 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.844266+0000", 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000132223 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891145+0000", 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046878523999999998 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.891156+0000", 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1148000000000001e-05 2024-08-29T15:20:09.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.940954+0000", 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049798430999999997 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.941041+0000", 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7043e-05 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:06.941091+0000", 2024-08-29T15:20:09.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9765999999999998e-05 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82159 5.1 5:8a273d61:::benchmark_data_smithi027_135490_object82158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 534.31052169700001, 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098041845000000002, 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.672 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82159 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315736+0000", 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315735+0000", 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315774+0000", 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8905000000000001e-05 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315787+0000", 2024-08-29T15:20:09.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.288e-05 2024-08-29T15:20:09.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.315794+0000", 2024-08-29T15:20:09.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9639999999999997e-06 2024-08-29T15:20:09.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:09.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:09.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322730+0000", 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0069358759999999997 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.322849+0000", 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119078 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323562+0000", 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00071293799999999998 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329649+0000", 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060869790000000002 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.329680+0000", 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1455999999999998e-05 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413675+0000", 2024-08-29T15:20:09.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083995138999999996 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413734+0000", 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9247999999999999e-05 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413778+0000", 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.384e-05 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.680 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:82163 5.1 5:8a4e1f99:::benchmark_data_smithi027_135490_object82162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 534.30958387600003, 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100377148, 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 82163 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.681 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316674+0000", 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316673+0000", 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316702+0000", 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8294e-05 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316719+0000", 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7774e-05 2024-08-29T15:20:09.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.316726+0000", 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9430000000000003e-06 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326240+0000", 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0095137480000000007 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326418+0000", 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017811300000000001 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.326530+0000", 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111513 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.335223+0000", 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086934460000000005 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.413816+0000", 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078592598999999999 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416754+0000", 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029382990000000001 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416879+0000", 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012490299999999999 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.417051+0000", 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000172492 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:82171 5.9 e118/111 5:952386eb:::benchmark_data_smithi027_135490_object82170:head v 118'5035, mlcod=118'5031)", 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 534.30283730199994, 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093309638, 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323421+0000", 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323420+0000", 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323447+0000", 2024-08-29T15:20:09.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.656e-05 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323458+0000", 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1242e-05 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323464+0000", 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5130000000000002e-06 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323833+0000", 2024-08-29T15:20:09.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00036861199999999999 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.323850+0000", 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6948000000000001e-05 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416578+0000", 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.092727863999999993 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:11:15.416730+0000", 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015279099999999999 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:220847 5.e 5:70ceb708:::benchmark_data_smithi027_135490_object87095:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 385.55292705599999, 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095839788999999995, 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.692 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 220847 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073333+0000", 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2689999999999998e-06 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073338+0000", 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4009999999999996e-06 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073342+0000", 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0799999999999999e-06 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.077819+0000", 2024-08-29T15:20:09.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044769440000000001 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.077934+0000", 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000115091 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.078049+0000", 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000114052 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.082327+0000", 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042779640000000004 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.086356+0000", 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040299300000000001 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169109+0000", 2024-08-29T15:20:09.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082753023999999994 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169144+0000", 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5011000000000001e-05 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169171+0000", 2024-08-29T15:20:09.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6423000000000001e-05 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239381 5.e 5:711fbf86:::benchmark_data_smithi027_135490_object105629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 363.91218695399999, 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.21797998, 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:09.699 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239381 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.700 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:20:09.701 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.701 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.701 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.701 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.701 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:20:09.701 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714074+0000", 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.54e-06 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714081+0000", 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8430000000000004e-06 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.702 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714086+0000", 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6070000000000001e-06 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714321+0000", 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000235135 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.703 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714648+0000", 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00032731600000000002 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.715035+0000", 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000386602 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728628+0000", 2024-08-29T15:20:09.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013593190999999999 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728671+0000", 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2849000000000003e-05 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755904+0000", 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027233362000000001 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755912+0000", 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9650000000000008e-06 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693075+0000", 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.937162461 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693084+0000", 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3929999999999997e-06 2024-08-29T15:20:09.706 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.931934+0000", 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2388502020000001 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932051+0000", 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117007 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.707 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239384 5.e 5:760d1bed:::benchmark_data_smithi027_135490_object105632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 363.89778172699999, 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2039694540000001, 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239384 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.708 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728475+0000", 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.709 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728482+0000", 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5410000000000003e-06 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728496+0000", 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4047e-05 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.710 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728504+0000", 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3010000000000001e-06 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728714+0000", 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021033 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728818+0000", 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000103436 2024-08-29T15:20:09.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728983+0000", 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016501800000000001 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755958+0000", 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026975717 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.712 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755964+0000", 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2480000000000004e-06 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693203+0000", 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.937239665 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693213+0000", 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7189999999999996e-06 2024-08-29T15:20:09.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932372+0000", 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2391595259999999 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932446+0000", 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3459000000000003e-05 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239387 5.e 5:742a0183:::benchmark_data_smithi027_135490_object105635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 363.89774563899999, 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.204054685, 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239387 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.715 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728512+0000", 2024-08-29T15:20:09.716 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728515+0000", 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5450000000000002e-06 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728521+0000", 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.703e-06 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.717 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728526+0000", 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4220000000000002e-06 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.732840+0000", 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043146590000000002 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.733088+0000", 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00024788399999999997 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.733205+0000", 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011640400000000001 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755994+0000", 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022789411999999998 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.756004+0000", 2024-08-29T15:20:09.719 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7180000000000008e-06 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693242+0000", 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93723825599999999 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693251+0000", 2024-08-29T15:20:09.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9320000000000002e-06 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932479+0000", 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.239228132 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932567+0000", 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8008000000000002e-05 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.721 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239382 5.f 5:fda737cc:::benchmark_data_smithi027_135490_object105630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 363.87810703299999, 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2015833570000001, 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239382 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.722 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.723 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748149+0000", 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748157+0000", 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.109e-06 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748168+0000", 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1785e-05 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.724 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748177+0000", 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6270000000000006e-06 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748229+0000", 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2408999999999999e-05 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748239+0000", 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9629999999999993e-06 2024-08-29T15:20:09.725 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693049+0000", 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.944809695 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693062+0000", 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.29e-05 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.947700+0000", 2024-08-29T15:20:09.726 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.254637813 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949734+0000", 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020346959999999999 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239402 5.f 5:f2c183c5:::benchmark_data_smithi027_135490_object105650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:20:09.727 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 363.86731502800001, 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2791852029999999, 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239402 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:20:09.728 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758942+0000", 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.729 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758947+0000", 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5459999999999999e-06 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758958+0000", 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0305e-05 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758964+0000", 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0689999999999998e-06 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758998+0000", 2024-08-29T15:20:09.730 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4258999999999998e-05 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.759005+0000", 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5520000000000001e-06 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693436+0000", 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93443171199999997 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693448+0000", 2024-08-29T15:20:09.731 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1666e-05 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949915+0000", 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2564665989999999 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.950510+0000", 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00059576300000000002 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5,6", 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.950770+0000", 2024-08-29T15:20:09.732 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00025979900000000001 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980049+0000", 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029278985 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980078+0000", 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9043e-05 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.733 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.981086+0000", 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0010076250000000001 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038008+0000", 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.056922683000000002 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038051+0000", 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2484000000000002e-05 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.734 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038128+0000", 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7443999999999999e-05 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239405 5.e 5:7ac1943d:::benchmark_data_smithi027_135490_object105653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 361.69399105000002, 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.149048244, 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.735 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239405 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.736 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932265+0000", 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.737 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932278+0000", 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2337999999999999e-05 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932292+0000", 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3723000000000001e-05 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932301+0000", 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8660000000000001e-06 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.738 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932591+0000", 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00028943000000000003 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932743+0000", 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015197400000000001 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,2,5", 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.933306+0000", 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00056278000000000003 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.739 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.941711+0000", 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0084057430000000002 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.941847+0000", 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136174 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.955062+0000", 2024-08-29T15:20:09.740 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013214889 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081140+0000", 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12607742999999999 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081195+0000", 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4777000000000001e-05 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.741 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081315+0000", 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012078000000000001 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239409 5.f 5:f1f503e2:::benchmark_data_smithi027_135490_object105657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 361.68132919999999, 2024-08-29T15:20:09.742 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093393523000000006, 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239409 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.743 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944927+0000", 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.744 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944936+0000", 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3550000000000005e-06 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944950+0000", 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3862e-05 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944956+0000", 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3049999999999998e-06 2024-08-29T15:20:09.745 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.976857+0000", 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031901438999999997 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.976996+0000", 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000138603 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.746 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5,6", 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.977150+0000", 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000154559 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980087+0000", 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029361729999999998 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980644+0000", 2024-08-29T15:20:09.747 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00055748400000000002 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.981163+0000", 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00051858500000000001 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038192+0000", 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.057029253000000002 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.748 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038240+0000", 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7635e-05 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038322+0000", 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2867999999999993e-05 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:242790 5.e 5:7ba30800:::benchmark_data_smithi027_135490_object109038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:20:09.749 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.77012943199998, 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100980449, 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242790 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.750 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856128+0000", 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.751 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856134+0000", 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.669e-06 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856145+0000", 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0298e-05 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856150+0000", 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2220000000000001e-06 2024-08-29T15:20:09.752 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856187+0000", 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7452e-05 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856270+0000", 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2522999999999996e-05 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.753 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,2,5", 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856349+0000", 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9153000000000006e-05 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.859142+0000", 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027929640000000002 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.859413+0000", 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00027162699999999998 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.754 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954345+0000", 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.094931353999999996 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956931+0000", 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025865459999999999 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956981+0000", 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9325e-05 2024-08-29T15:20:09.755 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.957109+0000", 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012847100000000001 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:242790 5.e e143/140 ondisk, result = 0)", 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.76567663600002, 2024-08-29T15:20:09.756 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093766336000000006, 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.757 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860581+0000", 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860586+0000", 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9599999999999999e-06 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.758 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954254+0000", 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093668543000000007 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954271+0000", 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6373999999999999e-05 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954325+0000", 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4342999999999998e-05 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.759 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954337+0000", 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1558e-05 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954348+0000", 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1318000000000001e-05 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.760 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:242791 5.7 e143/140 5:e93ed9d7:::benchmark_data_smithi027_135490_object109039:head v 143'15344, mlcod=143'15341)", 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.76026999700002, 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.120497535, 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.761 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865987+0000", 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:09.762 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865994+0000", 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8990000000000004e-06 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954486+0000", 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088492092999999994 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954496+0000", 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5550000000000008e-06 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.763 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954810+0000", 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031430999999999999 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954830+0000", 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0162000000000001e-05 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.986452+0000", 2024-08-29T15:20:09.764 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031621761999999998 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.986486+0000", 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3630000000000002e-05 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:09.765 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:20:09.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:09.800 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:09.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:09.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:09.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:10.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:10.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:10.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:10.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:10.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:10.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:10.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:10.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:10.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:10.506 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:10 smithi171 ceph-mon[90031]: pgmap v711: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 30102/301926 objects degraded (9.970%); 18780/301926 objects misplaced (6.220%); 526 KiB/s, 8 objects/s recovering 2024-08-29T15:20:10.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:10.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:20:10.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:10.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:10.561 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:10 smithi027 ceph-mon[93921]: pgmap v711: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 30102/301926 objects degraded (9.970%); 18780/301926 objects misplaced (6.220%); 526 KiB/s, 8 objects/s recovering 2024-08-29T15:20:10.562 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:10 smithi027 ceph-mon[98970]: pgmap v711: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 30102/301926 objects degraded (9.970%); 18780/301926 objects misplaced (6.220%); 526 KiB/s, 8 objects/s recovering 2024-08-29T15:20:10.652 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:10.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:10.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:20:10.746 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:10.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:10.800 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:10.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:10.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:10.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:10.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:10.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:11.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:11.057 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:11.159 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:11.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:11.270 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:20:11.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:11.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:11.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:11.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:11.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3893829897' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:11.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:11.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3893829897' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:11.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3893829897' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:11.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:11.654 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:11.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:11.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:11.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:11.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:11.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:11.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:11.926 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:11.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:12.028 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:12.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:12.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:12.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:12.278 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:12.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:12.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[98970]: pgmap v712: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 30102/301554 objects degraded (9.982%); 18780/301554 objects misplaced (6.228%); 435 KiB/s, 6 objects/s recovering 2024-08-29T15:20:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:20:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[98970]: osdmap e239: 8 total, 8 up, 8 in 2024-08-29T15:20:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/301554 objects degraded (9.982%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[93921]: pgmap v712: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 30102/301554 objects degraded (9.982%); 18780/301554 objects misplaced (6.228%); 435 KiB/s, 6 objects/s recovering 2024-08-29T15:20:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:20:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[93921]: osdmap e239: 8 total, 8 up, 8 in 2024-08-29T15:20:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/301554 objects degraded (9.982%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:12.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:12.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:12.468 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:12 smithi171 ceph-mon[90031]: pgmap v712: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 92 op/s; 30102/301554 objects degraded (9.982%); 18780/301554 objects misplaced (6.228%); 435 KiB/s, 6 objects/s recovering 2024-08-29T15:20:12.468 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:12 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:20:12.469 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:12 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:12.469 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:12 smithi171 ceph-mon[90031]: osdmap e239: 8 total, 8 up, 8 in 2024-08-29T15:20:12.469 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/301554 objects degraded (9.982%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:12.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:12.557 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:12.583 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:20:12.583 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150809 6.3 6.b854e143 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.311161+0000", 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.27030617400000001, 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.270326662, 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.584 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150809 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.311161+0000", 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.311161+0000", 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.311160+0000", 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.311167+0000", 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7300000000000005e-06 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.311180+0000", 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2923e-05 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.311189+0000", 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9209999999999998e-06 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150812 6.3 6.4679d6a3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.319416+0000", 2024-08-29T15:20:12.587 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.26205118500000002, 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26211374300000001, 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150812 2024-08-29T15:20:12.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.319416+0000", 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.319416+0000", 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.319414+0000", 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.319424+0000", 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2420000000000006e-06 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.319438+0000", 2024-08-29T15:20:12.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4251e-05 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.319449+0000", 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1473e-05 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150615 6.3 6:c66ec092:::benchmark_data_smithi027_169445_object25188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:09.441411+0000", 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 3.1400559829999999, 2024-08-29T15:20:12.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1401516250000001, 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150615 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.441411+0000", 2024-08-29T15:20:12.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.441411+0000", 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.441410+0000", 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.593 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.441418+0000", 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3799999999999994e-06 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.441433+0000", 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4717e-05 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.441443+0000", 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9229999999999997e-06 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.540519+0000", 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.099076655999999999 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.550381+0000", 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0098616889999999999 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.713262+0000", 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16288071900000001 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.721872+0000", 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086103340000000007 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150618 6.3 6:c681d1e8:::benchmark_data_smithi027_169445_object25191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.596 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:09.524936+0000", 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 3.0565316469999999, 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0566690840000001, 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150618 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.524936+0000", 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.524936+0000", 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.524934+0000", 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.524942+0000", 2024-08-29T15:20:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9289999999999998e-06 2024-08-29T15:20:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.524957+0000", 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.432e-05 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.524966+0000", 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6770000000000007e-06 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.551487+0000", 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026520878000000001 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.561208+0000", 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0097205859999999998 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.659745+0000", 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098537391000000002 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.668911+0000", 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0091658620000000003 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150841 6.3 6.7017e053 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2024-08-29T15:20:12.602 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.477238+0000", 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.10422922599999999, 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10440765, 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150841 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.477238+0000", 2024-08-29T15:20:12.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.477238+0000", 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.477236+0000", 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.477245+0000", 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8750000000000006e-06 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.477261+0000", 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6192999999999999e-05 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.477269+0000", 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0209999999999999e-06 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.605 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150698 6.3 6:c2661aa8:::benchmark_data_smithi027_169445_object25271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:10.164015+0000", 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.4174523109999999, 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4176620259999999, 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.606 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150698 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.164015+0000", 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.164015+0000", 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.164013+0000", 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.164023+0000", 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0308e-05 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.164044+0000", 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0611e-05 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.164054+0000", 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0172e-05 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.231806+0000", 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067751807999999997 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.240763+0000", 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0089570559999999997 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.303688+0000", 2024-08-29T15:20:12.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062924956000000004 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.312432+0000", 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087445279999999997 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150771 6.3 6.3b645be3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.153233+0000", 2024-08-29T15:20:12.611 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.42823391599999999, 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.428499717, 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150771 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.612 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.153233+0000", 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.153233+0000", 2024-08-29T15:20:12.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.153231+0000", 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.153241+0000", 2024-08-29T15:20:12.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1020000000000003e-06 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.153257+0000", 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6467999999999999e-05 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.153266+0000", 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7490000000000004e-06 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150704 6.3 6:c896c1fd:::benchmark_data_smithi027_169445_object25277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:10.186399+0000", 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.395067981, 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3953656109999999, 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150704 2024-08-29T15:20:12.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.186399+0000", 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.186399+0000", 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.186398+0000", 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.186408+0000", 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0319999999999999e-05 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.186421+0000", 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2468e-05 2024-08-29T15:20:12.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.186429+0000", 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1999999999999994e-06 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.313212+0000", 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12678277399999999 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.321744+0000", 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0085321530000000007 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.365732+0000", 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.043988622999999998 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.374529+0000", 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087970010000000005 2024-08-29T15:20:12.620 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44912.0:150715 6.3 e239/218 ondisk, result = 0)", 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.499971+0000", 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.081496440000000003, 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.081832632000000002, 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.621 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.499971+0000", 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.499971+0000", 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.499970+0000", 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.499975+0000", 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2329999999999997e-06 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.499987+0000", 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1800999999999999e-05 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.499993+0000", 2024-08-29T15:20:12.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4990000000000002e-06 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.575758+0000", 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.075765158999999999 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.576979+0000", 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012207920000000001 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44912.0:150710 6.3 e239/218)", 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.554285+0000", 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.027182165000000001, 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027547275999999999, 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554285+0000", 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554285+0000", 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554284+0000", 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:12.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554291+0000", 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7179999999999999e-06 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554302+0000", 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1316e-05 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554308+0000", 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7230000000000001e-06 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44912.0:150710 6.3 e239/218)", 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.554765+0000", 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.026702061999999999, 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027090270999999999, 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:20:12.628 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554765+0000", 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554765+0000", 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554765+0000", 2024-08-29T15:20:12.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554770+0000", 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5999999999999997e-06 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554784+0000", 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.348e-05 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.554790+0000", 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7619999999999998e-06 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150710 6.3 6:c05455fa:::benchmark_data_smithi027_169445_object25283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:10.273943+0000", 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.307524693, 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3079390150000001, 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.631 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150710 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.273943+0000", 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.273943+0000", 2024-08-29T15:20:12.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.273941+0000", 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.273948+0000", 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6559999999999992e-06 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.273963+0000", 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4669e-05 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.273971+0000", 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4229999999999999e-06 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.325042+0000", 2024-08-29T15:20:12.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051071274 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.333783+0000", 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087412159999999996 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.406650+0000", 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072866623000000005 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.417671+0000", 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011021697 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.513041+0000", 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0953700949999998 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.525204+0000", 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012162956000000001 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.550042+0000", 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024837109999999999 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150790 6.3 6.29d70d13 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:12.240846+0000", 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.34062179100000001, 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.34108299199999997, 2024-08-29T15:20:12.637 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150790 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.240846+0000", 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.240846+0000", 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.240844+0000", 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.240851+0000", 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4139999999999999e-06 2024-08-29T15:20:12.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.240863+0000", 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2087e-05 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.240872+0000", 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7269999999999996e-06 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150648 6.3 6:c3355c0e:::benchmark_data_smithi027_169445_object25221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:09.875688+0000", 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.705779368, 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7062749080000001, 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150648 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.641 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.875688+0000", 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.875688+0000", 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.875687+0000", 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.875691+0000", 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.527e-06 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.875699+0000", 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0309999999999998e-06 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.875714+0000", 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4844000000000001e-05 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.918205+0000", 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042490417000000003 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.931248+0000", 2024-08-29T15:20:12.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013043163 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.123968+0000", 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19272006 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.134727+0000", 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010758861 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.645 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.646 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150654 6.3 6:ce1320d9:::benchmark_data_smithi027_169445_object25227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.646 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:09.908022+0000", 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.6734453629999999, 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.67401234, 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.647 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150654 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.908022+0000", 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.908022+0000", 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.908020+0000", 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.908029+0000", 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6230000000000003e-06 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.908042+0000", 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3328e-05 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.908051+0000", 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5990000000000002e-06 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.932314+0000", 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024263408 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:09.940991+0000", 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086763940000000005 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.066890+0000", 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12589941800000001 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.082526+0000", 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015635719999999999 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150715 6.3 6:cbad72a2:::benchmark_data_smithi027_169445_object25288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e238)", 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:10.612028+0000", 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.9694391360000001, 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9700832029999999, 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:20:12.652 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150715 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.612028+0000", 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.612028+0000", 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.612026+0000", 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.612036+0000", 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8640000000000008e-06 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.612053+0000", 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7283e-05 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.612063+0000", 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4709999999999996e-06 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.632171+0000", 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020108134 2024-08-29T15:20:12.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.641572+0000", 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0094013340000000008 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.766426+0000", 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.124853597 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:12.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:10.777155+0000", 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010729117 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.460980+0000", 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.683824846 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.475807+0000", 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014827217 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.495868+0000", 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020060646000000001 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.568709+0000", 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072841885999999995 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.574390+0000", 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056805659999999997 2024-08-29T15:20:12.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:12.577995+0000", 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003605376 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150744 6.3 6.43dfe143 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:11.585915+0000", 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.99555250100000003, 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99625265600000001, 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.659 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "op must wait for map", 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150744 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.585915+0000", 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.585915+0000", 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.585913+0000", 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.585921+0000", 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7349999999999996e-06 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.585936+0000", 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4408e-05 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.585943+0000", 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3529999999999997e-06 2024-08-29T15:20:12.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.614771+0000", 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028828168000000001 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op must wait for map", 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.615533+0000", 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00076237099999999997 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:150745 6.3 6.1a995363 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e239)", 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:11.640199+0000", 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.94126852999999999, 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94200210600000001, 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting_for_map not empty", 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 150745 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.664 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.640199+0000", 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.640199+0000", 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.640197+0000", 2024-08-29T15:20:12.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.640204+0000", 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4489999999999996e-06 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.640217+0000", 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3028e-05 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.640224+0000", 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3339999999999996e-06 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.683753+0000", 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.043528606999999997 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting_for_map not empty", 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:11.684536+0000", 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00078302700000000005 2024-08-29T15:20:12.667 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.668 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:12.668 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.668 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:12.668 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:20:12.668 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 18 2024-08-29T15:20:12.668 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:20:12.668 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:12.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:12.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:12.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:12.920 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:12.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:12.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:13.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:13.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:13.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:13.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:13.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:13.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:13.259 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.6 to 1 (10000) 2024-08-29T15:20:13.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:13.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:13.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:13.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:13.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1590146504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:20:13.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:13.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:13.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:13.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1590146504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:20:13.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1590146504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:20:13.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:20:13.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:13.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:13.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:13.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:13.863 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:13.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:13.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:13.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:13.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:14.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:14.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:14.102 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-29T15:20:14.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:20:14.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:14.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:14.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:14.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:14.312 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:14.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:14.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:14 smithi171 ceph-mon[90031]: pgmap v714: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 30102/300588 objects degraded (10.014%); 18690/300588 objects misplaced (6.218%); 632 KiB/s, 9 objects/s recovering 2024-08-29T15:20:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1590146504' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:20:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:14 smithi171 ceph-mon[90031]: osdmap e240: 8 total, 8 up, 8 in 2024-08-29T15:20:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:14.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:14.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[98970]: pgmap v714: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 30102/300588 objects degraded (10.014%); 18690/300588 objects misplaced (6.218%); 632 KiB/s, 9 objects/s recovering 2024-08-29T15:20:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1590146504' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:20:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[98970]: osdmap e240: 8 total, 8 up, 8 in 2024-08-29T15:20:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[93921]: pgmap v714: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 30102/300588 objects degraded (10.014%); 18690/300588 objects misplaced (6.218%); 632 KiB/s, 9 objects/s recovering 2024-08-29T15:20:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1590146504' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:20:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[93921]: osdmap e240: 8 total, 8 up, 8 in 2024-08-29T15:20:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:14.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:20:14.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:14.685 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:14.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:14.810 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:14.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:14.936 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:14.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:15.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:15.061 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:15.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:15.189 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:20:15.189 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:20:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:15.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:15.193 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:15.213 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:20:15.213 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:20:15.242 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:20:15.242 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:15.265 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:15.265 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:15.285 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:15.286 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:15.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:15.307 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:15.308 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:15.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:15.327 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:15.327 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:15.345 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:15.346 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:15.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:15.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:15.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:15 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2439390707' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:15.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:15 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2439390707' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:15 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2439390707' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:15.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:15.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:15.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:15.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:15.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:15.698 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:15.698 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:15.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:15.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:15.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:15.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:15.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:15.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:16.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:16.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:16.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:16.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:16.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:16.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:16.297 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:16.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:16.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:16.415 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:16 smithi171 ceph-mon[90031]: pgmap v716: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 30102/300435 objects degraded (10.019%); 18690/300435 objects misplaced (6.221%); 376 KiB/s, 5 objects/s recovering 2024-08-29T15:20:16.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:16.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:20:16.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:16.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:16.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:16 smithi027 ceph-mon[98970]: pgmap v716: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 30102/300435 objects degraded (10.019%); 18690/300435 objects misplaced (6.221%); 376 KiB/s, 5 objects/s recovering 2024-08-29T15:20:16.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:16 smithi027 ceph-mon[93921]: pgmap v716: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 30102/300435 objects degraded (10.019%); 18690/300435 objects misplaced (6.221%); 376 KiB/s, 5 objects/s recovering 2024-08-29T15:20:16.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:16.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:16.661 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:16.670 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:16.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:16.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:16.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:16.921 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:16.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:16.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:16.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:17.021 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:17.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:17.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:17.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:17.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:17.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:17.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:17.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:17.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:17.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:17.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:17.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/300435 objects degraded (10.019%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:17.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:17.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/300435 objects degraded (10.019%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/300435 objects degraded (10.019%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:17.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:17.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:17.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:17.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:17.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:17.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:17.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:17.951 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:17.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:17.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:18.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:18.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:18.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:18.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:20:18.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:18.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:18.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:18.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:18.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:18.423 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:18.423 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:18.423 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:18 smithi171 ceph-mon[90031]: pgmap v717: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 30102/299847 objects degraded (10.039%); 18659/299847 objects misplaced (6.223%); 376 KiB/s, 5 objects/s recovering 2024-08-29T15:20:18.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:18.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:18.495 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:18 smithi027 ceph-mon[98970]: pgmap v717: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 30102/299847 objects degraded (10.039%); 18659/299847 objects misplaced (6.223%); 376 KiB/s, 5 objects/s recovering 2024-08-29T15:20:18.495 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:18 smithi027 ceph-mon[93921]: pgmap v717: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 30102/299847 objects degraded (10.039%); 18659/299847 objects misplaced (6.223%); 376 KiB/s, 5 objects/s recovering 2024-08-29T15:20:18.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:18.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:18.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:18.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:18.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:18.831 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:18.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:18.956 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:20:18.958 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:20:18.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:18.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:18.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:19.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:19.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:19.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:19.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:19.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:20:19.271 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:20:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:20:19] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:20:19.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:19.337 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:19.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:19.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:19.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:19.481 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:19.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:19.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:19.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:19.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:19.716 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:19.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:19.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:19.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:19.911 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":718,"stamp":"2024-08-29T15:20:18.536704+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6538139645,"num_objects":99720,"num_object_clones":0,"num_object_copies":299160,"num_objects_missing_on_primary":7205,"num_objects_missing":7205,"num_objects_degraded":30102,"num_objects_misplaced":18599,"num_objects_unfound":0,"num_objects_dirty":99720,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":152281,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14261,"num_bytes_recovered":813436109,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120417,"ondisk_log_size":120417,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20749276,"kb_used_data":19049800,"kb_used_omap":43,"kb_used_meta":1699348,"kb_avail":729211940,"statfs":{"total":767960285184,"available":746713026560,"internally_reserved":0,"allocated":19506995200,"data_stored":19496431566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1740132844},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,2,0,2,8,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-52297728,"num_objects":-798,"num_object_clones":0,"num_object_copies":-2394,"num_objects_missing_on_primary":-67,"num_objects_missing":-67,"num_objects_degraded":0,"num_objects_misplaced":-181,"num_objects_unfound":0,"num_objects_dirty":-798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":798,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":4456448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002269"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":363,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.262478+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:20:13.262478+0000","last_peered":"2024-08-29T15:20:13.262478+0000","last_clean":"2024-08-29T15:20:13.262478+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:20:13.262478+0000","last_undegraded":"2024-08-29T15:20:13.262478+0000","last_fullsized":"2024-08-29T15:20:13.262478+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.267782+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:20:11.267782+0000","last_peered":"2024-08-29T15:20:11.267782+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:20:11.267782+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:20:11.267782+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":106,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.266232+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:20:11.266232+0000","last_peered":"2024-08-29T15:20:11.266232+0000","last_clean":"2024-08-29T15:20:11.266232+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:20:11.266232+0000","last_undegraded":"2024-08-29T15:20:11.266232+0000","last_fullsized":"2024-08-29T15:20:11.266232+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":329,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488668+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:20:13.488668+0000","last_peered":"2024-08-29T15:20:13.488668+0000","last_clean":"2024-08-29T15:20:13.488668+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:20:13.488668+0000","last_undegraded":"2024-08-29T15:20:13.488668+0000","last_fullsized":"2024-08-29T15:20:13.488668+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":329,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488337+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:20:13.488337+0000","last_peered":"2024-08-29T15:20:13.488337+0000","last_clean":"2024-08-29T15:20:13.488337+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:20:13.488337+0000","last_undegraded":"2024-08-29T15:20:13.488337+0000","last_fullsized":"2024-08-29T15:20:13.488337+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267490+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:20:11.267490+0000","last_peered":"2024-08-29T15:20:11.267490+0000","last_clean":"2024-08-29T15:20:11.267490+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:20:11.267490+0000","last_undegraded":"2024-08-29T15:20:11.267490+0000","last_fullsized":"2024-08-29T15:20:11.267490+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":140,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265932+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:20:13.265932+0000","last_peered":"2024-08-29T15:20:13.265932+0000","last_clean":"2024-08-29T15:20:13.265932+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:20:13.265932+0000","last_undegraded":"2024-08-29T15:20:13.265932+0000","last_fullsized":"2024-08-29T15:20:13.265932+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":107,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.929413+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:20:11.929413+0000","last_peered":"2024-08-29T15:20:11.929413+0000","last_clean":"2024-08-29T15:20:11.929413+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:20:11.929413+0000","last_undegraded":"2024-08-29T15:20:11.929413+0000","last_fullsized":"2024-08-29T15:20:11.929413+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"240'9601","reported_seq":17618,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:16.045301+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:20:16.045301+0000","last_peered":"2024-08-29T15:20:16.045301+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:20:16.045301+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9601,"log_dups_size":0,"ondisk_log_size":9601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413597696,"num_objects":6311,"num_object_clones":0,"num_object_copies":18933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9601,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":3386}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.268001+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:20:11.268001+0000","last_peered":"2024-08-29T15:20:11.268001+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:20:11.268001+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:20:11.268001+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":124,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.265425+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:20:11.265425+0000","last_peered":"2024-08-29T15:20:11.265425+0000","last_clean":"2024-08-29T15:20:11.265425+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:20:11.265425+0000","last_undegraded":"2024-08-29T15:20:11.265425+0000","last_fullsized":"2024-08-29T15:20:11.265425+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"240'9365","reported_seq":9522,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:16.477298+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:20:16.477298+0000","last_peered":"2024-08-29T15:20:16.477298+0000","last_clean":"2024-08-29T15:20:16.477298+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:20:16.477298+0000","last_undegraded":"2024-08-29T15:20:16.477298+0000","last_fullsized":"2024-08-29T15:20:16.477298+0000","mapping_epoch":168,"log_start":"177'2628","ondisk_log_start":"177'2628","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6737,"log_dups_size":0,"ondisk_log_size":6737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405340160,"num_objects":6185,"num_object_clones":0,"num_object_copies":18555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9365,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":15,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.271242+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:20:11.271242+0000","last_peered":"2024-08-29T15:20:11.271242+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:20:11.271242+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:20:11.271242+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267403+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:20:11.267403+0000","last_peered":"2024-08-29T15:20:11.267403+0000","last_clean":"2024-08-29T15:20:11.267403+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:20:11.267403+0000","last_undegraded":"2024-08-29T15:20:11.267403+0000","last_fullsized":"2024-08-29T15:20:11.267403+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"240'9444","reported_seq":9570,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:16.485676+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:20:16.485676+0000","last_peered":"2024-08-29T15:20:16.485676+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:20:16.485676+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"177'2774","ondisk_log_start":"177'2774","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410779648,"num_objects":6268,"num_object_clones":0,"num_object_copies":18804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9444,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3791}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":145,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488732+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:20:13.488732+0000","last_peered":"2024-08-29T15:20:13.488732+0000","last_clean":"2024-08-29T15:20:13.488732+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:20:13.488732+0000","last_undegraded":"2024-08-29T15:20:13.488732+0000","last_fullsized":"2024-08-29T15:20:13.488732+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":62,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.929701+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:20:11.929701+0000","last_peered":"2024-08-29T15:20:11.929701+0000","last_clean":"2024-08-29T15:20:11.929701+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:20:11.929701+0000","last_undegraded":"2024-08-29T15:20:11.929701+0000","last_fullsized":"2024-08-29T15:20:11.929701+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"239'9552","reported_seq":14929,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:13.487569+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:20:13.487569+0000","last_peered":"2024-08-29T15:20:13.487569+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:20:13.487569+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"175'1362","ondisk_log_start":"175'1362","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8190,"log_dups_size":0,"ondisk_log_size":8190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408027136,"num_objects":6226,"num_object_clones":0,"num_object_copies":18678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9552,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":6226}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":124,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.265906+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:20:11.265906+0000","last_peered":"2024-08-29T15:20:11.265906+0000","last_clean":"2024-08-29T15:20:11.265906+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:20:11.265906+0000","last_undegraded":"2024-08-29T15:20:11.265906+0000","last_fullsized":"2024-08-29T15:20:11.265906+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":343,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488055+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:20:13.488055+0000","last_peered":"2024-08-29T15:20:13.488055+0000","last_clean":"2024-08-29T15:20:13.488055+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:20:13.488055+0000","last_undegraded":"2024-08-29T15:20:13.488055+0000","last_fullsized":"2024-08-29T15:20:13.488055+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"240'9507","reported_seq":9632,"reported_epoch":240,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:14.368126+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:20:14.368126+0000","last_peered":"2024-08-29T15:20:14.368126+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:20:14.368126+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:20:14.368126+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7174,"log_dups_size":0,"ondisk_log_size":7174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413335552,"num_objects":6307,"num_object_clones":0,"num_object_copies":18921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9507,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3830},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":338,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.262361+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:20:13.262361+0000","last_peered":"2024-08-29T15:20:13.262361+0000","last_clean":"2024-08-29T15:20:13.262361+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:20:13.262361+0000","last_undegraded":"2024-08-29T15:20:13.262361+0000","last_fullsized":"2024-08-29T15:20:13.262361+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267765+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:20:11.267765+0000","last_peered":"2024-08-29T15:20:11.267765+0000","last_clean":"2024-08-29T15:20:11.267765+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:20:11.267765+0000","last_undegraded":"2024-08-29T15:20:11.267765+0000","last_fullsized":"2024-08-29T15:20:11.267765+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"240'9517","reported_seq":9690,"reported_epoch":240,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:15.677750+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:20:15.677750+0000","last_peered":"2024-08-29T15:20:15.677750+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:20:15.677750+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:20:15.677750+0000","mapping_epoch":234,"log_start":"178'2768","ondisk_log_start":"178'2768","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414515200,"num_objects":6325,"num_object_clones":0,"num_object_copies":18975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9517,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3860},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":80,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.929608+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:20:11.929608+0000","last_peered":"2024-08-29T15:20:11.929608+0000","last_clean":"2024-08-29T15:20:11.929608+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:20:11.929608+0000","last_undegraded":"2024-08-29T15:20:11.929608+0000","last_fullsized":"2024-08-29T15:20:11.929608+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267847+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:20:11.267847+0000","last_peered":"2024-08-29T15:20:11.267847+0000","last_clean":"2024-08-29T15:20:11.267847+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:20:11.267847+0000","last_undegraded":"2024-08-29T15:20:11.267847+0000","last_fullsized":"2024-08-29T15:20:11.267847+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"240'9402","reported_seq":9760,"reported_epoch":240,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:20:16.192021+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:20:16.192021+0000","last_peered":"2024-08-29T15:20:16.192021+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:20:16.192021+0000","last_undegraded":"2024-08-29T15:20:16.192021+0000","last_fullsized":"2024-08-29T15:20:16.192021+0000","mapping_epoch":202,"log_start":"177'2667","ondisk_log_start":"177'2667","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409993216,"num_objects":6256,"num_object_clones":0,"num_object_copies":18768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6256,"num_objects_unfound":0,"num_objects_dirty":6256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9402,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":6256}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267379+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:20:11.267379+0000","last_peered":"2024-08-29T15:20:11.267379+0000","last_clean":"2024-08-29T15:20:11.267379+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:20:11.267379+0000","last_undegraded":"2024-08-29T15:20:11.267379+0000","last_fullsized":"2024-08-29T15:20:11.267379+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":222,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.271264+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:20:11.271264+0000","last_peered":"2024-08-29T15:20:11.271264+0000","last_clean":"2024-08-29T15:20:11.271264+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:20:11.271264+0000","last_undegraded":"2024-08-29T15:20:11.271264+0000","last_fullsized":"2024-08-29T15:20:11.271264+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"240'9475","reported_seq":9594,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:15.526341+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:20:15.526341+0000","last_peered":"2024-08-29T15:20:15.526341+0000","last_clean":"2024-08-29T15:20:15.526341+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:20:15.526341+0000","last_undegraded":"2024-08-29T15:20:15.526341+0000","last_fullsized":"2024-08-29T15:20:15.526341+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8197,"log_dups_size":0,"ondisk_log_size":8197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411828248,"num_objects":6285,"num_object_clones":0,"num_object_copies":18855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6285,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9475,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":341,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.262110+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:20:13.262110+0000","last_peered":"2024-08-29T15:20:13.262110+0000","last_clean":"2024-08-29T15:20:13.262110+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:20:13.262110+0000","last_undegraded":"2024-08-29T15:20:13.262110+0000","last_fullsized":"2024-08-29T15:20:13.262110+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":333,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265967+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:20:13.265967+0000","last_peered":"2024-08-29T15:20:13.265967+0000","last_clean":"2024-08-29T15:20:13.265967+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:20:13.265967+0000","last_undegraded":"2024-08-29T15:20:13.265967+0000","last_fullsized":"2024-08-29T15:20:13.265967+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"240'9187","reported_seq":9354,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:15.794991+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:20:15.794991+0000","last_peered":"2024-08-29T15:20:15.794991+0000","last_clean":"2024-08-29T15:20:15.794991+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:20:15.794991+0000","last_undegraded":"2024-08-29T15:20:15.794991+0000","last_fullsized":"2024-08-29T15:20:15.794991+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9187,"log_dups_size":0,"ondisk_log_size":9187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394592256,"num_objects":6021,"num_object_clones":0,"num_object_copies":18063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9187,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":432,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.487654+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:20:13.487654+0000","last_peered":"2024-08-29T15:20:13.487654+0000","last_clean":"2024-08-29T15:20:13.487654+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:20:13.487654+0000","last_undegraded":"2024-08-29T15:20:13.487654+0000","last_fullsized":"2024-08-29T15:20:13.487654+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":708,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.265571+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:20:11.265571+0000","last_peered":"2024-08-29T15:20:11.265571+0000","last_clean":"2024-08-29T15:20:11.265571+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:20:11.265571+0000","last_undegraded":"2024-08-29T15:20:11.265571+0000","last_fullsized":"2024-08-29T15:20:11.265571+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":216,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.265684+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:20:11.265684+0000","last_peered":"2024-08-29T15:20:11.265684+0000","last_clean":"2024-08-29T15:20:11.265684+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:20:11.265684+0000","last_undegraded":"2024-08-29T15:20:11.265684+0000","last_fullsized":"2024-08-29T15:20:11.265684+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":78,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.929848+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:20:11.929848+0000","last_peered":"2024-08-29T15:20:11.929848+0000","last_clean":"2024-08-29T15:20:11.929848+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:20:11.929848+0000","last_undegraded":"2024-08-29T15:20:11.929848+0000","last_fullsized":"2024-08-29T15:20:11.929848+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"240'9530","reported_seq":9885,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:15.636716+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:20:15.636716+0000","last_peered":"2024-08-29T15:20:15.636716+0000","last_clean":"2024-08-29T15:20:15.636716+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:20:15.636716+0000","last_undegraded":"2024-08-29T15:20:15.636716+0000","last_fullsized":"2024-08-29T15:20:15.636716+0000","mapping_epoch":201,"log_start":"177'2553","ondisk_log_start":"177'2553","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403177472,"num_objects":6152,"num_object_clones":0,"num_object_copies":18456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9530,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":806,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.264687+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:20:13.264687+0000","last_peered":"2024-08-29T15:20:13.264687+0000","last_clean":"2024-08-29T15:20:13.264687+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:20:13.264687+0000","last_undegraded":"2024-08-29T15:20:13.264687+0000","last_fullsized":"2024-08-29T15:20:13.264687+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":344,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488009+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:20:13.488009+0000","last_peered":"2024-08-29T15:20:13.488009+0000","last_clean":"2024-08-29T15:20:13.488009+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:20:13.488009+0000","last_undegraded":"2024-08-29T15:20:13.488009+0000","last_fullsized":"2024-08-29T15:20:13.488009+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":351,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488616+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:20:13.488616+0000","last_peered":"2024-08-29T15:20:13.488616+0000","last_clean":"2024-08-29T15:20:13.488616+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:20:13.488616+0000","last_undegraded":"2024-08-29T15:20:13.488616+0000","last_fullsized":"2024-08-29T15:20:13.488616+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"240'9411","reported_seq":15189,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:15.790887+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:20:15.790887+0000","last_peered":"2024-08-29T15:20:15.790887+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:20:15.790887+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'2730","ondisk_log_start":"178'2730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411500544,"num_objects":6279,"num_object_clones":0,"num_object_copies":18837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9411,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1845},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":614,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.262156+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:20:13.262156+0000","last_peered":"2024-08-29T15:20:13.262156+0000","last_clean":"2024-08-29T15:20:13.262156+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:20:13.262156+0000","last_undegraded":"2024-08-29T15:20:13.262156+0000","last_fullsized":"2024-08-29T15:20:13.262156+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":69,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.384882+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:20:13.384882+0000","last_peered":"2024-08-29T15:20:13.384882+0000","last_clean":"2024-08-29T15:20:13.384882+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:20:13.384882+0000","last_undegraded":"2024-08-29T15:20:13.384882+0000","last_fullsized":"2024-08-29T15:20:13.384882+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":378,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265815+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:20:13.265815+0000","last_peered":"2024-08-29T15:20:13.265815+0000","last_clean":"2024-08-29T15:20:13.265815+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:20:13.265815+0000","last_undegraded":"2024-08-29T15:20:13.265815+0000","last_fullsized":"2024-08-29T15:20:13.265815+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"240'9465","reported_seq":9632,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:16.457711+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:20:16.457711+0000","last_peered":"2024-08-29T15:20:16.457711+0000","last_clean":"2024-08-29T15:20:16.457711+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:20:16.457711+0000","last_undegraded":"2024-08-29T15:20:16.457711+0000","last_fullsized":"2024-08-29T15:20:16.457711+0000","mapping_epoch":234,"log_start":"178'2797","ondisk_log_start":"178'2797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409403392,"num_objects":6247,"num_object_clones":0,"num_object_copies":18741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9465,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":176,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.265563+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:20:11.265563+0000","last_peered":"2024-08-29T15:20:11.265563+0000","last_clean":"2024-08-29T15:20:11.265563+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:20:11.265563+0000","last_undegraded":"2024-08-29T15:20:11.265563+0000","last_fullsized":"2024-08-29T15:20:11.265563+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267466+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:20:11.267466+0000","last_peered":"2024-08-29T15:20:11.267466+0000","last_clean":"2024-08-29T15:20:11.267466+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:20:11.267466+0000","last_undegraded":"2024-08-29T15:20:11.267466+0000","last_fullsized":"2024-08-29T15:20:11.267466+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.266075+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:20:11.266075+0000","last_peered":"2024-08-29T15:20:11.266075+0000","last_clean":"2024-08-29T15:20:11.266075+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:20:11.266075+0000","last_undegraded":"2024-08-29T15:20:11.266075+0000","last_fullsized":"2024-08-29T15:20:11.266075+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"239'9508","reported_seq":9673,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:13.487445+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:20:13.487445+0000","last_peered":"2024-08-29T15:20:13.487445+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:20:13.487445+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"175'1342","ondisk_log_start":"175'1342","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414056448,"num_objects":6318,"num_object_clones":0,"num_object_copies":18954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9508,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3895}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.266160+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:20:11.266160+0000","last_peered":"2024-08-29T15:20:11.266160+0000","last_clean":"2024-08-29T15:20:11.266160+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:20:11.266160+0000","last_undegraded":"2024-08-29T15:20:11.266160+0000","last_fullsized":"2024-08-29T15:20:11.266160+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":770,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265162+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:20:13.265162+0000","last_peered":"2024-08-29T15:20:13.265162+0000","last_clean":"2024-08-29T15:20:13.265162+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:20:13.265162+0000","last_undegraded":"2024-08-29T15:20:13.265162+0000","last_fullsized":"2024-08-29T15:20:13.265162+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":347,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265511+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:20:13.265511+0000","last_peered":"2024-08-29T15:20:13.265511+0000","last_clean":"2024-08-29T15:20:13.265511+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:20:13.265511+0000","last_undegraded":"2024-08-29T15:20:13.265511+0000","last_fullsized":"2024-08-29T15:20:13.265511+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"240'9435","reported_seq":10156,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:15.550136+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:20:15.550136+0000","last_peered":"2024-08-29T15:20:15.550136+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:20:15.550136+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"177'2380","ondisk_log_start":"177'2380","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408485888,"num_objects":6233,"num_object_clones":0,"num_object_copies":18699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":6195,"num_objects_unfound":0,"num_objects_dirty":6233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9435,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":4082}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":352,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265070+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:20:13.265070+0000","last_peered":"2024-08-29T15:20:13.265070+0000","last_clean":"2024-08-29T15:20:13.265070+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:20:13.265070+0000","last_undegraded":"2024-08-29T15:20:13.265070+0000","last_fullsized":"2024-08-29T15:20:13.265070+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":665,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.997753+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:20:11.997753+0000","last_peered":"2024-08-29T15:20:11.997753+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:20:11.997753+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:20:11.997753+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":344,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.487966+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:20:13.487966+0000","last_peered":"2024-08-29T15:20:13.487966+0000","last_clean":"2024-08-29T15:20:13.487966+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:20:13.487966+0000","last_undegraded":"2024-08-29T15:20:13.487966+0000","last_fullsized":"2024-08-29T15:20:13.487966+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"240'9311","reported_seq":16434,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:14.546698+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:20:14.546698+0000","last_peered":"2024-08-29T15:20:14.546698+0000","last_clean":"2024-08-29T15:20:14.546698+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:20:14.546698+0000","last_undegraded":"2024-08-29T15:20:14.546698+0000","last_fullsized":"2024-08-29T15:20:14.546698+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6912,"log_dups_size":0,"ondisk_log_size":6912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400883712,"num_objects":6117,"num_object_clones":0,"num_object_copies":18351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9311,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":338,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.262994+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:20:13.262994+0000","last_peered":"2024-08-29T15:20:13.262994+0000","last_clean":"2024-08-29T15:20:13.262994+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:20:13.262994+0000","last_undegraded":"2024-08-29T15:20:13.262994+0000","last_fullsized":"2024-08-29T15:20:13.262994+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":960,"reported_epoch":240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:13.264303+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:20:13.264303+0000","last_peered":"2024-08-29T15:20:13.264303+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:20:13.264303+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":335,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.261950+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:20:13.261950+0000","last_peered":"2024-08-29T15:20:13.261950+0000","last_clean":"2024-08-29T15:20:13.261950+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:20:13.261950+0000","last_undegraded":"2024-08-29T15:20:13.261950+0000","last_fullsized":"2024-08-29T15:20:13.261950+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"240'9448","reported_seq":10634,"reported_epoch":240,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:16.936288+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:20:16.936288+0000","last_peered":"2024-08-29T15:20:16.936288+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:20:16.936288+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"176'1912","ondisk_log_start":"176'1912","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7536,"log_dups_size":0,"ondisk_log_size":7536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403832832,"num_objects":6162,"num_object_clones":0,"num_object_copies":18486,"num_objects_missing_on_primary":7197,"num_objects_missing":7197,"num_objects_degraded":2313,"num_objects_misplaced":6148,"num_objects_unfound":0,"num_objects_dirty":6162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9448,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":525,"num_bytes_recovered":34275328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":3777},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":333,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488240+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:20:13.488240+0000","last_peered":"2024-08-29T15:20:13.488240+0000","last_clean":"2024-08-29T15:20:13.488240+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:20:13.488240+0000","last_undegraded":"2024-08-29T15:20:13.488240+0000","last_fullsized":"2024-08-29T15:20:13.488240+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":89,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.929893+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:20:11.929893+0000","last_peered":"2024-08-29T15:20:11.929893+0000","last_clean":"2024-08-29T15:20:11.929893+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:20:11.929893+0000","last_undegraded":"2024-08-29T15:20:11.929893+0000","last_fullsized":"2024-08-29T15:20:11.929893+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267456+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:20:11.267456+0000","last_peered":"2024-08-29T15:20:11.267456+0000","last_clean":"2024-08-29T15:20:11.267456+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:20:11.267456+0000","last_undegraded":"2024-08-29T15:20:11.267456+0000","last_fullsized":"2024-08-29T15:20:11.267456+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267553+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:20:11.267553+0000","last_peered":"2024-08-29T15:20:11.267553+0000","last_clean":"2024-08-29T15:20:11.267553+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:20:11.267553+0000","last_undegraded":"2024-08-29T15:20:11.267553+0000","last_fullsized":"2024-08-29T15:20:11.267553+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":15,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.267837+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:20:11.267837+0000","last_peered":"2024-08-29T15:20:11.267837+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:20:11.267837+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:20:11.267837+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":342,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265365+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:20:13.265365+0000","last_peered":"2024-08-29T15:20:13.265365+0000","last_clean":"2024-08-29T15:20:13.265365+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:20:13.265365+0000","last_undegraded":"2024-08-29T15:20:13.265365+0000","last_fullsized":"2024-08-29T15:20:13.265365+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":92,"reported_epoch":240,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:13.263040+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:20:13.263040+0000","last_peered":"2024-08-29T15:20:13.263040+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:20:13.263040+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:20:13.263040+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.265663+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:20:11.265663+0000","last_peered":"2024-08-29T15:20:11.265663+0000","last_clean":"2024-08-29T15:20:11.265663+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:20:11.265663+0000","last_undegraded":"2024-08-29T15:20:11.265663+0000","last_fullsized":"2024-08-29T15:20:11.265663+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":301,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.261850+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:20:13.261850+0000","last_peered":"2024-08-29T15:20:13.261850+0000","last_clean":"2024-08-29T15:20:13.261850+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:20:13.261850+0000","last_undegraded":"2024-08-29T15:20:13.261850+0000","last_fullsized":"2024-08-29T15:20:13.261850+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":355,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.487801+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:20:13.487801+0000","last_peered":"2024-08-29T15:20:13.487801+0000","last_clean":"2024-08-29T15:20:13.487801+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:20:13.487801+0000","last_undegraded":"2024-08-29T15:20:13.487801+0000","last_fullsized":"2024-08-29T15:20:13.487801+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":353,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.261885+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:20:13.261885+0000","last_peered":"2024-08-29T15:20:13.261885+0000","last_clean":"2024-08-29T15:20:13.261885+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:20:13.261885+0000","last_undegraded":"2024-08-29T15:20:13.261885+0000","last_fullsized":"2024-08-29T15:20:13.261885+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":77,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.929904+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:20:11.929904+0000","last_peered":"2024-08-29T15:20:11.929904+0000","last_clean":"2024-08-29T15:20:11.929904+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:20:11.929904+0000","last_undegraded":"2024-08-29T15:20:11.929904+0000","last_fullsized":"2024-08-29T15:20:11.929904+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":210,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.487498+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:20:13.487498+0000","last_peered":"2024-08-29T15:20:13.487498+0000","last_clean":"2024-08-29T15:20:13.487498+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:20:13.487498+0000","last_undegraded":"2024-08-29T15:20:13.487498+0000","last_fullsized":"2024-08-29T15:20:13.487498+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":330,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.488513+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:20:13.488513+0000","last_peered":"2024-08-29T15:20:13.488513+0000","last_clean":"2024-08-29T15:20:13.488513+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:20:13.488513+0000","last_undegraded":"2024-08-29T15:20:13.488513+0000","last_fullsized":"2024-08-29T15:20:13.488513+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":334,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.264991+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:20:13.264991+0000","last_peered":"2024-08-29T15:20:13.264991+0000","last_clean":"2024-08-29T15:20:13.264991+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:20:13.264991+0000","last_undegraded":"2024-08-29T15:20:13.264991+0000","last_fullsized":"2024-08-29T15:20:13.264991+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":338,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.262895+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:20:13.262895+0000","last_peered":"2024-08-29T15:20:13.262895+0000","last_clean":"2024-08-29T15:20:13.262895+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:20:13.262895+0000","last_undegraded":"2024-08-29T15:20:13.262895+0000","last_fullsized":"2024-08-29T15:20:13.262895+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267563+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:20:11.267563+0000","last_peered":"2024-08-29T15:20:11.267563+0000","last_clean":"2024-08-29T15:20:11.267563+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:20:11.267563+0000","last_undegraded":"2024-08-29T15:20:11.267563+0000","last_fullsized":"2024-08-29T15:20:11.267563+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":358,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.265255+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:20:13.265255+0000","last_peered":"2024-08-29T15:20:13.265255+0000","last_clean":"2024-08-29T15:20:13.265255+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:20:13.265255+0000","last_undegraded":"2024-08-29T15:20:13.265255+0000","last_fullsized":"2024-08-29T15:20:13.265255+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":322,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.487855+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:20:13.487855+0000","last_peered":"2024-08-29T15:20:13.487855+0000","last_clean":"2024-08-29T15:20:13.487855+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:20:13.487855+0000","last_undegraded":"2024-08-29T15:20:13.487855+0000","last_fullsized":"2024-08-29T15:20:13.487855+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267471+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:20:11.267471+0000","last_peered":"2024-08-29T15:20:11.267471+0000","last_clean":"2024-08-29T15:20:11.267471+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:20:11.267471+0000","last_undegraded":"2024-08-29T15:20:11.267471+0000","last_fullsized":"2024-08-29T15:20:11.267471+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":130,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.265651+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:20:11.265651+0000","last_peered":"2024-08-29T15:20:11.265651+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:20:11.265651+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:20:11.265651+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267648+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:20:11.267648+0000","last_peered":"2024-08-29T15:20:11.267648+0000","last_clean":"2024-08-29T15:20:11.267648+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:20:11.267648+0000","last_undegraded":"2024-08-29T15:20:11.267648+0000","last_fullsized":"2024-08-29T15:20:11.267648+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":136,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.266310+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:20:11.266310+0000","last_peered":"2024-08-29T15:20:11.266310+0000","last_clean":"2024-08-29T15:20:11.266310+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:20:11.266310+0000","last_undegraded":"2024-08-29T15:20:11.266310+0000","last_fullsized":"2024-08-29T15:20:11.266310+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":369,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.262246+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:20:13.262246+0000","last_peered":"2024-08-29T15:20:13.262246+0000","last_clean":"2024-08-29T15:20:13.262246+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:20:13.262246+0000","last_undegraded":"2024-08-29T15:20:13.262246+0000","last_fullsized":"2024-08-29T15:20:13.262246+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":370,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.263339+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:20:13.263339+0000","last_peered":"2024-08-29T15:20:13.263339+0000","last_clean":"2024-08-29T15:20:13.263339+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:20:13.263339+0000","last_undegraded":"2024-08-29T15:20:13.263339+0000","last_fullsized":"2024-08-29T15:20:13.263339+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267281+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:20:11.267281+0000","last_peered":"2024-08-29T15:20:11.267281+0000","last_clean":"2024-08-29T15:20:11.267281+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:20:11.267281+0000","last_undegraded":"2024-08-29T15:20:11.267281+0000","last_fullsized":"2024-08-29T15:20:11.267281+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":378,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.487700+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:20:13.487700+0000","last_peered":"2024-08-29T15:20:13.487700+0000","last_clean":"2024-08-29T15:20:13.487700+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:20:13.487700+0000","last_undegraded":"2024-08-29T15:20:13.487700+0000","last_fullsized":"2024-08-29T15:20:13.487700+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6533349400,"num_objects":99692,"num_object_clones":0,"num_object_copies":299076,"num_objects_missing_on_primary":7197,"num_objects_missing":7197,"num_objects_degraded":30073,"num_objects_misplaced":18599,"num_objects_unfound":0,"num_objects_dirty":99692,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":151158,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14152,"num_bytes_recovered":807469056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19458764800,"data_stored":19458752584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119235,"ondisk_log_size":119235,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234307,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3494620,"kb_used_data":3206736,"kb_used_omap":1,"kb_used_meta":287870,"kb_avail":90250532,"statfs":{"total":95995035648,"available":92416544768,"internally_reserved":0,"allocated":3283697664,"data_stored":3282365563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":294779339},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495940,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":511060,"kb_used_data":345420,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":93234092,"statfs":{"total":95995035648,"available":95471710208,"internally_reserved":0,"allocated":353710080,"data_stored":352381779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":169605575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2,0,2,8,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426525,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2040008,"kb_used_data":1988992,"kb_used_omap":8,"kb_used_meta":50999,"kb_avail":91705144,"statfs":{"total":95995035648,"available":93906067456,"internally_reserved":0,"allocated":2036727808,"data_stored":2035406973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":52223882},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019217,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2275776,"kb_used_data":1991984,"kb_used_omap":1,"kb_used_meta":283774,"kb_avail":91469376,"statfs":{"total":95995035648,"available":93664641024,"internally_reserved":0,"allocated":2039791616,"data_stored":2038486564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290585030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435402,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4251524,"kb_used_data":3981116,"kb_used_omap":4,"kb_used_meta":270395,"kb_avail":89493628,"statfs":{"total":95995035648,"available":91641475072,"internally_reserved":0,"allocated":4076662784,"data_stored":4075345889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":276884745},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903254,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3572936,"kb_used_data":3491200,"kb_used_omap":10,"kb_used_meta":81717,"kb_avail":90172216,"statfs":{"total":95995035648,"available":92336349184,"internally_reserved":0,"allocated":3574988800,"data_stored":3573671841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":83678736},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762455,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2678216,"kb_used_data":2425788,"kb_used_omap":7,"kb_used_meta":252408,"kb_avail":91066936,"statfs":{"total":95995035648,"available":93252542464,"internally_reserved":0,"allocated":2484006912,"data_stored":2482688777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":258466146},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925981,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1925136,"kb_used_data":1618564,"kb_used_omap":7,"kb_used_meta":306552,"kb_avail":91820016,"statfs":{"total":95995035648,"available":94023696384,"internally_reserved":0,"allocated":1657409536,"data_stored":1656084180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":313909391},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1649545216,"data_stored":1649541144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2476150784,"data_stored":2476146712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3570466816,"data_stored":3570466816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4071424000,"data_stored":4071424000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2035220480,"data_stored":2035220480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2033258496,"data_stored":2033254424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":349962240,"data_stored":349962240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3272736768,"data_stored":3272736768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:20:19.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:20:19.943 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:19.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:19.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:19.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:20.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:20.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:20.180 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:20.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:20.222 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:20.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:20.224 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:20.289 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:20:20.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.349 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:20.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:20.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:20.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:20 smithi027 ceph-mon[93921]: pgmap v718: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 30102/299160 objects degraded (10.062%); 18599/299160 objects misplaced (6.217%); 544 KiB/s, 8 objects/s recovering 2024-08-29T15:20:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:20 smithi027 ceph-mon[93921]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:20 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4239448636' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:20:20.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:20 smithi027 ceph-mon[98970]: pgmap v718: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 30102/299160 objects degraded (10.062%); 18599/299160 objects misplaced (6.217%); 544 KiB/s, 8 objects/s recovering 2024-08-29T15:20:20.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:20 smithi027 ceph-mon[98970]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:20.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:20 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4239448636' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:20:20.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:20.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:20.669 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:20 smithi171 ceph-mon[90031]: pgmap v718: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 30102/299160 objects degraded (10.062%); 18599/299160 objects misplaced (6.217%); 544 KiB/s, 8 objects/s recovering 2024-08-29T15:20:20.670 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:20 smithi171 ceph-mon[90031]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:20.670 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:20 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4239448636' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:20:20.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:20.731 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:20.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:20.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:20.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:20.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:20.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:20.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:21.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:21.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:21.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:21.210 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:21.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:21.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:21.262 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:21.262 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:21.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:21.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:21.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:21.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:21.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:21 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4239448636' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:20:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:21 smithi171 ceph-mon[90031]: osdmap e241: 8 total, 8 up, 8 in 2024-08-29T15:20:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:21 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4239448636' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:20:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:21 smithi027 ceph-mon[98970]: osdmap e241: 8 total, 8 up, 8 in 2024-08-29T15:20:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:21 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4239448636' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:20:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:21 smithi027 ceph-mon[93921]: osdmap e241: 8 total, 8 up, 8 in 2024-08-29T15:20:21.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:21.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:21.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:21.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:21.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:21.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:21.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:21.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:21.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:21.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:21.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:21.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:20:22.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:22.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:22.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:22.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:22.214 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:22.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:22.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:22.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:22.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:22.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:22.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:22.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:22.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:22 smithi027 ceph-mon[98970]: pgmap v720: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 30102/298695 objects degraded (10.078%); 18599/298695 objects misplaced (6.227%); 168 KiB/s, 2 objects/s recovering 2024-08-29T15:20:22.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/298695 objects degraded (10.078%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:22 smithi027 ceph-mon[93921]: pgmap v720: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 30102/298695 objects degraded (10.078%); 18599/298695 objects misplaced (6.227%); 168 KiB/s, 2 objects/s recovering 2024-08-29T15:20:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/298695 objects degraded (10.078%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:22.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:22.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:22.624 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:22 smithi171 ceph-mon[90031]: pgmap v720: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 78 op/s; 30102/298695 objects degraded (10.078%); 18599/298695 objects misplaced (6.227%); 168 KiB/s, 2 objects/s recovering 2024-08-29T15:20:22.625 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/298695 objects degraded (10.078%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:22.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:22.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:22.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:22.858 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:22.888 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:20:22.889 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:20:22.904 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:20:22.904 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:20:22.919 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:20:22.919 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:22.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:22.936 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:22.953 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:22.953 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:22.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:22.971 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:22.971 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:22.982 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:22.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:22.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:22.987 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:22.987 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:23.004 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:23.004 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:23.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:23.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:23.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:23.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:23.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:23.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:23.357 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:23.405 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1632698179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:23.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:23.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1632698179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:23.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1632698179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:23.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:23.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:23.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:23.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:23.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:23.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:23.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:23.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:23.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:23.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:23.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:24.062 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:20:24.062 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:20:24.062 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:20:24.062 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:20:24.063 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:20:24.083 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:24.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:24.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:24.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:24.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:24.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:24.458 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:24.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:24.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:24 smithi027 ceph-mon[93921]: pgmap v721: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 30102/297450 objects degraded (10.120%); 18496/297450 objects misplaced (6.218%); 372 KiB/s, 5 objects/s recovering 2024-08-29T15:20:24.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:24 smithi027 ceph-mon[98970]: pgmap v721: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 30102/297450 objects degraded (10.120%); 18496/297450 objects misplaced (6.218%); 372 KiB/s, 5 objects/s recovering 2024-08-29T15:20:24.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:24.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:24.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:24.609 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:24 smithi171 ceph-mon[90031]: pgmap v721: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 30102/297450 objects degraded (10.120%); 18496/297450 objects misplaced (6.218%); 372 KiB/s, 5 objects/s recovering 2024-08-29T15:20:24.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:20:24.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:24.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:24.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:24.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:20:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:24.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:24.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:24.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:24.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:25.067 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:25.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:25.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:25.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:25.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:25.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:25.255 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:25.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:25.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:25.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:25.431 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:20:25.432 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:20:25.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:25.487 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:25.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:25.588 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:25.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:25.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:25.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:25.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:25.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:25.966 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:25.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:25.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:26.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:26.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:26.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:26.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:20:26.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:26.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:26.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:26.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:26.246 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":722,"stamp":"2024-08-29T15:20:24.538398+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6497835005,"num_objects":99105,"num_object_clones":0,"num_object_copies":297315,"num_objects_missing_on_primary":7171,"num_objects_missing":7171,"num_objects_degraded":30102,"num_objects_misplaced":18496,"num_objects_unfound":0,"num_objects_dirty":99105,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":152896,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14294,"num_bytes_recovered":815598797,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120532,"ondisk_log_size":120532,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20643640,"kb_used_data":18940072,"kb_used_omap":43,"kb_used_meta":1703444,"kb_avail":729317576,"statfs":{"total":767960285184,"available":746821197824,"internally_reserved":0,"allocated":19394633728,"data_stored":19384024206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1744327148},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-68157440,"num_objects":-1040,"num_object_clones":0,"num_object_copies":-3120,"num_objects_missing_on_primary":-68,"num_objects_missing":-68,"num_objects_degraded":0,"num_objects_misplaced":-194,"num_objects_unfound":0,"num_objects_dirty":-1040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1040,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":3538944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002736"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":365,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.462125+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:20:20.462125+0000","last_peered":"2024-08-29T15:20:20.462125+0000","last_clean":"2024-08-29T15:20:20.462125+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:20:20.462125+0000","last_undegraded":"2024-08-29T15:20:20.462125+0000","last_fullsized":"2024-08-29T15:20:20.462125+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.267782+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:20:11.267782+0000","last_peered":"2024-08-29T15:20:11.267782+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:20:11.267782+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:20:11.267782+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":107,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260729+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:20:13.260729+0000","last_peered":"2024-08-29T15:20:13.260729+0000","last_clean":"2024-08-29T15:20:13.260729+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:20:13.260729+0000","last_undegraded":"2024-08-29T15:20:13.260729+0000","last_fullsized":"2024-08-29T15:20:13.260729+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":331,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290282+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:20:20.290282+0000","last_peered":"2024-08-29T15:20:20.290282+0000","last_clean":"2024-08-29T15:20:20.290282+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:20:20.290282+0000","last_undegraded":"2024-08-29T15:20:20.290282+0000","last_fullsized":"2024-08-29T15:20:20.290282+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":331,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290716+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:20:20.290716+0000","last_peered":"2024-08-29T15:20:20.290716+0000","last_clean":"2024-08-29T15:20:20.290716+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:20:20.290716+0000","last_undegraded":"2024-08-29T15:20:20.290716+0000","last_fullsized":"2024-08-29T15:20:20.290716+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267490+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:20:11.267490+0000","last_peered":"2024-08-29T15:20:11.267490+0000","last_clean":"2024-08-29T15:20:11.267490+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:20:11.267490+0000","last_undegraded":"2024-08-29T15:20:11.267490+0000","last_fullsized":"2024-08-29T15:20:11.267490+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.465155+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:20:20.465155+0000","last_peered":"2024-08-29T15:20:20.465155+0000","last_clean":"2024-08-29T15:20:20.465155+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:20:20.465155+0000","last_undegraded":"2024-08-29T15:20:20.465155+0000","last_fullsized":"2024-08-29T15:20:20.465155+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":108,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.385002+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:20:13.385002+0000","last_peered":"2024-08-29T15:20:13.385002+0000","last_clean":"2024-08-29T15:20:13.385002+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:20:13.385002+0000","last_undegraded":"2024-08-29T15:20:13.385002+0000","last_fullsized":"2024-08-29T15:20:13.385002+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"241'9645","reported_seq":17663,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:21.112346+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:20:21.112346+0000","last_peered":"2024-08-29T15:20:21.112346+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:20:21.112346+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9645,"log_dups_size":0,"ondisk_log_size":9645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410714112,"num_objects":6267,"num_object_clones":0,"num_object_copies":18801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9645,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":3342}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.268001+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:20:11.268001+0000","last_peered":"2024-08-29T15:20:11.268001+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:20:11.268001+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:20:11.268001+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":125,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260066+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:20:13.260066+0000","last_peered":"2024-08-29T15:20:13.260066+0000","last_clean":"2024-08-29T15:20:13.260066+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:20:13.260066+0000","last_undegraded":"2024-08-29T15:20:13.260066+0000","last_fullsized":"2024-08-29T15:20:13.260066+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"241'9398","reported_seq":9557,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:21.654492+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:20:21.654492+0000","last_peered":"2024-08-29T15:20:21.654492+0000","last_clean":"2024-08-29T15:20:21.654492+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:20:21.654492+0000","last_undegraded":"2024-08-29T15:20:21.654492+0000","last_fullsized":"2024-08-29T15:20:21.654492+0000","mapping_epoch":168,"log_start":"178'2728","ondisk_log_start":"178'2728","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403177472,"num_objects":6152,"num_object_clones":0,"num_object_copies":18456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9398,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":15,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.271242+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:20:11.271242+0000","last_peered":"2024-08-29T15:20:11.271242+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:20:11.271242+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:20:11.271242+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267403+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:20:11.267403+0000","last_peered":"2024-08-29T15:20:11.267403+0000","last_clean":"2024-08-29T15:20:11.267403+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:20:11.267403+0000","last_undegraded":"2024-08-29T15:20:11.267403+0000","last_fullsized":"2024-08-29T15:20:11.267403+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"241'9476","reported_seq":9605,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:21.714956+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:20:21.714956+0000","last_peered":"2024-08-29T15:20:21.714956+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:20:21.714956+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"177'2774","ondisk_log_start":"177'2774","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408682496,"num_objects":6236,"num_object_clones":0,"num_object_copies":18708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9476,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3759}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":147,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290715+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:20:20.290715+0000","last_peered":"2024-08-29T15:20:20.290715+0000","last_clean":"2024-08-29T15:20:20.290715+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:20:20.290715+0000","last_undegraded":"2024-08-29T15:20:20.290715+0000","last_fullsized":"2024-08-29T15:20:20.290715+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":63,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.385197+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:20:13.385197+0000","last_peered":"2024-08-29T15:20:13.385197+0000","last_clean":"2024-08-29T15:20:13.385197+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:20:13.385197+0000","last_undegraded":"2024-08-29T15:20:13.385197+0000","last_fullsized":"2024-08-29T15:20:13.385197+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"241'9615","reported_seq":14994,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:23.709340+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:20:23.709340+0000","last_peered":"2024-08-29T15:20:23.709340+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:20:23.709340+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'1462","ondisk_log_start":"176'1462","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8153,"log_dups_size":0,"ondisk_log_size":8153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403898368,"num_objects":6163,"num_object_clones":0,"num_object_copies":18489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9615,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":6163}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":125,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.261005+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:20:13.261005+0000","last_peered":"2024-08-29T15:20:13.261005+0000","last_clean":"2024-08-29T15:20:13.261005+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:20:13.261005+0000","last_undegraded":"2024-08-29T15:20:13.261005+0000","last_fullsized":"2024-08-29T15:20:13.261005+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":345,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290353+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:20:20.290353+0000","last_peered":"2024-08-29T15:20:20.290353+0000","last_clean":"2024-08-29T15:20:20.290353+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:20:20.290353+0000","last_undegraded":"2024-08-29T15:20:20.290353+0000","last_fullsized":"2024-08-29T15:20:20.290353+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"240'9545","reported_seq":9670,"reported_epoch":240,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:19.615035+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:20:19.615035+0000","last_peered":"2024-08-29T15:20:19.615035+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:20:19.615035+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:20:19.615035+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7212,"log_dups_size":0,"ondisk_log_size":7212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410845184,"num_objects":6269,"num_object_clones":0,"num_object_copies":18807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9545,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3792},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":340,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.461962+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:20:20.461962+0000","last_peered":"2024-08-29T15:20:20.461962+0000","last_clean":"2024-08-29T15:20:20.461962+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:20:20.461962+0000","last_undegraded":"2024-08-29T15:20:20.461962+0000","last_fullsized":"2024-08-29T15:20:20.461962+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267765+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:20:11.267765+0000","last_peered":"2024-08-29T15:20:11.267765+0000","last_clean":"2024-08-29T15:20:11.267765+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:20:11.267765+0000","last_undegraded":"2024-08-29T15:20:11.267765+0000","last_fullsized":"2024-08-29T15:20:11.267765+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"241'9551","reported_seq":9726,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:21.039010+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:20:21.039010+0000","last_peered":"2024-08-29T15:20:21.039010+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:20:21.039010+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:20:21.039010+0000","mapping_epoch":234,"log_start":"178'2868","ondisk_log_start":"178'2868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412286976,"num_objects":6291,"num_object_clones":0,"num_object_copies":18873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9551,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3826},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.385112+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:20:13.385112+0000","last_peered":"2024-08-29T15:20:13.385112+0000","last_clean":"2024-08-29T15:20:13.385112+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:20:13.385112+0000","last_undegraded":"2024-08-29T15:20:13.385112+0000","last_fullsized":"2024-08-29T15:20:13.385112+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267847+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:20:11.267847+0000","last_peered":"2024-08-29T15:20:11.267847+0000","last_clean":"2024-08-29T15:20:11.267847+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:20:11.267847+0000","last_undegraded":"2024-08-29T15:20:11.267847+0000","last_fullsized":"2024-08-29T15:20:11.267847+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"241'9433","reported_seq":9793,"reported_epoch":241,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:20:21.611508+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:20:21.611508+0000","last_peered":"2024-08-29T15:20:21.611508+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:20:21.611508+0000","last_undegraded":"2024-08-29T15:20:21.611508+0000","last_fullsized":"2024-08-29T15:20:21.611508+0000","mapping_epoch":202,"log_start":"177'2667","ondisk_log_start":"177'2667","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6766,"log_dups_size":0,"ondisk_log_size":6766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407961600,"num_objects":6225,"num_object_clones":0,"num_object_copies":18675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6225,"num_objects_unfound":0,"num_objects_dirty":6225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9433,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":6225}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267379+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:20:11.267379+0000","last_peered":"2024-08-29T15:20:11.267379+0000","last_clean":"2024-08-29T15:20:11.267379+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:20:11.267379+0000","last_undegraded":"2024-08-29T15:20:11.267379+0000","last_fullsized":"2024-08-29T15:20:11.267379+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":222,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.271264+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:20:11.271264+0000","last_peered":"2024-08-29T15:20:11.271264+0000","last_clean":"2024-08-29T15:20:11.271264+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:20:11.271264+0000","last_undegraded":"2024-08-29T15:20:11.271264+0000","last_fullsized":"2024-08-29T15:20:11.271264+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"241'9519","reported_seq":9639,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.985822+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:20:20.985822+0000","last_peered":"2024-08-29T15:20:20.985822+0000","last_clean":"2024-08-29T15:20:20.985822+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:20:20.985822+0000","last_undegraded":"2024-08-29T15:20:20.985822+0000","last_fullsized":"2024-08-29T15:20:20.985822+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8241,"log_dups_size":0,"ondisk_log_size":8241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408944664,"num_objects":6241,"num_object_clones":0,"num_object_copies":18723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6241,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9519,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":343,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.462226+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:20:20.462226+0000","last_peered":"2024-08-29T15:20:20.462226+0000","last_clean":"2024-08-29T15:20:20.462226+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:20:20.462226+0000","last_undegraded":"2024-08-29T15:20:20.462226+0000","last_fullsized":"2024-08-29T15:20:20.462226+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":335,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.465244+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:20:20.465244+0000","last_peered":"2024-08-29T15:20:20.465244+0000","last_clean":"2024-08-29T15:20:20.465244+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:20:20.465244+0000","last_undegraded":"2024-08-29T15:20:20.465244+0000","last_fullsized":"2024-08-29T15:20:20.465244+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"241'9222","reported_seq":9390,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.976216+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:20:20.976216+0000","last_peered":"2024-08-29T15:20:20.976216+0000","last_clean":"2024-08-29T15:20:20.976216+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:20:20.976216+0000","last_undegraded":"2024-08-29T15:20:20.976216+0000","last_fullsized":"2024-08-29T15:20:20.976216+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9222,"log_dups_size":0,"ondisk_log_size":9222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392298496,"num_objects":5986,"num_object_clones":0,"num_object_copies":17958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9222,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":434,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290104+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:20:20.290104+0000","last_peered":"2024-08-29T15:20:20.290104+0000","last_clean":"2024-08-29T15:20:20.290104+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:20:20.290104+0000","last_undegraded":"2024-08-29T15:20:20.290104+0000","last_fullsized":"2024-08-29T15:20:20.290104+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":709,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260208+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:20:13.260208+0000","last_peered":"2024-08-29T15:20:13.260208+0000","last_clean":"2024-08-29T15:20:13.260208+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:20:13.260208+0000","last_undegraded":"2024-08-29T15:20:13.260208+0000","last_fullsized":"2024-08-29T15:20:13.260208+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":217,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260613+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:20:13.260613+0000","last_peered":"2024-08-29T15:20:13.260613+0000","last_clean":"2024-08-29T15:20:13.260613+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:20:13.260613+0000","last_undegraded":"2024-08-29T15:20:13.260613+0000","last_fullsized":"2024-08-29T15:20:13.260613+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":79,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.385459+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:20:13.385459+0000","last_peered":"2024-08-29T15:20:13.385459+0000","last_clean":"2024-08-29T15:20:13.385459+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:20:13.385459+0000","last_undegraded":"2024-08-29T15:20:13.385459+0000","last_fullsized":"2024-08-29T15:20:13.385459+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"240'9556","reported_seq":9913,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.465081+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:20:20.465081+0000","last_peered":"2024-08-29T15:20:20.465081+0000","last_clean":"2024-08-29T15:20:20.465081+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:20:20.465081+0000","last_undegraded":"2024-08-29T15:20:20.465081+0000","last_fullsized":"2024-08-29T15:20:20.465081+0000","mapping_epoch":201,"log_start":"177'2553","ondisk_log_start":"177'2553","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7003,"log_dups_size":0,"ondisk_log_size":7003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401473536,"num_objects":6126,"num_object_clones":0,"num_object_copies":18378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9556,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":808,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.464362+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:20:20.464362+0000","last_peered":"2024-08-29T15:20:20.464362+0000","last_clean":"2024-08-29T15:20:20.464362+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:20:20.464362+0000","last_undegraded":"2024-08-29T15:20:20.464362+0000","last_fullsized":"2024-08-29T15:20:20.464362+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":346,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290452+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:20:20.290452+0000","last_peered":"2024-08-29T15:20:20.290452+0000","last_clean":"2024-08-29T15:20:20.290452+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:20:20.290452+0000","last_undegraded":"2024-08-29T15:20:20.290452+0000","last_fullsized":"2024-08-29T15:20:20.290452+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":354,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290990+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:20:20.290990+0000","last_peered":"2024-08-29T15:20:20.290990+0000","last_clean":"2024-08-29T15:20:20.290990+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:20:20.290990+0000","last_undegraded":"2024-08-29T15:20:20.290990+0000","last_fullsized":"2024-08-29T15:20:20.290990+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"241'9443","reported_seq":15223,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:21.664736+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:20:21.664736+0000","last_peered":"2024-08-29T15:20:21.664736+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:20:21.664736+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'2730","ondisk_log_start":"178'2730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6713,"log_dups_size":0,"ondisk_log_size":6713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409403392,"num_objects":6247,"num_object_clones":0,"num_object_copies":18741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9443,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1813},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":616,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.461144+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:20:20.461144+0000","last_peered":"2024-08-29T15:20:20.461144+0000","last_clean":"2024-08-29T15:20:20.461144+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:20:20.461144+0000","last_undegraded":"2024-08-29T15:20:20.461144+0000","last_fullsized":"2024-08-29T15:20:20.461144+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":70,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:16.966536+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:20:16.966536+0000","last_peered":"2024-08-29T15:20:16.966536+0000","last_clean":"2024-08-29T15:20:16.966536+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:20:16.966536+0000","last_undegraded":"2024-08-29T15:20:16.966536+0000","last_fullsized":"2024-08-29T15:20:16.966536+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":380,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.464497+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:20:20.464497+0000","last_peered":"2024-08-29T15:20:20.464497+0000","last_clean":"2024-08-29T15:20:20.464497+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:20:20.464497+0000","last_undegraded":"2024-08-29T15:20:20.464497+0000","last_fullsized":"2024-08-29T15:20:20.464497+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"241'9497","reported_seq":9666,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:21.689069+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:20:21.689069+0000","last_peered":"2024-08-29T15:20:21.689069+0000","last_clean":"2024-08-29T15:20:21.689069+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:20:21.689069+0000","last_undegraded":"2024-08-29T15:20:21.689069+0000","last_fullsized":"2024-08-29T15:20:21.689069+0000","mapping_epoch":234,"log_start":"178'2797","ondisk_log_start":"178'2797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407306240,"num_objects":6215,"num_object_clones":0,"num_object_copies":18645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9497,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":177,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260100+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:20:13.260100+0000","last_peered":"2024-08-29T15:20:13.260100+0000","last_clean":"2024-08-29T15:20:13.260100+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:20:13.260100+0000","last_undegraded":"2024-08-29T15:20:13.260100+0000","last_fullsized":"2024-08-29T15:20:13.260100+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267466+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:20:11.267466+0000","last_peered":"2024-08-29T15:20:11.267466+0000","last_clean":"2024-08-29T15:20:11.267466+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:20:11.267466+0000","last_undegraded":"2024-08-29T15:20:11.267466+0000","last_fullsized":"2024-08-29T15:20:11.267466+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":84,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260319+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:20:13.260319+0000","last_peered":"2024-08-29T15:20:13.260319+0000","last_clean":"2024-08-29T15:20:13.260319+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:20:13.260319+0000","last_undegraded":"2024-08-29T15:20:13.260319+0000","last_fullsized":"2024-08-29T15:20:13.260319+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"241'9564","reported_seq":9731,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:23.823044+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:20:23.823044+0000","last_peered":"2024-08-29T15:20:23.823044+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:20:23.823044+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"176'1442","ondisk_log_start":"176'1442","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410386432,"num_objects":6262,"num_object_clones":0,"num_object_copies":18786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9564,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3839}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260660+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:20:13.260660+0000","last_peered":"2024-08-29T15:20:13.260660+0000","last_clean":"2024-08-29T15:20:13.260660+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:20:13.260660+0000","last_undegraded":"2024-08-29T15:20:13.260660+0000","last_fullsized":"2024-08-29T15:20:13.260660+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":773,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.465318+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:20:20.465318+0000","last_peered":"2024-08-29T15:20:20.465318+0000","last_clean":"2024-08-29T15:20:20.465318+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:20:20.465318+0000","last_undegraded":"2024-08-29T15:20:20.465318+0000","last_fullsized":"2024-08-29T15:20:20.465318+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":350,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.464903+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:20:20.464903+0000","last_peered":"2024-08-29T15:20:20.464903+0000","last_clean":"2024-08-29T15:20:20.464903+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:20:20.464903+0000","last_undegraded":"2024-08-29T15:20:20.464903+0000","last_fullsized":"2024-08-29T15:20:20.464903+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"241'9473","reported_seq":10196,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:20.468361+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:20:20.468361+0000","last_peered":"2024-08-29T15:20:20.468361+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:20:20.468361+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"177'2480","ondisk_log_start":"177'2480","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6993,"log_dups_size":0,"ondisk_log_size":6993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405995520,"num_objects":6195,"num_object_clones":0,"num_object_copies":18585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":6157,"num_objects_unfound":0,"num_objects_dirty":6195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9473,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":4044}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":355,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.465227+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:20:20.465227+0000","last_peered":"2024-08-29T15:20:20.465227+0000","last_clean":"2024-08-29T15:20:20.465227+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:20:20.465227+0000","last_undegraded":"2024-08-29T15:20:20.465227+0000","last_fullsized":"2024-08-29T15:20:20.465227+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":665,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.997753+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:20:11.997753+0000","last_peered":"2024-08-29T15:20:11.997753+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:20:11.997753+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:20:11.997753+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":346,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290264+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:20:20.290264+0000","last_peered":"2024-08-29T15:20:20.290264+0000","last_clean":"2024-08-29T15:20:20.290264+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:20:20.290264+0000","last_undegraded":"2024-08-29T15:20:20.290264+0000","last_fullsized":"2024-08-29T15:20:20.290264+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"240'9354","reported_seq":16477,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:19.444354+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:20:19.444354+0000","last_peered":"2024-08-29T15:20:19.444354+0000","last_clean":"2024-08-29T15:20:19.444354+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:20:19.444354+0000","last_undegraded":"2024-08-29T15:20:19.444354+0000","last_fullsized":"2024-08-29T15:20:19.444354+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6955,"log_dups_size":0,"ondisk_log_size":6955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398065664,"num_objects":6074,"num_object_clones":0,"num_object_copies":18222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9354,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":340,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.468016+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:20:20.468016+0000","last_peered":"2024-08-29T15:20:20.468016+0000","last_clean":"2024-08-29T15:20:20.468016+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:20:20.468016+0000","last_undegraded":"2024-08-29T15:20:20.468016+0000","last_fullsized":"2024-08-29T15:20:20.468016+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":962,"reported_epoch":241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:20.464087+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:20:20.464087+0000","last_peered":"2024-08-29T15:20:20.464087+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:20:20.464087+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":337,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.461153+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:20:20.461153+0000","last_peered":"2024-08-29T15:20:20.461153+0000","last_clean":"2024-08-29T15:20:20.461153+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:20:20.461153+0000","last_undegraded":"2024-08-29T15:20:20.461153+0000","last_fullsized":"2024-08-29T15:20:20.461153+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"241'9482","reported_seq":10737,"reported_epoch":241,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:22.217891+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:20:22.217891+0000","last_peered":"2024-08-29T15:20:22.217891+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:20:22.217891+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"176'1912","ondisk_log_start":"176'1912","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401604608,"num_objects":6128,"num_object_clones":0,"num_object_copies":18384,"num_objects_missing_on_primary":7163,"num_objects_missing":7163,"num_objects_degraded":2313,"num_objects_misplaced":6114,"num_objects_unfound":0,"num_objects_dirty":6128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9482,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":558,"num_bytes_recovered":36438016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":3743},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":336,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290636+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:20:20.290636+0000","last_peered":"2024-08-29T15:20:20.290636+0000","last_clean":"2024-08-29T15:20:20.290636+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:20:20.290636+0000","last_undegraded":"2024-08-29T15:20:20.290636+0000","last_fullsized":"2024-08-29T15:20:20.290636+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":90,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.385332+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:20:13.385332+0000","last_peered":"2024-08-29T15:20:13.385332+0000","last_clean":"2024-08-29T15:20:13.385332+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:20:13.385332+0000","last_undegraded":"2024-08-29T15:20:13.385332+0000","last_fullsized":"2024-08-29T15:20:13.385332+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267456+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:20:11.267456+0000","last_peered":"2024-08-29T15:20:11.267456+0000","last_clean":"2024-08-29T15:20:11.267456+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:20:11.267456+0000","last_undegraded":"2024-08-29T15:20:11.267456+0000","last_fullsized":"2024-08-29T15:20:11.267456+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267553+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:20:11.267553+0000","last_peered":"2024-08-29T15:20:11.267553+0000","last_clean":"2024-08-29T15:20:11.267553+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:20:11.267553+0000","last_undegraded":"2024-08-29T15:20:11.267553+0000","last_fullsized":"2024-08-29T15:20:11.267553+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":15,"reported_epoch":239,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:11.267837+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:20:11.267837+0000","last_peered":"2024-08-29T15:20:11.267837+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:20:11.267837+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:20:11.267837+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":344,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.464663+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:20:20.464663+0000","last_peered":"2024-08-29T15:20:20.464663+0000","last_clean":"2024-08-29T15:20:20.464663+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:20:20.464663+0000","last_undegraded":"2024-08-29T15:20:20.464663+0000","last_fullsized":"2024-08-29T15:20:20.464663+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":94,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:20.462597+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:20:20.462597+0000","last_peered":"2024-08-29T15:20:20.462597+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:20:20.462597+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:20:20.462597+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":107,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260515+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:20:13.260515+0000","last_peered":"2024-08-29T15:20:13.260515+0000","last_clean":"2024-08-29T15:20:13.260515+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:20:13.260515+0000","last_undegraded":"2024-08-29T15:20:13.260515+0000","last_fullsized":"2024-08-29T15:20:13.260515+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":303,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.461013+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:20:20.461013+0000","last_peered":"2024-08-29T15:20:20.461013+0000","last_clean":"2024-08-29T15:20:20.461013+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:20:20.461013+0000","last_undegraded":"2024-08-29T15:20:20.461013+0000","last_fullsized":"2024-08-29T15:20:20.461013+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":357,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290846+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:20:20.290846+0000","last_peered":"2024-08-29T15:20:20.290846+0000","last_clean":"2024-08-29T15:20:20.290846+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:20:20.290846+0000","last_undegraded":"2024-08-29T15:20:20.290846+0000","last_fullsized":"2024-08-29T15:20:20.290846+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":355,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.461020+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:20:20.461020+0000","last_peered":"2024-08-29T15:20:20.461020+0000","last_clean":"2024-08-29T15:20:20.461020+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:20:20.461020+0000","last_undegraded":"2024-08-29T15:20:20.461020+0000","last_fullsized":"2024-08-29T15:20:20.461020+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.385369+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:20:13.385369+0000","last_peered":"2024-08-29T15:20:13.385369+0000","last_clean":"2024-08-29T15:20:13.385369+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:20:13.385369+0000","last_undegraded":"2024-08-29T15:20:13.385369+0000","last_fullsized":"2024-08-29T15:20:13.385369+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.289993+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:20:20.289993+0000","last_peered":"2024-08-29T15:20:20.289993+0000","last_clean":"2024-08-29T15:20:20.289993+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:20:20.289993+0000","last_undegraded":"2024-08-29T15:20:20.289993+0000","last_fullsized":"2024-08-29T15:20:20.289993+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":332,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290902+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:20:20.290902+0000","last_peered":"2024-08-29T15:20:20.290902+0000","last_clean":"2024-08-29T15:20:20.290902+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:20:20.290902+0000","last_undegraded":"2024-08-29T15:20:20.290902+0000","last_fullsized":"2024-08-29T15:20:20.290902+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":336,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.464790+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:20:20.464790+0000","last_peered":"2024-08-29T15:20:20.464790+0000","last_clean":"2024-08-29T15:20:20.464790+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:20:20.464790+0000","last_undegraded":"2024-08-29T15:20:20.464790+0000","last_fullsized":"2024-08-29T15:20:20.464790+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":340,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.462461+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:20:20.462461+0000","last_peered":"2024-08-29T15:20:20.462461+0000","last_clean":"2024-08-29T15:20:20.462461+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:20:20.462461+0000","last_undegraded":"2024-08-29T15:20:20.462461+0000","last_fullsized":"2024-08-29T15:20:20.462461+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267563+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:20:11.267563+0000","last_peered":"2024-08-29T15:20:11.267563+0000","last_clean":"2024-08-29T15:20:11.267563+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:20:11.267563+0000","last_undegraded":"2024-08-29T15:20:11.267563+0000","last_fullsized":"2024-08-29T15:20:11.267563+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":360,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.465472+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:20:20.465472+0000","last_peered":"2024-08-29T15:20:20.465472+0000","last_clean":"2024-08-29T15:20:20.465472+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:20:20.465472+0000","last_undegraded":"2024-08-29T15:20:20.465472+0000","last_fullsized":"2024-08-29T15:20:20.465472+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":324,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290146+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:20:20.290146+0000","last_peered":"2024-08-29T15:20:20.290146+0000","last_clean":"2024-08-29T15:20:20.290146+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:20:20.290146+0000","last_undegraded":"2024-08-29T15:20:20.290146+0000","last_fullsized":"2024-08-29T15:20:20.290146+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267471+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:20:11.267471+0000","last_peered":"2024-08-29T15:20:11.267471+0000","last_clean":"2024-08-29T15:20:11.267471+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:20:11.267471+0000","last_undegraded":"2024-08-29T15:20:11.267471+0000","last_fullsized":"2024-08-29T15:20:11.267471+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":131,"reported_epoch":240,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:13.260931+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:20:13.260931+0000","last_peered":"2024-08-29T15:20:13.260931+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:20:13.260931+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:20:13.260931+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267648+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:20:11.267648+0000","last_peered":"2024-08-29T15:20:11.267648+0000","last_clean":"2024-08-29T15:20:11.267648+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:20:11.267648+0000","last_undegraded":"2024-08-29T15:20:11.267648+0000","last_fullsized":"2024-08-29T15:20:11.267648+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":137,"reported_epoch":240,"state":"active+clean","last_fresh":"2024-08-29T15:20:13.260820+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:20:13.260820+0000","last_peered":"2024-08-29T15:20:13.260820+0000","last_clean":"2024-08-29T15:20:13.260820+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:20:13.260820+0000","last_undegraded":"2024-08-29T15:20:13.260820+0000","last_fullsized":"2024-08-29T15:20:13.260820+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":371,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.461152+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:20:20.461152+0000","last_peered":"2024-08-29T15:20:20.461152+0000","last_clean":"2024-08-29T15:20:20.461152+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:20:20.461152+0000","last_undegraded":"2024-08-29T15:20:20.461152+0000","last_fullsized":"2024-08-29T15:20:20.461152+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":372,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.462764+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:20:20.462764+0000","last_peered":"2024-08-29T15:20:20.462764+0000","last_clean":"2024-08-29T15:20:20.462764+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:20:20.462764+0000","last_undegraded":"2024-08-29T15:20:20.462764+0000","last_fullsized":"2024-08-29T15:20:20.462764+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":239,"state":"active+clean","last_fresh":"2024-08-29T15:20:11.267281+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:20:11.267281+0000","last_peered":"2024-08-29T15:20:11.267281+0000","last_clean":"2024-08-29T15:20:11.267281+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:20:11.267281+0000","last_undegraded":"2024-08-29T15:20:11.267281+0000","last_fullsized":"2024-08-29T15:20:11.267281+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":380,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290775+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:20:20.290775+0000","last_peered":"2024-08-29T15:20:20.290775+0000","last_clean":"2024-08-29T15:20:20.290775+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:20:20.290775+0000","last_undegraded":"2024-08-29T15:20:20.290775+0000","last_fullsized":"2024-08-29T15:20:20.290775+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6493044760,"num_objects":99077,"num_object_clones":0,"num_object_copies":297231,"num_objects_missing_on_primary":7163,"num_objects_missing":7163,"num_objects_degraded":30073,"num_objects_misplaced":18496,"num_objects_unfound":0,"num_objects_dirty":99077,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":151773,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14185,"num_bytes_recovered":809631744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19342503936,"data_stored":19342491720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119350,"ondisk_log_size":119350,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234308,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3480600,"kb_used_data":3188624,"kb_used_omap":1,"kb_used_meta":291966,"kb_avail":90264552,"statfs":{"total":95995035648,"available":92430901248,"internally_reserved":0,"allocated":3265150976,"data_stored":3263818875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":298973643},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495941,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":506084,"kb_used_data":340444,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":93239068,"statfs":{"total":95995035648,"available":95476805632,"internally_reserved":0,"allocated":348614656,"data_stored":347278707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":169605575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426526,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2029276,"kb_used_data":1978256,"kb_used_omap":8,"kb_used_meta":50999,"kb_avail":91715876,"statfs":{"total":95995035648,"available":93917057024,"internally_reserved":0,"allocated":2025734144,"data_stored":2024405661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":52223882},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019218,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2264568,"kb_used_data":1980784,"kb_used_omap":1,"kb_used_meta":283774,"kb_avail":91480584,"statfs":{"total":95995035648,"available":93676118016,"internally_reserved":0,"allocated":2028322816,"data_stored":2027017764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290585030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435403,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4232984,"kb_used_data":3962572,"kb_used_omap":4,"kb_used_meta":270395,"kb_avail":89512168,"statfs":{"total":95995035648,"available":91660460032,"internally_reserved":0,"allocated":4057673728,"data_stored":4056349185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":276884745},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903255,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3554588,"kb_used_data":3472848,"kb_used_omap":10,"kb_used_meta":81717,"kb_avail":90190564,"statfs":{"total":95995035648,"available":92355137536,"internally_reserved":0,"allocated":3556196352,"data_stored":3554871745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":83678736},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762456,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2666840,"kb_used_data":2414412,"kb_used_omap":7,"kb_used_meta":252408,"kb_avail":91078312,"statfs":{"total":95995035648,"available":93264191488,"internally_reserved":0,"allocated":2472357888,"data_stored":2471032105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":258466146},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925983,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1908700,"kb_used_data":1602132,"kb_used_omap":7,"kb_used_meta":306552,"kb_avail":91836452,"statfs":{"total":95995035648,"available":94040526848,"internally_reserved":0,"allocated":1640583168,"data_stored":1639250164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":313909391},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1632571392,"data_stored":1632567320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2464878592,"data_stored":2464874520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3552051200,"data_stored":3552051200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4050190336,"data_stored":4050190336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2022572032,"data_stored":2022572032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2021330944,"data_stored":2021326872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":344653824,"data_stored":344653824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3254255616,"data_stored":3254255616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:20:26.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:20:26.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:26.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:26.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:26.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:26.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:26.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:26.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:26 smithi027 ceph-mon[93921]: pgmap v722: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 30102/297315 objects degraded (10.125%); 18496/297315 objects misplaced (6.221%); 346 KiB/s, 5 objects/s recovering 2024-08-29T15:20:26.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:26 smithi027 ceph-mon[98970]: pgmap v722: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 30102/297315 objects degraded (10.125%); 18496/297315 objects misplaced (6.221%); 346 KiB/s, 5 objects/s recovering 2024-08-29T15:20:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:26 smithi171 ceph-mon[90031]: pgmap v722: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 30102/297315 objects degraded (10.125%); 18496/297315 objects misplaced (6.221%); 346 KiB/s, 5 objects/s recovering 2024-08-29T15:20:26.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:26.638 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:26.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:26.740 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:26.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:26.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:26.830 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:23209 5.4 5:24099454:::benchmark_data_smithi027_135490_object23208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:10:40.957442+0000", 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 585.87031334699998, 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13668750399999999, 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.831 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 23209 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957442+0000", 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957442+0000", 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957441+0000", 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957483+0000", 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1879999999999999e-05 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957498+0000", 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5175e-05 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957506+0000", 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3560000000000004e-06 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957555+0000", 2024-08-29T15:20:26.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9413000000000003e-05 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957687+0000", 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000131707 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.957804+0000", 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011721499999999999 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.959256+0000", 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014519629999999999 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:40.959307+0000", 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1561999999999999e-05 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.094046+0000", 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13473817199999999 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.094089+0000", 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3547999999999997e-05 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:10:41.094130+0000", 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0639999999999997e-05 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.837 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:221741 5.4 5:285b0149:::benchmark_data_smithi027_135490_object87989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:45.178817+0000", 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 401.64893882699999, 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.123431307, 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221741 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178817+0000", 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178817+0000", 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178815+0000", 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178822+0000", 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4949999999999996e-06 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178837+0000", 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4715999999999999e-05 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179041+0000", 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00020451700000000001 2024-08-29T15:20:26.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179118+0000", 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7063000000000004e-05 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179232+0000", 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000113742 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179852+0000", 2024-08-29T15:20:26.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00061991200000000002 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.187825+0000", 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079734139999999998 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.190682+0000", 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028564969999999999 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.302155+0000", 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.111472904 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.302207+0000", 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2240999999999997e-05 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.302248+0000", 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1214000000000002e-05 2024-08-29T15:20:26.843 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124404 6.3 6:cbf2ce75:::benchmark_data_smithi027_169445_object124403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 208.79848874000001, 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3120749140000001, 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.844 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124404 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029265+0000", 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029301+0000", 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5939000000000003e-05 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029311+0000", 2024-08-29T15:20:26.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5869999999999999e-06 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029318+0000", 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0829999999999998e-06 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.036882+0000", 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075643280000000004 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.036891+0000", 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3100000000000001e-06 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029775+0000", 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99288402499999995 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029786+0000", 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1311000000000001e-05 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.313838+0000", 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2840517499999999 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.313991+0000", 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015349299999999999 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.314109+0000", 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011786300000000001 2024-08-29T15:20:26.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.320971+0000", 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0068620870000000002 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325526+0000", 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045548979999999999 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337613+0000", 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012086573999999999 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337664+0000", 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1085999999999998e-05 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.341342+0000", 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036781280000000001 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.851 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124411 6.5 6:a9772c5a:::benchmark_data_smithi027_169445_object124410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 208.79049799399999, 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2782022199999998, 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.852 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124411 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037257+0000", 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037284+0000", 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6716e-05 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037290+0000", 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0850000000000002e-06 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037294+0000", 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4610000000000001e-06 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037613+0000", 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031895900000000001 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037620+0000", 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7800000000000003e-06 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031231+0000", 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99361049700000004 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031239+0000", 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.473e-06 2024-08-29T15:20:26.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.294818+0000", 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2635784000000001 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.294989+0000", 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000171375 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:20:26.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.295176+0000", 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000187002 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303052+0000", 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078760210000000004 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307520+0000", 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044681469999999996 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.312052+0000", 2024-08-29T15:20:26.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045315149999999998 2024-08-29T15:20:26.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.312092+0000", 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0621000000000003e-05 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315460+0000", 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033675900000000002 2024-08-29T15:20:26.860 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124424 6.5 6:a6739d5e:::benchmark_data_smithi027_169445_object124423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 208.76942225900001, 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.268536273, 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.861 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124424 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058331+0000", 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058377+0000", 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6134000000000002e-05 2024-08-29T15:20:26.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058391+0000", 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3995000000000001e-05 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058398+0000", 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2060000000000001e-06 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058448+0000", 2024-08-29T15:20:26.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9456999999999998e-05 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058460+0000", 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1622e-05 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031289+0000", 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97282904800000003 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031295+0000", 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8249999999999999e-06 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303104+0000", 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.271809059 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303275+0000", 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017008999999999999 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303459+0000", 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018486800000000001 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315479+0000", 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012019724000000001 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315517+0000", 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7370999999999998e-05 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.322771+0000", 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072540670000000003 2024-08-29T15:20:26.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.322827+0000", 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6232e-05 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326870+0000", 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040427620000000001 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124428 6.3 6:cbc20026:::benchmark_data_smithi027_169445_object124427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 208.76726557699999, 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3095511559999999, 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124428 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060489+0000", 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:26.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060581+0000", 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1643000000000003e-05 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060594+0000", 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3222e-05 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060601+0000", 2024-08-29T15:20:26.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8449999999999997e-06 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060631+0000", 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0372999999999999e-05 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060637+0000", 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4740000000000002e-06 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029808+0000", 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96917036400000001 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029815+0000", 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8190000000000003e-06 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321001+0000", 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.29118635 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321186+0000", 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018538300000000001 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321355+0000", 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000168238 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337515+0000", 2024-08-29T15:20:26.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016160774999999999 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337526+0000", 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1053000000000001e-05 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.366213+0000", 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028686204 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.366281+0000", 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8071999999999995e-05 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370041+0000", 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037604409999999998 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.877 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124433 6.3 6:ca9f8931:::benchmark_data_smithi027_169445_object124432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 208.760744668, 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3129230789999999, 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124433 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067010+0000", 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067116+0000", 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106204 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067125+0000", 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.285e-06 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067130+0000", 2024-08-29T15:20:26.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1660000000000002e-06 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067165+0000", 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5108000000000003e-05 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067172+0000", 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0650000000000001e-06 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029836+0000", 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96266382100000003 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029843+0000", 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3280000000000002e-06 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325550+0000", 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2957075179999999 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325689+0000", 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013860200000000001 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325852+0000", 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016336399999999999 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337576+0000", 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011724399 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347548+0000", 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099717829999999997 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370096+0000", 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0225477 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370146+0000", 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0419e-05 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.379934+0000", 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0097876270000000001 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124444 6.5 6:af81ef90:::benchmark_data_smithi027_169445_object124443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 208.748734214, 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.251856976, 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.885 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124444 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079020+0000", 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079087+0000", 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6513999999999994e-05 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079099+0000", 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2391999999999999e-05 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079104+0000", 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3589999999999999e-06 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079138+0000", 2024-08-29T15:20:26.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4041000000000002e-05 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079145+0000", 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9870000000000002e-06 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031398+0000", 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95225246100000005 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031406+0000", 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7789999999999996e-06 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307552+0000", 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.276146349 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307695+0000", 2024-08-29T15:20:26.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014288899999999999 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307892+0000", 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019679800000000001 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315574+0000", 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0076828130000000001 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315619+0000", 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5049000000000002e-05 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326936+0000", 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011316794 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326989+0000", 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2293999999999998e-05 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.892 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330878+0000", 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038897950000000001 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124462 6.3 6:cf3b5247:::benchmark_data_smithi027_169445_object124461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 206.662443075, 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.21866085199999999, 2024-08-29T15:20:26.893 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124462 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.894 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165309+0000", 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165450+0000", 2024-08-29T15:20:26.895 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014043500000000001 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165476+0000", 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5859000000000001e-05 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165489+0000", 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2765999999999999e-05 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.896 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165566+0000", 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7019000000000002e-05 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165580+0000", 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4005999999999999e-05 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330157+0000", 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16457785 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.897 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330324+0000", 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016682599999999999 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330523+0000", 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019885899999999999 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.341411+0000", 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010887468000000001 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.898 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347578+0000", 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0061672610000000003 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.379991+0000", 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032413320000000002 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.380174+0000", 2024-08-29T15:20:26.899 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018272399999999999 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.383973+0000", 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037995310000000001 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124473 6.3 6:c5e714a2:::benchmark_data_smithi027_169445_object124472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 206.63221834699999, 2024-08-29T15:20:26.900 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19243169900000001, 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124473 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.901 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195531+0000", 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999933 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195597+0000", 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6513000000000005e-05 2024-08-29T15:20:26.902 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195613+0000", 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5865000000000001e-05 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195620+0000", 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2259999999999999e-06 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.903 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195679+0000", 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8695e-05 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195690+0000", 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1473e-05 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342341+0000", 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.146650369 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.904 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342504+0000", 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000162812 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342677+0000", 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000172874 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.905 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347795+0000", 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051186010000000004 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347888+0000", 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2949999999999996e-05 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.384025+0000", 2024-08-29T15:20:26.906 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036137057 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.384081+0000", 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5881000000000003e-05 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.387969+0000", 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038879259999999999 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.907 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPushReply(6.5 191/190 [PushReplyOp(6:a4839e33:::benchmark_data_smithi027_169445_object147:head)])", 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 205.847972986, 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13693580699999999, 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:20:26.908 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979781+0000", 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.909 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979790+0000", 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8929999999999994e-06 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979811+0000", 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0508000000000001e-05 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979820+0000", 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2040000000000003e-06 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.910 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:01.107955+0000", 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12813488200000001 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:01.116718+0000", 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087634329999999993 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.911 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137594 6.3 6:c9cad5ba:::benchmark_data_smithi027_169445_object12167:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 189.12287727200001, 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13802763300000001, 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137594 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.912 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.913 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704881+0000", 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.627e-06 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704886+0000", 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1390000000000003e-06 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704890+0000", 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1139999999999999e-06 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711314+0000", 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0064242639999999998 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711480+0000", 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016600600000000001 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711632+0000", 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015182300000000001 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717197+0000", 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055650309999999998 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732501+0000", 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015303654 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.838889+0000", 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10638834899999999 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.838943+0000", 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4279e-05 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.842906+0000", 2024-08-29T15:20:26.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039627280000000004 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137598 6.3 6:cf32642d:::benchmark_data_smithi027_169445_object12171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 189.12048889900001, 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.139080008, 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.918 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137598 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707265+0000", 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707271+0000", 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.48e-06 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707279+0000", 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8459999999999997e-06 2024-08-29T15:20:26.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707285+0000", 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7899999999999996e-06 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717221+0000", 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099359559999999993 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717372+0000", 2024-08-29T15:20:26.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015091400000000001 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717520+0000", 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014827699999999999 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.729379+0000", 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011858485 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732510+0000", 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003130678 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.842959+0000", 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.110449724 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.843014+0000", 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4672000000000001e-05 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846347+0000", 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033327650000000001 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137609 6.3 6:c98a9743:::benchmark_data_smithi027_169445_object12182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 189.10655573599999, 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12867278300000001, 2024-08-29T15:20:26.924 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137609 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721199+0000", 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721202+0000", 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0939999999999999e-06 2024-08-29T15:20:26.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721209+0000", 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0800000000000002e-06 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721213+0000", 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5600000000000004e-06 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721607+0000", 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00039427400000000001 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721766+0000", 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000158667 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721903+0000", 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013681500000000001 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.729409+0000", 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075061219999999996 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732515+0000", 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031063250000000001 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846478+0000", 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.113962623 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846534+0000", 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6375000000000001e-05 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.849873+0000", 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033383330000000002 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.930 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:138635 6.5 6:aff223a2:::benchmark_data_smithi027_169445_object13208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:20:26.931 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:20:26.931 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.845718046, 2024-08-29T15:20:26.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14247343300000001, 2024-08-29T15:20:26.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:26.932 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.932 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.932 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.932 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.932 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 138635 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982036+0000", 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982044+0000", 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0830000000000004e-06 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982598+0000", 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00055435700000000001 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982609+0000", 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0635999999999999e-05 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.003985+0000", 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021376350999999998 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.004101+0000", 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011584 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083026+0000", 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078925680999999998 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083103+0000", 2024-08-29T15:20:26.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6506000000000002e-05 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083205+0000", 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000101593 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.091336+0000", 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081309610000000008 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.095604+0000", 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042681050000000003 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.120534+0000", 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024930548 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.120652+0000", 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011811099999999999 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.124511+0000", 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038586829999999999 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:139269 6.5 6:a8585b14:::benchmark_data_smithi027_169445_object13842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.03013912399999, 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.174785774, 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.939 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 139269 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797615+0000", 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797621+0000", 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9689999999999999e-06 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797637+0000", 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6127999999999999e-05 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797645+0000", 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2130000000000008e-06 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.821075+0000", 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023429189 2024-08-29T15:20:26.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.821189+0000", 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011487799999999999 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.940614+0000", 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.119424749 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.940879+0000", 2024-08-29T15:20:26.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00026527700000000001 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.941261+0000", 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00038113399999999999 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968451+0000", 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027190783999999999 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968461+0000", 2024-08-29T15:20:26.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1959999999999997e-06 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968492+0000", 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1841999999999998e-05 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968542+0000", 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9348000000000003e-05 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.972402+0000", 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038604500000000001 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:139443 6.5 6:a02b9a8b:::benchmark_data_smithi027_169445_object14016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 186.79058224400001, 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15171564100000001, 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:20:26.946 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 139443 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037176+0000", 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.078e-06 2024-08-29T15:20:26.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037182+0000", 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8540000000000002e-06 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037187+0000", 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8110000000000002e-06 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.061998+0000", 2024-08-29T15:20:26.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024811600999999999 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.062127+0000", 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012901000000000001 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.100824+0000", 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038696362999999998 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.100935+0000", 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011161499999999999 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.104035+0000", 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030998649999999998 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.130597+0000", 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026562129 2024-08-29T15:20:26.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.152790+0000", 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022192587999999999 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.184767+0000", 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031977515999999997 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:20:26.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.184820+0000", 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3084999999999997e-05 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.188889+0000", 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040686769999999997 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:142989 6.3 6.e160cdf3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:20:26.953 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 136.97268301099999, 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98678860400000001, 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 142989 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855071+0000", 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:26.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855079+0000", 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5390000000000007e-06 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855093+0000", 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3835000000000001e-05 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855102+0000", 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9740000000000008e-06 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855146+0000", 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4082000000000002e-05 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855154+0000", 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.622e-06 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.841849+0000", 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98669542700000001 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.841861+0000", 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1803e-05 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.958 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:142999 6.5 6.8bdf2d15 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 136.668022022, 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.683494037, 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 142999 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159731+0000", 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159743+0000", 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1389000000000001e-05 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159759+0000", 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6514000000000001e-05 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159769+0000", 2024-08-29T15:20:26.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4409999999999999e-06 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159822+0000", 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2806999999999998e-05 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159834+0000", 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2047e-05 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843215+0000", 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.68338162599999996 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843228+0000", 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2296999999999999e-05 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:143011 6.5 6.1a21b555 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 136.294692935, 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.310191416, 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 143011 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.964 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:26.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533061+0000", 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533069+0000", 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5210000000000002e-06 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533083+0000", 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4115e-05 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533094+0000", 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0427e-05 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533134+0000", 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0201999999999999e-05 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533144+0000", 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0164e-05 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843246+0000", 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.31010232399999998 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843254+0000", 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7239999999999992e-06 2024-08-29T15:20:26.968 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.969 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.969 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.969 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:26.969 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:26.969 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:20:26.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:26.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:26.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:27.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:27.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:27.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:27.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:27.239 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:27.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:27.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:27.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:27.436 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:27.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:27.477 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:27.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:27 smithi171 ceph-mon[90031]: from='client.35394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/297315 objects degraded (10.125%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:27.561 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:27 smithi027 ceph-mon[98970]: from='client.35394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:27.561 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/297315 objects degraded (10.125%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:27.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:27 smithi027 ceph-mon[93921]: from='client.35394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:27.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/297315 objects degraded (10.125%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:27.591 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:27.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:27.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:27.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:27.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:27.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:27.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:27.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:27.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:27.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:20:27.971 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:27.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:27.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:28.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:28.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:28.148 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:28.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:28.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:28.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:28.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:28.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:28.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:28.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:28 smithi027 ceph-mon[98970]: pgmap v723: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 30102/296808 objects degraded (10.142%); 18468/296808 objects misplaced (6.222%); 346 KiB/s, 5 objects/s recovering 2024-08-29T15:20:28.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:28 smithi027 ceph-mon[93921]: pgmap v723: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 30102/296808 objects degraded (10.142%); 18468/296808 objects misplaced (6.222%); 346 KiB/s, 5 objects/s recovering 2024-08-29T15:20:28.542 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:28 smithi171 ceph-mon[90031]: pgmap v723: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 101 op/s; 30102/296808 objects degraded (10.142%); 18468/296808 objects misplaced (6.222%); 346 KiB/s, 5 objects/s recovering 2024-08-29T15:20:28.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:28.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:28.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:28.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:28.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:28.844 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:28.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:28.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:28.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:28.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:28.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:28.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:29.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:29.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:29.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:29.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:29.178 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:20:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:20:29] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:20:29.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:29.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:29.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:29.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:29.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:29.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:29.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:29.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:29.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:29.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:29.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:29.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:29.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:29.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:29.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:29.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:29.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:29.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:29.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:29.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:29.794 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:29 smithi027 ceph-mon[93921]: pgmap v724: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 30102/296250 objects degraded (10.161%); 18409/296250 objects misplaced (6.214%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:29.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:29.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:29.893 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:29 smithi027 ceph-mon[98970]: pgmap v724: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 30102/296250 objects degraded (10.161%); 18409/296250 objects misplaced (6.214%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:29.893 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:29.893 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:29.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:29.903 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:29 smithi171 ceph-mon[90031]: pgmap v724: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 30102/296250 objects degraded (10.161%); 18409/296250 objects misplaced (6.214%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:29.904 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:29.904 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:29.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:29.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:29.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:152657 6.7 6:e3646cc0:::benchmark_data_smithi027_169445_object27230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:29.918113+0000", 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.001343863, 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013646109999999999, 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:20:29.921 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 152657 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918113+0000", 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918113+0000", 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:20:29.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918111+0000", 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918122+0000", 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1634000000000001e-05 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918139+0000", 2024-08-29T15:20:29.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6857000000000001e-05 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918149+0000", 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4199999999999996e-06 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918201+0000", 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2398000000000002e-05 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918356+0000", 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000155317 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:29.918496+0000", 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000139476 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 1 2024-08-29T15:20:29.925 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:20:30.020 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:30.046 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:30.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:30.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:30.253 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:20:30.253 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:20:30.272 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:20:30.272 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:20:30.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:30.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:20:30.289 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:30.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:30.305 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:30.305 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:30.326 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:30.326 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:30.340 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:30.341 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:30.357 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:30.357 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:30.373 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:30.373 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:30.401 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:30.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:30.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:30.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:30.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:30.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:30.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:30.712 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:20:30.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:30.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:30.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:30.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:20:30.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:30.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:30.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:30.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2024084270' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2725571885' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:31.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:31.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2024084270' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2725571885' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2024084270' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2725571885' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:31.090 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:31.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:31.185 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:31.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:31.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:31.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:31.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:31.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:31.434 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:31.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:31.536 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:31.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:31.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:31.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:31.685 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:31.787 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:31.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:31.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:31.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:31.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:31.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:31.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:32.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:31 smithi171 ceph-mon[90031]: pgmap v725: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 93 op/s; 30102/295833 objects degraded (10.175%); 18409/295833 objects misplaced (6.223%); 418 KiB/s, 6 objects/s recovering 2024-08-29T15:20:32.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:31 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:32.003 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2725571885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:32.004 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:31 smithi171 ceph-mon[90031]: osdmap e242: 8 total, 8 up, 8 in 2024-08-29T15:20:32.020 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[98970]: pgmap v725: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 93 op/s; 30102/295833 objects degraded (10.175%); 18409/295833 objects misplaced (6.223%); 418 KiB/s, 6 objects/s recovering 2024-08-29T15:20:32.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:32.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2725571885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:32.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[98970]: osdmap e242: 8 total, 8 up, 8 in 2024-08-29T15:20:32.021 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[93921]: pgmap v725: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 93 op/s; 30102/295833 objects degraded (10.175%); 18409/295833 objects misplaced (6.223%); 418 KiB/s, 6 objects/s recovering 2024-08-29T15:20:32.021 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:32.021 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2725571885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:32.021 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:31 smithi027 ceph-mon[93921]: osdmap e242: 8 total, 8 up, 8 in 2024-08-29T15:20:32.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:32.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:32.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:32.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:32.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:32.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:32.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:32.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:32.435 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:32.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:32.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:32.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:32.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:32.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:32.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:32.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:32.787 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:32.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/295833 objects degraded (10.175%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/295833 objects degraded (10.175%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:32.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:32.917 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/295833 objects degraded (10.175%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:32.923 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:32.923 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:32.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:32.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:32.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:33.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:33.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:33.068 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:20:33.068 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/294474 objects degraded (10.222%), 17 pgs degraded, 7 pgs undersized","count":24},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1014,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":242,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":98158,"data_bytes":6435772413,"bytes_used":20948762624,"bytes_avail":747011522560,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":294474,"degraded_ratio":0.10222294667780517,"misplaced_objects":18322,"misplaced_total":294474,"misplaced_ratio":0.06221941495683829,"recovering_objects_per_sec":3,"recovering_bytes_per_sec":222762,"recovering_keys_per_sec":0,"num_objects_recovered":34,"num_bytes_recovered":2228224,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":99},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:20:33.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:20:33.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:33.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:33.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:33.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:33.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:33.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:33.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:33.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:33.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:33.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:33.562 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:33.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:33.602 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:20:33.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:33.685 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:33.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:33.810 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:33.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:33.816 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:20:33.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:33.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:33.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:33.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:33.938 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:33 smithi171 ceph-mon[90031]: pgmap v727: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 30102/294474 objects degraded (10.222%); 18322/294474 objects misplaced (6.222%); 218 KiB/s, 3 objects/s recovering 2024-08-29T15:20:33.939 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1572376497' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:20:33.979 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:20:33.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:20:33.989 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:34.003 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:33 smithi027 ceph-mon[98970]: pgmap v727: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 30102/294474 objects degraded (10.222%); 18322/294474 objects misplaced (6.222%); 218 KiB/s, 3 objects/s recovering 2024-08-29T15:20:34.003 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1572376497' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:20:34.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:33 smithi027 ceph-mon[93921]: pgmap v727: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 30102/294474 objects degraded (10.222%); 18322/294474 objects misplaced (6.222%); 218 KiB/s, 3 objects/s recovering 2024-08-29T15:20:34.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1572376497' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:20:34.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:34.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:34.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:34.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:34.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:34.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:34.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:34.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:34.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:34.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:34.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:34.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:34.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:34.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:20:34.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:34.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:34.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:34.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:34.730 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:34.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:34.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:34.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:34.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:34.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:35.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:35.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:35.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:35.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:35.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:35.297 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:35.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:35.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:35.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:35.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:35.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:35.568 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:35.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:35.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:35.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:35.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:35.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:35.819 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:35.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:35.921 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:35.937 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:35 smithi171 ceph-mon[90031]: pgmap v728: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 30102/294198 objects degraded (10.232%); 18290/294198 objects misplaced (6.217%); 422 KiB/s, 6 objects/s recovering 2024-08-29T15:20:35.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:35.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:35.952 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:20:35.952 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:20:36.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:36.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:35 smithi027 ceph-mon[98970]: pgmap v728: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 30102/294198 objects degraded (10.232%); 18290/294198 objects misplaced (6.217%); 422 KiB/s, 6 objects/s recovering 2024-08-29T15:20:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:35 smithi027 ceph-mon[93921]: pgmap v728: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 103 op/s; 30102/294198 objects degraded (10.232%); 18290/294198 objects misplaced (6.217%); 422 KiB/s, 6 objects/s recovering 2024-08-29T15:20:36.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:36.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:36.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:36.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:36.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:36.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:36.321 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:36.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:36.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:36.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:36.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:36.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:36.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:20:36.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:36.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.824 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:36.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:36.890 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:36.925 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:36.948 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:36.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:36.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:37.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:37.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:37.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:37.131 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:20:37.133 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:20:37.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:37.196 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:37.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:37.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:37.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:37.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:37.421 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:37.444 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:37.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:37.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:37.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:37.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:37.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:37.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:37.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:37.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:37.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:37.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:37.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:37.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:37 smithi171 ceph-mon[90031]: pgmap v729: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30102/293451 objects degraded (10.258%); 18254/293451 objects misplaced (6.220%); 422 KiB/s, 6 objects/s recovering 2024-08-29T15:20:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/293451 objects degraded (10.258%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:37 smithi171 ceph-mon[90031]: from='client.45335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:38.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:38.068 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:37 smithi027 ceph-mon[98970]: pgmap v729: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30102/293451 objects degraded (10.258%); 18254/293451 objects misplaced (6.220%); 422 KiB/s, 6 objects/s recovering 2024-08-29T15:20:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/293451 objects degraded (10.258%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:37 smithi027 ceph-mon[98970]: from='client.45335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:38.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:37 smithi027 ceph-mon[93921]: pgmap v729: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30102/293451 objects degraded (10.258%); 18254/293451 objects misplaced (6.220%); 422 KiB/s, 6 objects/s recovering 2024-08-29T15:20:38.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/293451 objects degraded (10.258%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:38.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:37 smithi027 ceph-mon[93921]: from='client.45335 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:38.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:38.196 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:38.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:38.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:38.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:38.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:38.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:38.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:38.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:38.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:38.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:38.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:38.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:38.816 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:38.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:38.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:38.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:38.952 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3286562487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:38.952 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3286562487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:39.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3286562487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:39.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:39.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:39.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:39.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:39.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:39.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:39.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:39.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:20:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:20:39] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:20:39.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:39.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:39.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:39.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:39.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:39.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:39.579 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:20:39.579 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:20:39.579 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:20:39.580 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:20:39.580 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:20:39.580 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:39.580 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:39.580 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:39.580 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:39.580 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:39.581 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:39.581 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:39.581 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:39.581 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:39.581 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:39.581 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:39.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:39.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:39.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:39.752 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:20:39.773 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:39.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:39.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:39.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:39.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:39.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:39 smithi171 ceph-mon[90031]: pgmap v730: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 30102/292773 objects degraded (10.282%); 18189/292773 objects misplaced (6.213%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:39 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/273518391' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:39 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:40.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:40.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:39 smithi027 ceph-mon[98970]: pgmap v730: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 30102/292773 objects degraded (10.282%); 18189/292773 objects misplaced (6.213%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:39 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/273518391' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:39 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:39 smithi027 ceph-mon[93921]: pgmap v730: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 30102/292773 objects degraded (10.282%); 18189/292773 objects misplaced (6.213%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:40.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:39 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/273518391' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:40.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:39 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:20:40.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:40.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:40.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:40.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:40.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:40.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:40.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:40.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.438 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:40.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:40.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:40.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:40.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:40.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:40.685 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:40.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:40.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:40.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:40.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:40.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:40.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:40 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:40 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:20:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:40 smithi171 ceph-mon[90031]: osdmap e243: 8 total, 8 up, 8 in 2024-08-29T15:20:41.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:41.057 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:41.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:40 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:41.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:40 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:20:41.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:40 smithi027 ceph-mon[93921]: osdmap e243: 8 total, 8 up, 8 in 2024-08-29T15:20:41.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:40 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:20:41.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:40 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:20:41.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:40 smithi027 ceph-mon[98970]: osdmap e243: 8 total, 8 up, 8 in 2024-08-29T15:20:41.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:41.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:41.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:41.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:41.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:41.432 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:41.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:41.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:41.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:41.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:41.680 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:41.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:41.713 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":729,"stamp":"2024-08-29T15:20:36.541540+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6413424637,"num_objects":97817,"num_object_clones":0,"num_object_copies":293451,"num_objects_missing_on_primary":7105,"num_objects_missing":7105,"num_objects_degraded":30102,"num_objects_misplaced":18254,"num_objects_unfound":0,"num_objects_dirty":97817,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":154184,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14360,"num_bytes_recovered":819924173,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121120,"ondisk_log_size":121120,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20408148,"kb_used_data":18696520,"kb_used_omap":43,"kb_used_meta":1711508,"kb_avail":729553068,"statfs":{"total":767960285184,"available":747062341632,"internally_reserved":0,"allocated":19145236480,"data_stored":19134550478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1752584684},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,1,0,0,2,1,3,7,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-73334784,"num_objects":-1119,"num_object_clones":0,"num_object_copies":-3357,"num_objects_missing_on_primary":-66,"num_objects_missing":-66,"num_objects_degraded":0,"num_objects_misplaced":-214,"num_objects_unfound":0,"num_objects_dirty":-1119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1119,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":4325376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002677"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":367,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715331+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:20:30.715331+0000","last_peered":"2024-08-29T15:20:30.715331+0000","last_clean":"2024-08-29T15:20:30.715331+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:20:30.715331+0000","last_undegraded":"2024-08-29T15:20:30.715331+0000","last_fullsized":"2024-08-29T15:20:30.715331+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":18,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:20.290873+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:20:20.290873+0000","last_peered":"2024-08-29T15:20:20.290873+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:20:20.290873+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:20:20.290873+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":108,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290732+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:20:20.290732+0000","last_peered":"2024-08-29T15:20:20.290732+0000","last_clean":"2024-08-29T15:20:20.290732+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:20:20.290732+0000","last_undegraded":"2024-08-29T15:20:20.290732+0000","last_fullsized":"2024-08-29T15:20:20.290732+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":333,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732323+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:20:30.732323+0000","last_peered":"2024-08-29T15:20:30.732323+0000","last_clean":"2024-08-29T15:20:30.732323+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:20:30.732323+0000","last_undegraded":"2024-08-29T15:20:30.732323+0000","last_fullsized":"2024-08-29T15:20:30.732323+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":333,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732672+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:20:30.732672+0000","last_peered":"2024-08-29T15:20:30.732672+0000","last_clean":"2024-08-29T15:20:30.732672+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:20:30.732672+0000","last_undegraded":"2024-08-29T15:20:30.732672+0000","last_fullsized":"2024-08-29T15:20:30.732672+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290585+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:20:20.290585+0000","last_peered":"2024-08-29T15:20:20.290585+0000","last_clean":"2024-08-29T15:20:20.290585+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:20:20.290585+0000","last_undegraded":"2024-08-29T15:20:20.290585+0000","last_fullsized":"2024-08-29T15:20:20.290585+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":145,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.719457+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:20:30.719457+0000","last_peered":"2024-08-29T15:20:30.719457+0000","last_clean":"2024-08-29T15:20:30.719457+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:20:30.719457+0000","last_undegraded":"2024-08-29T15:20:30.719457+0000","last_fullsized":"2024-08-29T15:20:30.719457+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":109,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.458936+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:20:20.458936+0000","last_peered":"2024-08-29T15:20:20.458936+0000","last_clean":"2024-08-29T15:20:20.458936+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:20:20.458936+0000","last_undegraded":"2024-08-29T15:20:20.458936+0000","last_fullsized":"2024-08-29T15:20:20.458936+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"242'9748","reported_seq":17767,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:35.810103+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:20:35.810103+0000","last_peered":"2024-08-29T15:20:35.810103+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:20:35.810103+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9748,"log_dups_size":0,"ondisk_log_size":9748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403963904,"num_objects":6164,"num_object_clones":0,"num_object_copies":18492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9748,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":3239}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":17,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:20.291163+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:20:20.291163+0000","last_peered":"2024-08-29T15:20:20.291163+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:20:20.291163+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:20:20.291163+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290262+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:20:20.290262+0000","last_peered":"2024-08-29T15:20:20.290262+0000","last_clean":"2024-08-29T15:20:20.290262+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:20:20.290262+0000","last_undegraded":"2024-08-29T15:20:20.290262+0000","last_fullsized":"2024-08-29T15:20:20.290262+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"242'9462","reported_seq":9623,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:31.784914+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:20:31.784914+0000","last_peered":"2024-08-29T15:20:31.784914+0000","last_clean":"2024-08-29T15:20:31.784914+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:20:31.784914+0000","last_undegraded":"2024-08-29T15:20:31.784914+0000","last_fullsized":"2024-08-29T15:20:31.784914+0000","mapping_epoch":168,"log_start":"178'2728","ondisk_log_start":"178'2728","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398983168,"num_objects":6088,"num_object_clones":0,"num_object_copies":18264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9462,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:20.291258+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:20:20.291258+0000","last_peered":"2024-08-29T15:20:20.291258+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:20:20.291258+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:20:20.291258+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290364+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:20:20.290364+0000","last_peered":"2024-08-29T15:20:20.290364+0000","last_clean":"2024-08-29T15:20:20.290364+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:20:20.290364+0000","last_undegraded":"2024-08-29T15:20:20.290364+0000","last_fullsized":"2024-08-29T15:20:20.290364+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"242'9546","reported_seq":9677,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:31.706841+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:20:31.706841+0000","last_peered":"2024-08-29T15:20:31.706841+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:20:31.706841+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'2874","ondisk_log_start":"178'2874","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404094976,"num_objects":6166,"num_object_clones":0,"num_object_copies":18498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9546,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3689}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":149,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732680+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:20:30.732680+0000","last_peered":"2024-08-29T15:20:30.732680+0000","last_clean":"2024-08-29T15:20:30.732680+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:20:30.732680+0000","last_undegraded":"2024-08-29T15:20:30.732680+0000","last_fullsized":"2024-08-29T15:20:30.732680+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":64,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.459292+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:20:20.459292+0000","last_peered":"2024-08-29T15:20:20.459292+0000","last_clean":"2024-08-29T15:20:20.459292+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:20:20.459292+0000","last_undegraded":"2024-08-29T15:20:20.459292+0000","last_fullsized":"2024-08-29T15:20:20.459292+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"242'9682","reported_seq":15063,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:33.572351+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:20:33.572351+0000","last_peered":"2024-08-29T15:20:33.572351+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:20:33.572351+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'1562","ondisk_log_start":"176'1562","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8120,"log_dups_size":0,"ondisk_log_size":8120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399507456,"num_objects":6096,"num_object_clones":0,"num_object_copies":18288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9682,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":6096}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":126,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290574+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:20:20.290574+0000","last_peered":"2024-08-29T15:20:20.290574+0000","last_clean":"2024-08-29T15:20:20.290574+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:20:20.290574+0000","last_undegraded":"2024-08-29T15:20:20.290574+0000","last_fullsized":"2024-08-29T15:20:20.290574+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":347,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732646+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:20:30.732646+0000","last_peered":"2024-08-29T15:20:30.732646+0000","last_clean":"2024-08-29T15:20:30.732646+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:20:30.732646+0000","last_undegraded":"2024-08-29T15:20:30.732646+0000","last_fullsized":"2024-08-29T15:20:30.732646+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"242'9658","reported_seq":9785,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:34.407843+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:20:34.407843+0000","last_peered":"2024-08-29T15:20:34.407843+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:20:34.407843+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:20:34.407843+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403439616,"num_objects":6156,"num_object_clones":0,"num_object_copies":18468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9658,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3679},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":342,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715046+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:20:30.715046+0000","last_peered":"2024-08-29T15:20:30.715046+0000","last_clean":"2024-08-29T15:20:30.715046+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:20:30.715046+0000","last_undegraded":"2024-08-29T15:20:30.715046+0000","last_fullsized":"2024-08-29T15:20:30.715046+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290359+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:20:20.290359+0000","last_peered":"2024-08-29T15:20:20.290359+0000","last_clean":"2024-08-29T15:20:20.290359+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:20:20.290359+0000","last_undegraded":"2024-08-29T15:20:20.290359+0000","last_fullsized":"2024-08-29T15:20:20.290359+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"242'9620","reported_seq":9797,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:31.801641+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:20:31.801641+0000","last_peered":"2024-08-29T15:20:31.801641+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:20:31.801641+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:20:31.801641+0000","mapping_epoch":234,"log_start":"178'2868","ondisk_log_start":"178'2868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407764992,"num_objects":6222,"num_object_clones":0,"num_object_copies":18666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9620,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3757},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":82,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.459219+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:20:20.459219+0000","last_peered":"2024-08-29T15:20:20.459219+0000","last_clean":"2024-08-29T15:20:20.459219+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:20:20.459219+0000","last_undegraded":"2024-08-29T15:20:20.459219+0000","last_fullsized":"2024-08-29T15:20:20.459219+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290881+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:20:20.290881+0000","last_peered":"2024-08-29T15:20:20.290881+0000","last_clean":"2024-08-29T15:20:20.290881+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:20:20.290881+0000","last_undegraded":"2024-08-29T15:20:20.290881+0000","last_fullsized":"2024-08-29T15:20:20.290881+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"242'9504","reported_seq":9866,"reported_epoch":242,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:20:31.780612+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:20:31.780612+0000","last_peered":"2024-08-29T15:20:31.780612+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:20:31.780612+0000","last_undegraded":"2024-08-29T15:20:31.780612+0000","last_fullsized":"2024-08-29T15:20:31.780612+0000","mapping_epoch":202,"log_start":"178'2767","ondisk_log_start":"178'2767","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6737,"log_dups_size":0,"ondisk_log_size":6737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403308544,"num_objects":6154,"num_object_clones":0,"num_object_copies":18462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6154,"num_objects_unfound":0,"num_objects_dirty":6154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9504,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":6154}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290437+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:20:20.290437+0000","last_peered":"2024-08-29T15:20:20.290437+0000","last_clean":"2024-08-29T15:20:20.290437+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:20:20.290437+0000","last_undegraded":"2024-08-29T15:20:20.290437+0000","last_fullsized":"2024-08-29T15:20:20.290437+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":225,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:24.770847+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:20:24.770847+0000","last_peered":"2024-08-29T15:20:24.770847+0000","last_clean":"2024-08-29T15:20:24.770847+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:20:24.770847+0000","last_undegraded":"2024-08-29T15:20:24.770847+0000","last_fullsized":"2024-08-29T15:20:24.770847+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"242'9613","reported_seq":9734,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:36.048466+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:20:36.048466+0000","last_peered":"2024-08-29T15:20:36.048466+0000","last_clean":"2024-08-29T15:20:36.048466+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:20:36.048466+0000","last_undegraded":"2024-08-29T15:20:36.048466+0000","last_fullsized":"2024-08-29T15:20:36.048466+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8335,"log_dups_size":0,"ondisk_log_size":8335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402784280,"num_objects":6147,"num_object_clones":0,"num_object_copies":18441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6147,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9613,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":345,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715235+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:20:30.715235+0000","last_peered":"2024-08-29T15:20:30.715235+0000","last_clean":"2024-08-29T15:20:30.715235+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:20:30.715235+0000","last_undegraded":"2024-08-29T15:20:30.715235+0000","last_fullsized":"2024-08-29T15:20:30.715235+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":337,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.719429+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:20:30.719429+0000","last_peered":"2024-08-29T15:20:30.719429+0000","last_clean":"2024-08-29T15:20:30.719429+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:20:30.719429+0000","last_undegraded":"2024-08-29T15:20:30.719429+0000","last_fullsized":"2024-08-29T15:20:30.719429+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"242'9312","reported_seq":9481,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:36.056813+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:20:36.056813+0000","last_peered":"2024-08-29T15:20:36.056813+0000","last_clean":"2024-08-29T15:20:36.056813+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:20:36.056813+0000","last_undegraded":"2024-08-29T15:20:36.056813+0000","last_fullsized":"2024-08-29T15:20:36.056813+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9312,"log_dups_size":0,"ondisk_log_size":9312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386400256,"num_objects":5896,"num_object_clones":0,"num_object_copies":17688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9312,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":436,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732459+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:20:30.732459+0000","last_peered":"2024-08-29T15:20:30.732459+0000","last_clean":"2024-08-29T15:20:30.732459+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:20:30.732459+0000","last_undegraded":"2024-08-29T15:20:30.732459+0000","last_fullsized":"2024-08-29T15:20:30.732459+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":710,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290743+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:20:20.290743+0000","last_peered":"2024-08-29T15:20:20.290743+0000","last_clean":"2024-08-29T15:20:20.290743+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:20:20.290743+0000","last_undegraded":"2024-08-29T15:20:20.290743+0000","last_fullsized":"2024-08-29T15:20:20.290743+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.291346+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:20:20.291346+0000","last_peered":"2024-08-29T15:20:20.291346+0000","last_clean":"2024-08-29T15:20:20.291346+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:20:20.291346+0000","last_undegraded":"2024-08-29T15:20:20.291346+0000","last_fullsized":"2024-08-29T15:20:20.291346+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.459494+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:20:20.459494+0000","last_peered":"2024-08-29T15:20:20.459494+0000","last_clean":"2024-08-29T15:20:20.459494+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:20:20.459494+0000","last_undegraded":"2024-08-29T15:20:20.459494+0000","last_fullsized":"2024-08-29T15:20:20.459494+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"242'9650","reported_seq":10009,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:35.060780+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:20:35.060780+0000","last_peered":"2024-08-29T15:20:35.060780+0000","last_clean":"2024-08-29T15:20:35.060780+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:20:35.060780+0000","last_undegraded":"2024-08-29T15:20:35.060780+0000","last_fullsized":"2024-08-29T15:20:35.060780+0000","mapping_epoch":201,"log_start":"177'2653","ondisk_log_start":"177'2653","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395313152,"num_objects":6032,"num_object_clones":0,"num_object_copies":18096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9650,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":810,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.717734+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:20:30.717734+0000","last_peered":"2024-08-29T15:20:30.717734+0000","last_clean":"2024-08-29T15:20:30.717734+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:20:30.717734+0000","last_undegraded":"2024-08-29T15:20:30.717734+0000","last_fullsized":"2024-08-29T15:20:30.717734+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":348,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732262+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:20:30.732262+0000","last_peered":"2024-08-29T15:20:30.732262+0000","last_clean":"2024-08-29T15:20:30.732262+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:20:30.732262+0000","last_undegraded":"2024-08-29T15:20:30.732262+0000","last_fullsized":"2024-08-29T15:20:30.732262+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":356,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732852+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:20:30.732852+0000","last_peered":"2024-08-29T15:20:30.732852+0000","last_clean":"2024-08-29T15:20:30.732852+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:20:30.732852+0000","last_undegraded":"2024-08-29T15:20:30.732852+0000","last_fullsized":"2024-08-29T15:20:30.732852+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"242'9504","reported_seq":15286,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:31.283087+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:20:31.283087+0000","last_peered":"2024-08-29T15:20:31.283087+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:20:31.283087+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'2830","ondisk_log_start":"178'2830","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405405696,"num_objects":6186,"num_object_clones":0,"num_object_copies":18558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9504,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1752},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":618,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.714742+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:20:30.714742+0000","last_peered":"2024-08-29T15:20:30.714742+0000","last_clean":"2024-08-29T15:20:30.714742+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:20:30.714742+0000","last_undegraded":"2024-08-29T15:20:30.714742+0000","last_fullsized":"2024-08-29T15:20:30.714742+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":71,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.458826+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:20:20.458826+0000","last_peered":"2024-08-29T15:20:20.458826+0000","last_clean":"2024-08-29T15:20:20.458826+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:20:20.458826+0000","last_undegraded":"2024-08-29T15:20:20.458826+0000","last_fullsized":"2024-08-29T15:20:20.458826+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":382,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.719316+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:20:30.719316+0000","last_peered":"2024-08-29T15:20:30.719316+0000","last_clean":"2024-08-29T15:20:30.719316+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:20:30.719316+0000","last_undegraded":"2024-08-29T15:20:30.719316+0000","last_fullsized":"2024-08-29T15:20:30.719316+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"242'9562","reported_seq":9733,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:31.726693+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:20:31.726693+0000","last_peered":"2024-08-29T15:20:31.726693+0000","last_clean":"2024-08-29T15:20:31.726693+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:20:31.726693+0000","last_undegraded":"2024-08-29T15:20:31.726693+0000","last_fullsized":"2024-08-29T15:20:31.726693+0000","mapping_epoch":234,"log_start":"178'2797","ondisk_log_start":"178'2797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403046400,"num_objects":6150,"num_object_clones":0,"num_object_copies":18450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9562,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":178,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290822+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:20:20.290822+0000","last_peered":"2024-08-29T15:20:20.290822+0000","last_clean":"2024-08-29T15:20:20.290822+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:20:20.290822+0000","last_undegraded":"2024-08-29T15:20:20.290822+0000","last_fullsized":"2024-08-29T15:20:20.290822+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290066+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:20:20.290066+0000","last_peered":"2024-08-29T15:20:20.290066+0000","last_clean":"2024-08-29T15:20:20.290066+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:20:20.290066+0000","last_undegraded":"2024-08-29T15:20:20.290066+0000","last_fullsized":"2024-08-29T15:20:20.290066+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":85,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290149+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:20:20.290149+0000","last_peered":"2024-08-29T15:20:20.290149+0000","last_clean":"2024-08-29T15:20:20.290149+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:20:20.290149+0000","last_undegraded":"2024-08-29T15:20:20.290149+0000","last_fullsized":"2024-08-29T15:20:20.290149+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"242'9612","reported_seq":9781,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:33.428434+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:20:33.428434+0000","last_peered":"2024-08-29T15:20:33.428434+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:20:33.428434+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"176'1442","ondisk_log_start":"176'1442","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8170,"log_dups_size":0,"ondisk_log_size":8170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407240704,"num_objects":6214,"num_object_clones":0,"num_object_copies":18642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9612,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3791}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":110,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290022+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:20:20.290022+0000","last_peered":"2024-08-29T15:20:20.290022+0000","last_clean":"2024-08-29T15:20:20.290022+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:20:20.290022+0000","last_undegraded":"2024-08-29T15:20:20.290022+0000","last_fullsized":"2024-08-29T15:20:20.290022+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":775,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.719422+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:20:30.719422+0000","last_peered":"2024-08-29T15:20:30.719422+0000","last_clean":"2024-08-29T15:20:30.719422+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:20:30.719422+0000","last_undegraded":"2024-08-29T15:20:30.719422+0000","last_fullsized":"2024-08-29T15:20:30.719422+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":352,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718557+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:20:30.718557+0000","last_peered":"2024-08-29T15:20:30.718557+0000","last_clean":"2024-08-29T15:20:30.718557+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:20:30.718557+0000","last_undegraded":"2024-08-29T15:20:30.718557+0000","last_fullsized":"2024-08-29T15:20:30.718557+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"242'9579","reported_seq":10304,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:35.007424+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:20:35.007424+0000","last_peered":"2024-08-29T15:20:35.007424+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:20:35.007424+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"177'2580","ondisk_log_start":"177'2580","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6999,"log_dups_size":0,"ondisk_log_size":6999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399048704,"num_objects":6089,"num_object_clones":0,"num_object_copies":18267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":6052,"num_objects_unfound":0,"num_objects_dirty":6089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9579,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3938}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":357,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.719318+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:20:30.719318+0000","last_peered":"2024-08-29T15:20:30.719318+0000","last_clean":"2024-08-29T15:20:30.719318+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:20:30.719318+0000","last_undegraded":"2024-08-29T15:20:30.719318+0000","last_fullsized":"2024-08-29T15:20:30.719318+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":667,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:20.509692+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:20:20.509692+0000","last_peered":"2024-08-29T15:20:20.509692+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:20:20.509692+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:20:20.509692+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":348,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732536+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:20:30.732536+0000","last_peered":"2024-08-29T15:20:30.732536+0000","last_clean":"2024-08-29T15:20:30.732536+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:20:30.732536+0000","last_undegraded":"2024-08-29T15:20:30.732536+0000","last_fullsized":"2024-08-29T15:20:30.732536+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"242'9461","reported_seq":16586,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:34.356742+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:20:34.356742+0000","last_peered":"2024-08-29T15:20:34.356742+0000","last_clean":"2024-08-29T15:20:34.356742+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:20:34.356742+0000","last_undegraded":"2024-08-29T15:20:34.356742+0000","last_fullsized":"2024-08-29T15:20:34.356742+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391053312,"num_objects":5967,"num_object_clones":0,"num_object_copies":17901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9461,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":342,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.716193+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:20:30.716193+0000","last_peered":"2024-08-29T15:20:30.716193+0000","last_clean":"2024-08-29T15:20:30.716193+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:20:30.716193+0000","last_undegraded":"2024-08-29T15:20:30.716193+0000","last_fullsized":"2024-08-29T15:20:30.716193+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":964,"reported_epoch":242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:30.718062+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:20:30.718062+0000","last_peered":"2024-08-29T15:20:30.718062+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:20:30.718062+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":339,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.714864+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:20:30.714864+0000","last_peered":"2024-08-29T15:20:30.714864+0000","last_clean":"2024-08-29T15:20:30.714864+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:20:30.714864+0000","last_undegraded":"2024-08-29T15:20:30.714864+0000","last_fullsized":"2024-08-29T15:20:30.714864+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"242'9548","reported_seq":10937,"reported_epoch":242,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:32.571132+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:20:32.571132+0000","last_peered":"2024-08-29T15:20:32.571132+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:20:32.571132+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"176'2012","ondisk_log_start":"176'2012","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7536,"log_dups_size":0,"ondisk_log_size":7536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397279232,"num_objects":6062,"num_object_clones":0,"num_object_copies":18186,"num_objects_missing_on_primary":7097,"num_objects_missing":7097,"num_objects_degraded":2313,"num_objects_misplaced":6048,"num_objects_unfound":0,"num_objects_dirty":6062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9548,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":624,"num_bytes_recovered":40763392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":3677},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":338,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732527+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:20:30.732527+0000","last_peered":"2024-08-29T15:20:30.732527+0000","last_clean":"2024-08-29T15:20:30.732527+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:20:30.732527+0000","last_undegraded":"2024-08-29T15:20:30.732527+0000","last_fullsized":"2024-08-29T15:20:30.732527+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":91,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.459592+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:20:20.459592+0000","last_peered":"2024-08-29T15:20:20.459592+0000","last_clean":"2024-08-29T15:20:20.459592+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:20:20.459592+0000","last_undegraded":"2024-08-29T15:20:20.459592+0000","last_fullsized":"2024-08-29T15:20:20.459592+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290512+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:20:20.290512+0000","last_peered":"2024-08-29T15:20:20.290512+0000","last_clean":"2024-08-29T15:20:20.290512+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:20:20.290512+0000","last_undegraded":"2024-08-29T15:20:20.290512+0000","last_fullsized":"2024-08-29T15:20:20.290512+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.291020+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:20:20.291020+0000","last_peered":"2024-08-29T15:20:20.291020+0000","last_clean":"2024-08-29T15:20:20.291020+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:20:20.291020+0000","last_undegraded":"2024-08-29T15:20:20.291020+0000","last_fullsized":"2024-08-29T15:20:20.291020+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":17,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:20.290477+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:20:20.290477+0000","last_peered":"2024-08-29T15:20:20.290477+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:20:20.290477+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:20:20.290477+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":346,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718368+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:20:30.718368+0000","last_peered":"2024-08-29T15:20:30.718368+0000","last_clean":"2024-08-29T15:20:30.718368+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:20:30.718368+0000","last_undegraded":"2024-08-29T15:20:30.718368+0000","last_fullsized":"2024-08-29T15:20:30.718368+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":96,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:30.715608+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:20:30.715608+0000","last_peered":"2024-08-29T15:20:30.715608+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:20:30.715608+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:20:30.715608+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":108,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290905+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:20:20.290905+0000","last_peered":"2024-08-29T15:20:20.290905+0000","last_clean":"2024-08-29T15:20:20.290905+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:20:20.290905+0000","last_undegraded":"2024-08-29T15:20:20.290905+0000","last_fullsized":"2024-08-29T15:20:20.290905+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":305,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.714544+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:20:30.714544+0000","last_peered":"2024-08-29T15:20:30.714544+0000","last_clean":"2024-08-29T15:20:30.714544+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:20:30.714544+0000","last_undegraded":"2024-08-29T15:20:30.714544+0000","last_fullsized":"2024-08-29T15:20:30.714544+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":359,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732792+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:20:30.732792+0000","last_peered":"2024-08-29T15:20:30.732792+0000","last_clean":"2024-08-29T15:20:30.732792+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:20:30.732792+0000","last_undegraded":"2024-08-29T15:20:30.732792+0000","last_fullsized":"2024-08-29T15:20:30.732792+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":357,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.714757+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:20:30.714757+0000","last_peered":"2024-08-29T15:20:30.714757+0000","last_clean":"2024-08-29T15:20:30.714757+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:20:30.714757+0000","last_undegraded":"2024-08-29T15:20:30.714757+0000","last_fullsized":"2024-08-29T15:20:30.714757+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.459639+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:20:20.459639+0000","last_peered":"2024-08-29T15:20:20.459639+0000","last_clean":"2024-08-29T15:20:20.459639+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:20:20.459639+0000","last_undegraded":"2024-08-29T15:20:20.459639+0000","last_fullsized":"2024-08-29T15:20:20.459639+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":214,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732158+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:20:30.732158+0000","last_peered":"2024-08-29T15:20:30.732158+0000","last_clean":"2024-08-29T15:20:30.732158+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:20:30.732158+0000","last_undegraded":"2024-08-29T15:20:30.732158+0000","last_fullsized":"2024-08-29T15:20:30.732158+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":334,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732796+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:20:30.732796+0000","last_peered":"2024-08-29T15:20:30.732796+0000","last_clean":"2024-08-29T15:20:30.732796+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:20:30.732796+0000","last_undegraded":"2024-08-29T15:20:30.732796+0000","last_fullsized":"2024-08-29T15:20:30.732796+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":338,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718679+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:20:30.718679+0000","last_peered":"2024-08-29T15:20:30.718679+0000","last_clean":"2024-08-29T15:20:30.718679+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:20:30.718679+0000","last_undegraded":"2024-08-29T15:20:30.718679+0000","last_fullsized":"2024-08-29T15:20:30.718679+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":342,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715473+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:20:30.715473+0000","last_peered":"2024-08-29T15:20:30.715473+0000","last_clean":"2024-08-29T15:20:30.715473+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:20:30.715473+0000","last_undegraded":"2024-08-29T15:20:30.715473+0000","last_fullsized":"2024-08-29T15:20:30.715473+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290071+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:20:20.290071+0000","last_peered":"2024-08-29T15:20:20.290071+0000","last_clean":"2024-08-29T15:20:20.290071+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:20:20.290071+0000","last_undegraded":"2024-08-29T15:20:20.290071+0000","last_fullsized":"2024-08-29T15:20:20.290071+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":362,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.719523+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:20:30.719523+0000","last_peered":"2024-08-29T15:20:30.719523+0000","last_clean":"2024-08-29T15:20:30.719523+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:20:30.719523+0000","last_undegraded":"2024-08-29T15:20:30.719523+0000","last_fullsized":"2024-08-29T15:20:30.719523+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":326,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732416+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:20:30.732416+0000","last_peered":"2024-08-29T15:20:30.732416+0000","last_clean":"2024-08-29T15:20:30.732416+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:20:30.732416+0000","last_undegraded":"2024-08-29T15:20:30.732416+0000","last_fullsized":"2024-08-29T15:20:30.732416+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.289961+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:20:20.289961+0000","last_peered":"2024-08-29T15:20:20.289961+0000","last_clean":"2024-08-29T15:20:20.289961+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:20:20.289961+0000","last_undegraded":"2024-08-29T15:20:20.289961+0000","last_fullsized":"2024-08-29T15:20:20.289961+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":132,"reported_epoch":241,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:20.290305+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:20:20.290305+0000","last_peered":"2024-08-29T15:20:20.290305+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:20:20.290305+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:20:20.290305+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290149+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:20:20.290149+0000","last_peered":"2024-08-29T15:20:20.290149+0000","last_clean":"2024-08-29T15:20:20.290149+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:20:20.290149+0000","last_undegraded":"2024-08-29T15:20:20.290149+0000","last_fullsized":"2024-08-29T15:20:20.290149+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":138,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.290131+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:20:20.290131+0000","last_peered":"2024-08-29T15:20:20.290131+0000","last_clean":"2024-08-29T15:20:20.290131+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:20:20.290131+0000","last_undegraded":"2024-08-29T15:20:20.290131+0000","last_fullsized":"2024-08-29T15:20:20.290131+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":373,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.717957+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:20:30.717957+0000","last_peered":"2024-08-29T15:20:30.717957+0000","last_clean":"2024-08-29T15:20:30.717957+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:20:30.717957+0000","last_undegraded":"2024-08-29T15:20:30.717957+0000","last_fullsized":"2024-08-29T15:20:30.717957+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":374,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.716599+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:20:30.716599+0000","last_peered":"2024-08-29T15:20:30.716599+0000","last_clean":"2024-08-29T15:20:30.716599+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:20:30.716599+0000","last_undegraded":"2024-08-29T15:20:30.716599+0000","last_fullsized":"2024-08-29T15:20:30.716599+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":19,"reported_epoch":241,"state":"active+clean","last_fresh":"2024-08-29T15:20:20.289851+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:20:20.289851+0000","last_peered":"2024-08-29T15:20:20.289851+0000","last_clean":"2024-08-29T15:20:20.289851+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:20:20.289851+0000","last_undegraded":"2024-08-29T15:20:20.289851+0000","last_fullsized":"2024-08-29T15:20:20.289851+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":382,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.732713+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:20:30.732713+0000","last_peered":"2024-08-29T15:20:30.732713+0000","last_clean":"2024-08-29T15:20:30.732713+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:20:30.732713+0000","last_undegraded":"2024-08-29T15:20:30.732713+0000","last_fullsized":"2024-08-29T15:20:30.732713+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6408634392,"num_objects":97789,"num_object_clones":0,"num_object_copies":293367,"num_objects_missing_on_primary":7097,"num_objects_missing":7097,"num_objects_degraded":30073,"num_objects_misplaced":18254,"num_objects_unfound":0,"num_objects_dirty":97789,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153061,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14251,"num_bytes_recovered":813957120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19089797120,"data_stored":19089784904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119938,"ondisk_log_size":119938,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234311,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3435384,"kb_used_data":3139312,"kb_used_omap":1,"kb_used_meta":296062,"kb_avail":90309768,"statfs":{"total":95995035648,"available":92477202432,"internally_reserved":0,"allocated":3214655488,"data_stored":3213308091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":303167947},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495943,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":501304,"kb_used_data":331564,"kb_used_omap":1,"kb_used_meta":169726,"kb_avail":93243848,"statfs":{"total":95995035648,"available":95481700352,"internally_reserved":0,"allocated":339521536,"data_stored":338177939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":173799879},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,0,2,1,3,7,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426529,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2002728,"kb_used_data":1947616,"kb_used_omap":8,"kb_used_meta":55095,"kb_avail":91742424,"statfs":{"total":95995035648,"available":93944242176,"internally_reserved":0,"allocated":1994358784,"data_stored":1993022653,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":56418186},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019221,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2237848,"kb_used_data":1949968,"kb_used_omap":1,"kb_used_meta":287870,"kb_avail":91507304,"statfs":{"total":95995035648,"available":93703479296,"internally_reserved":0,"allocated":1996767232,"data_stored":1995446884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":294779334},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435405,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4191720,"kb_used_data":3921308,"kb_used_omap":4,"kb_used_meta":270395,"kb_avail":89553432,"statfs":{"total":95995035648,"available":91702714368,"internally_reserved":0,"allocated":4015419392,"data_stored":4014087201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":276884745},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903257,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3507884,"kb_used_data":3438624,"kb_used_omap":10,"kb_used_meta":69237,"kb_avail":90237268,"statfs":{"total":95995035648,"available":92402962432,"internally_reserved":0,"allocated":3521150976,"data_stored":3519818721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":70899216},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762459,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2638884,"kb_used_data":2382300,"kb_used_omap":7,"kb_used_meta":256568,"kb_avail":91106268,"statfs":{"total":95995035648,"available":93292818432,"internally_reserved":0,"allocated":2439475200,"data_stored":2438141769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":262725986},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925985,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1892396,"kb_used_data":1585828,"kb_used_omap":7,"kb_used_meta":306552,"kb_avail":91852756,"statfs":{"total":95995035648,"available":94057222144,"internally_reserved":0,"allocated":1623887872,"data_stored":1622547220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":313909391},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1615990784,"data_stored":1615986712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2432176128,"data_stored":2432172056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3516727296,"data_stored":3516727296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4008181760,"data_stored":4008181760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1989804032,"data_stored":1989804032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1989480448,"data_stored":1989476376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":335413248,"data_stored":335413248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3202023424,"data_stored":3202023424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:20:41.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:41.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:41.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:41.906 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:41.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:41.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:41.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:41.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:41 smithi027 ceph-mon[98970]: pgmap v732: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30102/292338 objects degraded (10.297%); 18189/292338 objects misplaced (6.222%); 436 KiB/s, 6 objects/s recovering 2024-08-29T15:20:41.966 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:41 smithi027 ceph-mon[93921]: pgmap v732: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30102/292338 objects degraded (10.297%); 18189/292338 objects misplaced (6.222%); 436 KiB/s, 6 objects/s recovering 2024-08-29T15:20:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:41 smithi171 ceph-mon[90031]: pgmap v732: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30102/292338 objects degraded (10.297%); 18189/292338 objects misplaced (6.222%); 436 KiB/s, 6 objects/s recovering 2024-08-29T15:20:42.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:42.053 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:42.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:42.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.178 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:42.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:42.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:42.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:42.404 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:42.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:42.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:20:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=cleanup t=2024-08-29T15:20:42.199674195Z level=info msg="Completed cleanup jobs" duration=2.198688ms 2024-08-29T15:20:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:20:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana.update.checker t=2024-08-29T15:20:42.346936842Z level=info msg="Update check succeeded" duration=63.824344ms 2024-08-29T15:20:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:20:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugins.update.checker t=2024-08-29T15:20:42.35060088Z level=info msg="Update check succeeded" duration=60.586108ms 2024-08-29T15:20:42.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:42.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:42.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:42.671 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:42.773 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:42.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:42.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:42.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/292338 objects degraded (10.297%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:42.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/292338 objects degraded (10.297%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:42.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:20:42.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:42.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/292338 objects degraded (10.297%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:43.024 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:43.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:43.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:43.175 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:20:43.175 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:20:43.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.175 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:154087 6.9 e243/218 6:9df705a9:::benchmark_data_smithi027_169445_object28660:head v 243'9754, mlcod=194'8073)", 2024-08-29T15:20:43.175 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:20:43.171995+0000", 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0014414740000000001, 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014682670000000001, 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.171995+0000", 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:20:43.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.171995+0000", 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.171996+0000", 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2200000000000002e-07 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.172003+0000", 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2559999999999996e-06 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.172018+0000", 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4878000000000001e-05 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.172027+0000", 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3749999999999992e-06 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.172096+0000", 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8256999999999994e-05 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:20:43.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:43.172123+0000", 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7359e-05 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2024-08-29T15:20:43.179 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:20:43.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:43.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:43.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:20:43.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:43.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:43.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:43.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:43.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:43.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:43.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:43.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:43.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:43.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:43.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:43.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:43.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:43.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:43.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:43.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:43.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:20:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:43 smithi171 ceph-mon[90031]: pgmap v733: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 30102/291099 objects degraded (10.341%); 18108/291099 objects misplaced (6.221%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:44.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:44.053 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:44.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:43 smithi027 ceph-mon[98970]: pgmap v733: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 30102/291099 objects degraded (10.341%); 18108/291099 objects misplaced (6.221%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:43 smithi027 ceph-mon[93921]: pgmap v733: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 30102/291099 objects degraded (10.341%); 18108/291099 objects misplaced (6.221%); 429 KiB/s, 6 objects/s recovering 2024-08-29T15:20:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:44.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:44.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:44.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:44.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:44.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:44.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:44.440 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:44.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:44.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:44.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:44.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:44.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:44.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:44.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:44.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:20:44.775 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:44.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:44.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:44.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:44.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:44.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:45.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:45.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:45.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:45.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:45.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:45.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:45.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:45.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:45.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:45.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:45.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:45.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:45.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:45.690 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:45.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:45.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:45.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:45.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:45.922 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:20:45.923 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:20:45.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:45.941 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:20:45.941 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:45 smithi027 ceph-mon[98970]: pgmap v734: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/290796 objects degraded (10.352%); 18070/290796 objects misplaced (6.214%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:20:45.942 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:20:45.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:45 smithi027 ceph-mon[93921]: pgmap v734: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/290796 objects degraded (10.352%); 18070/290796 objects misplaced (6.214%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:20:45.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:45.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:45.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:45.965 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:20:45.965 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:45.981 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:45.981 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:45.998 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:45.998 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:46.014 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:46.014 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:45 smithi171 ceph-mon[90031]: pgmap v734: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/290796 objects degraded (10.352%); 18070/290796 objects misplaced (6.214%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:20:46.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:46.031 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:46.047 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:46.048 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:46.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:46.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:46.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:46.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:46.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:46.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:46.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:46.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.456 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:46.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:46.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:46.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:46.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:46.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:46.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:46.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:46.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:46.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3941725779' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3941725779' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:46.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:46.859 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:46.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:46.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:46.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:47.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:47.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:47.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:47.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:47.202 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:47.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3941725779' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:47.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:47.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:47.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:47.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:47.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:47.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:47.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:47.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:47.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:47.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:47 smithi027 ceph-mon[93921]: pgmap v735: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/290124 objects degraded (10.376%); 18038/290124 objects misplaced (6.217%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:20:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/290124 objects degraded (10.376%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:47 smithi027 ceph-mon[98970]: pgmap v735: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/290124 objects degraded (10.376%); 18038/290124 objects misplaced (6.217%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:20:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/290124 objects degraded (10.376%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:47.824 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:47.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:47.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:47.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:47.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:48.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:48.077 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:48.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:48.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:48.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:47 smithi171 ceph-mon[90031]: pgmap v735: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/290124 objects degraded (10.376%); 18038/290124 objects misplaced (6.217%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:20:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/290124 objects degraded (10.376%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:48.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:48.328 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:48.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:48.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:48.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:48.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:48.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:48.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:48.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:48.702 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:48.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:48.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:48.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:48.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:48.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:48.951 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:48.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:49.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:49.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:49.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:49.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:49.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:49.307 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:20:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:20:49] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-08-29T15:20:49.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:49.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:49.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:49.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:49.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:49.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:49.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:20:49.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:49.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:49.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:49.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:49.831 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:49.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:49.936 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:49 smithi027 ceph-mon[93921]: pgmap v736: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/289377 objects degraded (10.402%); 17975/289377 objects misplaced (6.212%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:20:49.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:49 smithi027 ceph-mon[98970]: pgmap v736: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/289377 objects degraded (10.402%); 17975/289377 objects misplaced (6.212%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:20:49.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:49.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:49.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:50.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:50.098 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:50.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:50.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:50.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:50.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:50.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:49 smithi171 ceph-mon[90031]: pgmap v736: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/289377 objects degraded (10.402%); 17975/289377 objects misplaced (6.212%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:20:50.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:50.378 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:50.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:50.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:50.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:50.523 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:50.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:50.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:50.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:50.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:50.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:50.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:50.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:50.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:50.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:50.781 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:20:50.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:50.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:50.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:50.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:50.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:51.020 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:51.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3835831480' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:51.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3835831480' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:51.122 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:51.146 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:51.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3835831480' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:20:51.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:51.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:51.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:51.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:51.523 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:51.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:51.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:51.769 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:51.794 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:51.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:51.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:51.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:51.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:51.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:52.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:52.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[98970]: pgmap v737: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/288930 objects degraded (10.418%); 17975/288930 objects misplaced (6.221%); 433 KiB/s, 6 objects/s recovering 2024-08-29T15:20:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:20:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3835831480' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[98970]: osdmap e244: 8 total, 8 up, 8 in 2024-08-29T15:20:52.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[93921]: pgmap v737: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/288930 objects degraded (10.418%); 17975/288930 objects misplaced (6.221%); 433 KiB/s, 6 objects/s recovering 2024-08-29T15:20:52.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:20:52.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3835831480' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:52.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:51 smithi027 ceph-mon[93921]: osdmap e244: 8 total, 8 up, 8 in 2024-08-29T15:20:52.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:52.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:51 smithi171 ceph-mon[90031]: pgmap v737: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/288930 objects degraded (10.418%); 17975/288930 objects misplaced (6.221%); 433 KiB/s, 6 objects/s recovering 2024-08-29T15:20:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:51 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:20:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3835831480' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:20:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:51 smithi171 ceph-mon[90031]: osdmap e244: 8 total, 8 up, 8 in 2024-08-29T15:20:52.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:52.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:52.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:52.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:52.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:52.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:52.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:52.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:52.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:52.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:52.789 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:52.891 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:52.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:52.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:52.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/288930 objects degraded (10.418%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:52.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/288930 objects degraded (10.418%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:53.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:53.039 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:53.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:53.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:53.188 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:20:53.189 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:20:53.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/288930 objects degraded (10.418%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:53.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:53.389 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:53.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:53.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:53.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:53.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:53.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:53.749 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:20:53.749 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:20:53.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:53.766 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:20:53.766 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:20:53.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:53.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:53.784 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:20:53.784 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:20:53.802 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:20:53.802 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:20:53.819 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:20:53.819 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:20:53.835 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:20:53.836 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:20:53.852 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:20:53.852 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:20:53.870 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:20:53.871 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:20:53.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:53.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:53.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:54.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:54.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:53 smithi027 ceph-mon[98970]: pgmap v739: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 30102/287577 objects degraded (10.467%); 17905/287577 objects misplaced (6.226%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:20:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:53 smithi027 ceph-mon[98970]: from='client.45365 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1961478163' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:53 smithi027 ceph-mon[93921]: pgmap v739: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 30102/287577 objects degraded (10.467%); 17905/287577 objects misplaced (6.226%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:20:54.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:53 smithi027 ceph-mon[93921]: from='client.45365 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:54.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1961478163' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:54.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:54.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:54.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:54.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:53 smithi171 ceph-mon[90031]: pgmap v739: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 30102/287577 objects degraded (10.467%); 17905/287577 objects misplaced (6.226%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:20:54.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:53 smithi171 ceph-mon[90031]: from='client.45365 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:20:54.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1961478163' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:20:54.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:54.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:54.384 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:20:54.384 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:20:54.384 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:20:54.385 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:20:54.385 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:20:54.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:54.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:54.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:54.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:54.623 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":739,"stamp":"2024-08-29T15:20:52.546257+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6285105149,"num_objects":95859,"num_object_clones":0,"num_object_copies":287577,"num_objects_missing_on_primary":6998,"num_objects_missing":6998,"num_objects_degraded":30102,"num_objects_misplaced":17905,"num_objects_unfound":0,"num_objects_dirty":95859,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":156142,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14468,"num_bytes_recovered":827002061,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121578,"ondisk_log_size":121578,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20049580,"kb_used_data":18334632,"kb_used_omap":43,"kb_used_meta":1714836,"kb_avail":729911636,"statfs":{"total":767960285184,"available":747429515264,"internally_reserved":0,"allocated":18774663168,"data_stored":18763900686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1755992556},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,1,0,1,2,3,8,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-76939264,"num_objects":-1174,"num_object_clones":0,"num_object_copies":-3522,"num_objects_missing_on_primary":-73,"num_objects_missing":-73,"num_objects_degraded":0,"num_objects_misplaced":-203,"num_objects_unfound":0,"num_objects_dirty":-1174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1174,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003058"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":372,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785749+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:20:50.785749+0000","last_peered":"2024-08-29T15:20:50.785749+0000","last_clean":"2024-08-29T15:20:50.785749+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:20:50.785749+0000","last_undegraded":"2024-08-29T15:20:50.785749+0000","last_fullsized":"2024-08-29T15:20:50.785749+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":19,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:30.718337+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:20:30.718337+0000","last_peered":"2024-08-29T15:20:30.718337+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:20:30.718337+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:20:30.718337+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748322+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:20:39.748322+0000","last_peered":"2024-08-29T15:20:39.748322+0000","last_clean":"2024-08-29T15:20:39.748322+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:20:39.748322+0000","last_undegraded":"2024-08-29T15:20:39.748322+0000","last_fullsized":"2024-08-29T15:20:39.748322+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":335,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.750137+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:20:39.750137+0000","last_peered":"2024-08-29T15:20:39.750137+0000","last_clean":"2024-08-29T15:20:39.750137+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:20:39.750137+0000","last_undegraded":"2024-08-29T15:20:39.750137+0000","last_fullsized":"2024-08-29T15:20:39.750137+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":335,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748994+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:20:39.748994+0000","last_peered":"2024-08-29T15:20:39.748994+0000","last_clean":"2024-08-29T15:20:39.748994+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:20:39.748994+0000","last_undegraded":"2024-08-29T15:20:39.748994+0000","last_fullsized":"2024-08-29T15:20:39.748994+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.717952+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:20:30.717952+0000","last_peered":"2024-08-29T15:20:30.717952+0000","last_clean":"2024-08-29T15:20:30.717952+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:20:30.717952+0000","last_undegraded":"2024-08-29T15:20:30.717952+0000","last_fullsized":"2024-08-29T15:20:30.717952+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":147,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.754464+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:20:39.754464+0000","last_peered":"2024-08-29T15:20:39.754464+0000","last_clean":"2024-08-29T15:20:39.754464+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:20:39.754464+0000","last_undegraded":"2024-08-29T15:20:39.754464+0000","last_fullsized":"2024-08-29T15:20:39.754464+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":110,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715769+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:20:30.715769+0000","last_peered":"2024-08-29T15:20:30.715769+0000","last_clean":"2024-08-29T15:20:30.715769+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:20:30.715769+0000","last_undegraded":"2024-08-29T15:20:30.715769+0000","last_fullsized":"2024-08-29T15:20:30.715769+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"244'9853","reported_seq":17874,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:50.999169+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:20:50.999169+0000","last_peered":"2024-08-29T15:20:50.999169+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:20:50.999169+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9853,"log_dups_size":0,"ondisk_log_size":9853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397082624,"num_objects":6059,"num_object_clones":0,"num_object_copies":18177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9853,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":3134}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":18,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:30.718820+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:20:30.718820+0000","last_peered":"2024-08-29T15:20:30.718820+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:20:30.718820+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:20:30.718820+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.747575+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:20:39.747575+0000","last_peered":"2024-08-29T15:20:39.747575+0000","last_clean":"2024-08-29T15:20:39.747575+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:20:39.747575+0000","last_undegraded":"2024-08-29T15:20:39.747575+0000","last_fullsized":"2024-08-29T15:20:39.747575+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"244'9611","reported_seq":9776,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:51.756301+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:20:51.756301+0000","last_peered":"2024-08-29T15:20:51.756301+0000","last_clean":"2024-08-29T15:20:51.756301+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:20:51.756301+0000","last_undegraded":"2024-08-29T15:20:51.756301+0000","last_fullsized":"2024-08-29T15:20:51.756301+0000","mapping_epoch":168,"log_start":"178'2928","ondisk_log_start":"178'2928","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389218304,"num_objects":5939,"num_object_clones":0,"num_object_copies":17817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9611,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":18,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:30.721458+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:20:30.721458+0000","last_peered":"2024-08-29T15:20:30.721458+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:20:30.721458+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:20:30.721458+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.717764+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:20:30.717764+0000","last_peered":"2024-08-29T15:20:30.717764+0000","last_clean":"2024-08-29T15:20:30.717764+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:20:30.717764+0000","last_undegraded":"2024-08-29T15:20:30.717764+0000","last_fullsized":"2024-08-29T15:20:30.717764+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"244'9727","reported_seq":9862,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:51.724878+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:20:51.724878+0000","last_peered":"2024-08-29T15:20:51.724878+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:20:51.724878+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'2974","ondisk_log_start":"178'2974","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392232960,"num_objects":5985,"num_object_clones":0,"num_object_copies":17955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9727,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3508}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":151,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.751047+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:20:39.751047+0000","last_peered":"2024-08-29T15:20:39.751047+0000","last_clean":"2024-08-29T15:20:39.751047+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:20:39.751047+0000","last_undegraded":"2024-08-29T15:20:39.751047+0000","last_fullsized":"2024-08-29T15:20:39.751047+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":65,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715326+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:20:30.715326+0000","last_peered":"2024-08-29T15:20:30.715326+0000","last_clean":"2024-08-29T15:20:30.715326+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:20:30.715326+0000","last_undegraded":"2024-08-29T15:20:30.715326+0000","last_fullsized":"2024-08-29T15:20:30.715326+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"243'9795","reported_seq":15178,"reported_epoch":243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:48.226516+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:20:48.226516+0000","last_peered":"2024-08-29T15:20:48.226516+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:20:48.226516+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'1662","ondisk_log_start":"176'1662","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8133,"log_dups_size":0,"ondisk_log_size":8133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392101888,"num_objects":5983,"num_object_clones":0,"num_object_copies":17949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9795,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5983}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":128,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748877+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:20:39.748877+0000","last_peered":"2024-08-29T15:20:39.748877+0000","last_clean":"2024-08-29T15:20:39.748877+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:20:39.748877+0000","last_undegraded":"2024-08-29T15:20:39.748877+0000","last_fullsized":"2024-08-29T15:20:39.748877+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":350,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748727+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:20:39.748727+0000","last_peered":"2024-08-29T15:20:39.748727+0000","last_clean":"2024-08-29T15:20:39.748727+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:20:39.748727+0000","last_undegraded":"2024-08-29T15:20:39.748727+0000","last_fullsized":"2024-08-29T15:20:39.748727+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"243'9762","reported_seq":9890,"reported_epoch":243,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:49.376880+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:20:49.376880+0000","last_peered":"2024-08-29T15:20:49.376880+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:20:49.376880+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:20:49.376880+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7429,"log_dups_size":0,"ondisk_log_size":7429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396623872,"num_objects":6052,"num_object_clones":0,"num_object_copies":18156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9762,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3575},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":346,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785315+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:20:50.785315+0000","last_peered":"2024-08-29T15:20:50.785315+0000","last_clean":"2024-08-29T15:20:50.785315+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:20:50.785315+0000","last_undegraded":"2024-08-29T15:20:50.785315+0000","last_fullsized":"2024-08-29T15:20:50.785315+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718354+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:20:30.718354+0000","last_peered":"2024-08-29T15:20:30.718354+0000","last_clean":"2024-08-29T15:20:30.718354+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:20:30.718354+0000","last_undegraded":"2024-08-29T15:20:30.718354+0000","last_fullsized":"2024-08-29T15:20:30.718354+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"244'9764","reported_seq":9945,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:51.541900+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:20:51.541900+0000","last_peered":"2024-08-29T15:20:51.541900+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:20:51.541900+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:20:51.541900+0000","mapping_epoch":234,"log_start":"178'3068","ondisk_log_start":"178'3068","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398327808,"num_objects":6078,"num_object_clones":0,"num_object_copies":18234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9764,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3613},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":83,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715203+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:20:30.715203+0000","last_peered":"2024-08-29T15:20:30.715203+0000","last_clean":"2024-08-29T15:20:30.715203+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:20:30.715203+0000","last_undegraded":"2024-08-29T15:20:30.715203+0000","last_fullsized":"2024-08-29T15:20:30.715203+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718666+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:20:30.718666+0000","last_peered":"2024-08-29T15:20:30.718666+0000","last_clean":"2024-08-29T15:20:30.718666+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:20:30.718666+0000","last_undegraded":"2024-08-29T15:20:30.718666+0000","last_fullsized":"2024-08-29T15:20:30.718666+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"244'9655","reported_seq":10021,"reported_epoch":244,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:20:51.768973+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:20:51.768973+0000","last_peered":"2024-08-29T15:20:51.768973+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:20:51.768973+0000","last_undegraded":"2024-08-29T15:20:51.768973+0000","last_fullsized":"2024-08-29T15:20:51.768973+0000","mapping_epoch":202,"log_start":"178'2967","ondisk_log_start":"178'2967","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393412608,"num_objects":6003,"num_object_clones":0,"num_object_copies":18009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6003,"num_objects_unfound":0,"num_objects_dirty":6003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9655,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":6003}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.717840+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:20:30.717840+0000","last_peered":"2024-08-29T15:20:30.717840+0000","last_clean":"2024-08-29T15:20:30.717840+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:20:30.717840+0000","last_undegraded":"2024-08-29T15:20:30.717840+0000","last_fullsized":"2024-08-29T15:20:30.717840+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":226,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.721468+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:20:30.721468+0000","last_peered":"2024-08-29T15:20:30.721468+0000","last_clean":"2024-08-29T15:20:30.721468+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:20:30.721468+0000","last_undegraded":"2024-08-29T15:20:30.721468+0000","last_fullsized":"2024-08-29T15:20:30.721468+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"244'9723","reported_seq":9846,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:51.099797+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:20:51.099797+0000","last_peered":"2024-08-29T15:20:51.099797+0000","last_clean":"2024-08-29T15:20:51.099797+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:20:51.099797+0000","last_undegraded":"2024-08-29T15:20:51.099797+0000","last_fullsized":"2024-08-29T15:20:51.099797+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8445,"log_dups_size":0,"ondisk_log_size":8445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395575320,"num_objects":6037,"num_object_clones":0,"num_object_copies":18111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6037,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9723,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":349,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785531+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:20:50.785531+0000","last_peered":"2024-08-29T15:20:50.785531+0000","last_clean":"2024-08-29T15:20:50.785531+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:20:50.785531+0000","last_undegraded":"2024-08-29T15:20:50.785531+0000","last_fullsized":"2024-08-29T15:20:50.785531+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":339,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.754524+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:20:39.754524+0000","last_peered":"2024-08-29T15:20:39.754524+0000","last_clean":"2024-08-29T15:20:39.754524+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:20:39.754524+0000","last_undegraded":"2024-08-29T15:20:39.754524+0000","last_fullsized":"2024-08-29T15:20:39.754524+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"244'9410","reported_seq":9581,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784193+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:20:50.784193+0000","last_peered":"2024-08-29T15:20:50.784193+0000","last_clean":"2024-08-29T15:20:50.784193+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:20:50.784193+0000","last_undegraded":"2024-08-29T15:20:50.784193+0000","last_fullsized":"2024-08-29T15:20:50.784193+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9410,"log_dups_size":0,"ondisk_log_size":9410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379977728,"num_objects":5798,"num_object_clones":0,"num_object_copies":17394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9410,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":438,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.750971+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:20:39.750971+0000","last_peered":"2024-08-29T15:20:39.750971+0000","last_clean":"2024-08-29T15:20:39.750971+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:20:39.750971+0000","last_undegraded":"2024-08-29T15:20:39.750971+0000","last_fullsized":"2024-08-29T15:20:39.750971+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":712,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.747841+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:20:39.747841+0000","last_peered":"2024-08-29T15:20:39.747841+0000","last_clean":"2024-08-29T15:20:39.747841+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:20:39.747841+0000","last_undegraded":"2024-08-29T15:20:39.747841+0000","last_fullsized":"2024-08-29T15:20:39.747841+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":220,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748953+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:20:39.748953+0000","last_peered":"2024-08-29T15:20:39.748953+0000","last_clean":"2024-08-29T15:20:39.748953+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:20:39.748953+0000","last_undegraded":"2024-08-29T15:20:39.748953+0000","last_fullsized":"2024-08-29T15:20:39.748953+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715387+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:20:30.715387+0000","last_peered":"2024-08-29T15:20:30.715387+0000","last_clean":"2024-08-29T15:20:30.715387+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:20:30.715387+0000","last_undegraded":"2024-08-29T15:20:30.715387+0000","last_fullsized":"2024-08-29T15:20:30.715387+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"243'9755","reported_seq":10116,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.327735+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:20:50.327735+0000","last_peered":"2024-08-29T15:20:50.327735+0000","last_clean":"2024-08-29T15:20:50.327735+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:20:50.327735+0000","last_undegraded":"2024-08-29T15:20:50.327735+0000","last_fullsized":"2024-08-29T15:20:50.327735+0000","mapping_epoch":201,"log_start":"177'2753","ondisk_log_start":"177'2753","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388431872,"num_objects":5927,"num_object_clones":0,"num_object_copies":17781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9755,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":812,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.753214+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:20:39.753214+0000","last_peered":"2024-08-29T15:20:39.753214+0000","last_clean":"2024-08-29T15:20:39.753214+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:20:39.753214+0000","last_undegraded":"2024-08-29T15:20:39.753214+0000","last_fullsized":"2024-08-29T15:20:39.753214+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":350,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.750486+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:20:39.750486+0000","last_peered":"2024-08-29T15:20:39.750486+0000","last_clean":"2024-08-29T15:20:39.750486+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:20:39.750486+0000","last_undegraded":"2024-08-29T15:20:39.750486+0000","last_fullsized":"2024-08-29T15:20:39.750486+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":358,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.749328+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:20:39.749328+0000","last_peered":"2024-08-29T15:20:39.749328+0000","last_clean":"2024-08-29T15:20:39.749328+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:20:39.749328+0000","last_undegraded":"2024-08-29T15:20:39.749328+0000","last_fullsized":"2024-08-29T15:20:39.749328+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"244'9628","reported_seq":15415,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:51.781173+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:20:51.781173+0000","last_peered":"2024-08-29T15:20:51.781173+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:20:51.781173+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'2930","ondisk_log_start":"178'2930","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397279232,"num_objects":6062,"num_object_clones":0,"num_object_copies":18186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9628,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1628},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":623,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785117+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:20:50.785117+0000","last_peered":"2024-08-29T15:20:50.785117+0000","last_clean":"2024-08-29T15:20:50.785117+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:20:50.785117+0000","last_undegraded":"2024-08-29T15:20:50.785117+0000","last_fullsized":"2024-08-29T15:20:50.785117+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.714814+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:20:30.714814+0000","last_peered":"2024-08-29T15:20:30.714814+0000","last_clean":"2024-08-29T15:20:30.714814+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:20:30.714814+0000","last_undegraded":"2024-08-29T15:20:30.714814+0000","last_fullsized":"2024-08-29T15:20:30.714814+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":385,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.754294+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:20:39.754294+0000","last_peered":"2024-08-29T15:20:39.754294+0000","last_clean":"2024-08-29T15:20:39.754294+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:20:39.754294+0000","last_undegraded":"2024-08-29T15:20:39.754294+0000","last_fullsized":"2024-08-29T15:20:39.754294+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"244'9727","reported_seq":9902,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:51.821633+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:20:51.821633+0000","last_peered":"2024-08-29T15:20:51.821633+0000","last_clean":"2024-08-29T15:20:51.821633+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:20:51.821633+0000","last_undegraded":"2024-08-29T15:20:51.821633+0000","last_fullsized":"2024-08-29T15:20:51.821633+0000","mapping_epoch":234,"log_start":"178'2997","ondisk_log_start":"178'2997","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392232960,"num_objects":5985,"num_object_clones":0,"num_object_copies":17955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9727,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":180,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748846+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:20:39.748846+0000","last_peered":"2024-08-29T15:20:39.748846+0000","last_clean":"2024-08-29T15:20:39.748846+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:20:39.748846+0000","last_undegraded":"2024-08-29T15:20:39.748846+0000","last_fullsized":"2024-08-29T15:20:39.748846+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718237+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:20:30.718237+0000","last_peered":"2024-08-29T15:20:30.718237+0000","last_clean":"2024-08-29T15:20:30.718237+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:20:30.718237+0000","last_undegraded":"2024-08-29T15:20:30.718237+0000","last_fullsized":"2024-08-29T15:20:30.718237+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748468+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:20:39.748468+0000","last_peered":"2024-08-29T15:20:39.748468+0000","last_clean":"2024-08-29T15:20:39.748468+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:20:39.748468+0000","last_undegraded":"2024-08-29T15:20:39.748468+0000","last_fullsized":"2024-08-29T15:20:39.748468+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"243'9721","reported_seq":9892,"reported_epoch":243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:48.874371+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:20:48.874371+0000","last_peered":"2024-08-29T15:20:48.874371+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:20:48.874371+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"176'1542","ondisk_log_start":"176'1542","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8179,"log_dups_size":0,"ondisk_log_size":8179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400097280,"num_objects":6105,"num_object_clones":0,"num_object_copies":18315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9721,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3682}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":112,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748257+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:20:39.748257+0000","last_peered":"2024-08-29T15:20:39.748257+0000","last_clean":"2024-08-29T15:20:39.748257+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:20:39.748257+0000","last_undegraded":"2024-08-29T15:20:39.748257+0000","last_fullsized":"2024-08-29T15:20:39.748257+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":777,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.754236+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:20:39.754236+0000","last_peered":"2024-08-29T15:20:39.754236+0000","last_clean":"2024-08-29T15:20:39.754236+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:20:39.754236+0000","last_undegraded":"2024-08-29T15:20:39.754236+0000","last_fullsized":"2024-08-29T15:20:39.754236+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":354,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.753752+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:20:39.753752+0000","last_peered":"2024-08-29T15:20:39.753752+0000","last_clean":"2024-08-29T15:20:39.753752+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:20:39.753752+0000","last_undegraded":"2024-08-29T15:20:39.753752+0000","last_fullsized":"2024-08-29T15:20:39.753752+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"243'9671","reported_seq":10398,"reported_epoch":243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:50.080469+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:20:50.080469+0000","last_peered":"2024-08-29T15:20:50.080469+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:20:50.080469+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"177'2680","ondisk_log_start":"177'2680","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393019392,"num_objects":5997,"num_object_clones":0,"num_object_copies":17991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5961,"num_objects_unfound":0,"num_objects_dirty":5997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9671,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3846}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":359,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.754132+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:20:39.754132+0000","last_peered":"2024-08-29T15:20:39.754132+0000","last_clean":"2024-08-29T15:20:39.754132+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:20:39.754132+0000","last_undegraded":"2024-08-29T15:20:39.754132+0000","last_fullsized":"2024-08-29T15:20:39.754132+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":668,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:30.727773+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:20:30.727773+0000","last_peered":"2024-08-29T15:20:30.727773+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:20:30.727773+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:20:30.727773+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":350,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748559+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:20:39.748559+0000","last_peered":"2024-08-29T15:20:39.748559+0000","last_clean":"2024-08-29T15:20:39.748559+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:20:39.748559+0000","last_undegraded":"2024-08-29T15:20:39.748559+0000","last_fullsized":"2024-08-29T15:20:39.748559+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"243'9562","reported_seq":16688,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:49.380338+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:20:49.380338+0000","last_peered":"2024-08-29T15:20:49.380338+0000","last_clean":"2024-08-29T15:20:49.380338+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:20:49.380338+0000","last_undegraded":"2024-08-29T15:20:49.380338+0000","last_fullsized":"2024-08-29T15:20:49.380338+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7163,"log_dups_size":0,"ondisk_log_size":7163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384434176,"num_objects":5866,"num_object_clones":0,"num_object_copies":17598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9562,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":346,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.786558+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:20:50.786558+0000","last_peered":"2024-08-29T15:20:50.786558+0000","last_clean":"2024-08-29T15:20:50.786558+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:20:50.786558+0000","last_undegraded":"2024-08-29T15:20:50.786558+0000","last_fullsized":"2024-08-29T15:20:50.786558+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":967,"reported_epoch":243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:39.755117+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:20:39.755117+0000","last_peered":"2024-08-29T15:20:39.755117+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:20:39.755117+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":343,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785123+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:20:50.785123+0000","last_peered":"2024-08-29T15:20:50.785123+0000","last_clean":"2024-08-29T15:20:50.785123+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:20:50.785123+0000","last_undegraded":"2024-08-29T15:20:50.785123+0000","last_fullsized":"2024-08-29T15:20:50.785123+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"243'9655","reported_seq":11260,"reported_epoch":243,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:47.854034+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:20:47.854034+0000","last_peered":"2024-08-29T15:20:47.854034+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:20:47.854034+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"176'2112","ondisk_log_start":"176'2112","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7543,"log_dups_size":0,"ondisk_log_size":7543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390266880,"num_objects":5955,"num_object_clones":0,"num_object_copies":17865,"num_objects_missing_on_primary":6990,"num_objects_missing":6990,"num_objects_degraded":2313,"num_objects_misplaced":5941,"num_objects_unfound":0,"num_objects_dirty":5955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9655,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":732,"num_bytes_recovered":47841280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":3570},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":340,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748894+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:20:39.748894+0000","last_peered":"2024-08-29T15:20:39.748894+0000","last_clean":"2024-08-29T15:20:39.748894+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:20:39.748894+0000","last_undegraded":"2024-08-29T15:20:39.748894+0000","last_fullsized":"2024-08-29T15:20:39.748894+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":92,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715469+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:20:30.715469+0000","last_peered":"2024-08-29T15:20:30.715469+0000","last_clean":"2024-08-29T15:20:30.715469+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:20:30.715469+0000","last_undegraded":"2024-08-29T15:20:30.715469+0000","last_fullsized":"2024-08-29T15:20:30.715469+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.717925+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:20:30.717925+0000","last_peered":"2024-08-29T15:20:30.717925+0000","last_clean":"2024-08-29T15:20:30.717925+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:20:30.717925+0000","last_undegraded":"2024-08-29T15:20:30.717925+0000","last_fullsized":"2024-08-29T15:20:30.717925+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718489+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:20:30.718489+0000","last_peered":"2024-08-29T15:20:30.718489+0000","last_clean":"2024-08-29T15:20:30.718489+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:20:30.718489+0000","last_undegraded":"2024-08-29T15:20:30.718489+0000","last_fullsized":"2024-08-29T15:20:30.718489+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":18,"reported_epoch":242,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:30.718469+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:20:30.718469+0000","last_peered":"2024-08-29T15:20:30.718469+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:20:30.718469+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:20:30.718469+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":348,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.753647+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:20:39.753647+0000","last_peered":"2024-08-29T15:20:39.753647+0000","last_clean":"2024-08-29T15:20:39.753647+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:20:39.753647+0000","last_undegraded":"2024-08-29T15:20:39.753647+0000","last_fullsized":"2024-08-29T15:20:39.753647+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":101,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:50.787545+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:20:50.787545+0000","last_peered":"2024-08-29T15:20:50.787545+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:20:50.787545+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:20:50.787545+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748140+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:20:39.748140+0000","last_peered":"2024-08-29T15:20:39.748140+0000","last_clean":"2024-08-29T15:20:39.748140+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:20:39.748140+0000","last_undegraded":"2024-08-29T15:20:39.748140+0000","last_fullsized":"2024-08-29T15:20:39.748140+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":310,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784847+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:20:50.784847+0000","last_peered":"2024-08-29T15:20:50.784847+0000","last_clean":"2024-08-29T15:20:50.784847+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:20:50.784847+0000","last_undegraded":"2024-08-29T15:20:50.784847+0000","last_fullsized":"2024-08-29T15:20:50.784847+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":361,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.749554+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:20:39.749554+0000","last_peered":"2024-08-29T15:20:39.749554+0000","last_clean":"2024-08-29T15:20:39.749554+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:20:39.749554+0000","last_undegraded":"2024-08-29T15:20:39.749554+0000","last_fullsized":"2024-08-29T15:20:39.749554+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":362,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784946+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:20:50.784946+0000","last_peered":"2024-08-29T15:20:50.784946+0000","last_clean":"2024-08-29T15:20:50.784946+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:20:50.784946+0000","last_undegraded":"2024-08-29T15:20:50.784946+0000","last_fullsized":"2024-08-29T15:20:50.784946+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":80,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.715561+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:20:30.715561+0000","last_peered":"2024-08-29T15:20:30.715561+0000","last_clean":"2024-08-29T15:20:30.715561+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:20:30.715561+0000","last_undegraded":"2024-08-29T15:20:30.715561+0000","last_fullsized":"2024-08-29T15:20:30.715561+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":216,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.750873+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:20:39.750873+0000","last_peered":"2024-08-29T15:20:39.750873+0000","last_clean":"2024-08-29T15:20:39.750873+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:20:39.750873+0000","last_undegraded":"2024-08-29T15:20:39.750873+0000","last_fullsized":"2024-08-29T15:20:39.750873+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":336,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.749136+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:20:39.749136+0000","last_peered":"2024-08-29T15:20:39.749136+0000","last_clean":"2024-08-29T15:20:39.749136+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:20:39.749136+0000","last_undegraded":"2024-08-29T15:20:39.749136+0000","last_fullsized":"2024-08-29T15:20:39.749136+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":340,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.753554+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:20:39.753554+0000","last_peered":"2024-08-29T15:20:39.753554+0000","last_clean":"2024-08-29T15:20:39.753554+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:20:39.753554+0000","last_undegraded":"2024-08-29T15:20:39.753554+0000","last_fullsized":"2024-08-29T15:20:39.753554+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":346,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.787392+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:20:50.787392+0000","last_peered":"2024-08-29T15:20:50.787392+0000","last_clean":"2024-08-29T15:20:50.787392+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:20:50.787392+0000","last_undegraded":"2024-08-29T15:20:50.787392+0000","last_fullsized":"2024-08-29T15:20:50.787392+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718397+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:20:30.718397+0000","last_peered":"2024-08-29T15:20:30.718397+0000","last_clean":"2024-08-29T15:20:30.718397+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:20:30.718397+0000","last_undegraded":"2024-08-29T15:20:30.718397+0000","last_fullsized":"2024-08-29T15:20:30.718397+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":364,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.754333+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:20:39.754333+0000","last_peered":"2024-08-29T15:20:39.754333+0000","last_clean":"2024-08-29T15:20:39.754333+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:20:39.754333+0000","last_undegraded":"2024-08-29T15:20:39.754333+0000","last_fullsized":"2024-08-29T15:20:39.754333+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":328,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748464+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:20:39.748464+0000","last_peered":"2024-08-29T15:20:39.748464+0000","last_clean":"2024-08-29T15:20:39.748464+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:20:39.748464+0000","last_undegraded":"2024-08-29T15:20:39.748464+0000","last_fullsized":"2024-08-29T15:20:39.748464+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718111+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:20:30.718111+0000","last_peered":"2024-08-29T15:20:30.718111+0000","last_clean":"2024-08-29T15:20:30.718111+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:20:30.718111+0000","last_undegraded":"2024-08-29T15:20:30.718111+0000","last_fullsized":"2024-08-29T15:20:30.718111+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":134,"reported_epoch":243,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:39.748620+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:20:39.748620+0000","last_peered":"2024-08-29T15:20:39.748620+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:20:39.748620+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:20:39.748620+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.718111+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:20:30.718111+0000","last_peered":"2024-08-29T15:20:30.718111+0000","last_clean":"2024-08-29T15:20:30.718111+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:20:30.718111+0000","last_undegraded":"2024-08-29T15:20:30.718111+0000","last_fullsized":"2024-08-29T15:20:30.718111+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":140,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748594+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:20:39.748594+0000","last_peered":"2024-08-29T15:20:39.748594+0000","last_clean":"2024-08-29T15:20:39.748594+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:20:39.748594+0000","last_undegraded":"2024-08-29T15:20:39.748594+0000","last_fullsized":"2024-08-29T15:20:39.748594+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":377,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785208+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:20:50.785208+0000","last_peered":"2024-08-29T15:20:50.785208+0000","last_clean":"2024-08-29T15:20:50.785208+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:20:50.785208+0000","last_undegraded":"2024-08-29T15:20:50.785208+0000","last_fullsized":"2024-08-29T15:20:50.785208+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":378,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785958+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:20:50.785958+0000","last_peered":"2024-08-29T15:20:50.785958+0000","last_clean":"2024-08-29T15:20:50.785958+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:20:50.785958+0000","last_undegraded":"2024-08-29T15:20:50.785958+0000","last_fullsized":"2024-08-29T15:20:50.785958+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":20,"reported_epoch":242,"state":"active+clean","last_fresh":"2024-08-29T15:20:30.717595+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:20:30.717595+0000","last_peered":"2024-08-29T15:20:30.717595+0000","last_clean":"2024-08-29T15:20:30.717595+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:20:30.717595+0000","last_undegraded":"2024-08-29T15:20:30.717595+0000","last_fullsized":"2024-08-29T15:20:30.717595+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":384,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.749897+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:20:39.749897+0000","last_peered":"2024-08-29T15:20:39.749897+0000","last_clean":"2024-08-29T15:20:39.749897+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:20:39.749897+0000","last_undegraded":"2024-08-29T15:20:39.749897+0000","last_fullsized":"2024-08-29T15:20:39.749897+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6280314904,"num_objects":95831,"num_object_clones":0,"num_object_copies":287493,"num_objects_missing_on_primary":6990,"num_objects_missing":6990,"num_objects_degraded":30073,"num_objects_misplaced":17905,"num_objects_unfound":0,"num_objects_dirty":95831,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155019,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14359,"num_bytes_recovered":821035008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18711588864,"data_stored":18711576648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120396,"ondisk_log_size":120396,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3363464,"kb_used_data":3083776,"kb_used_omap":1,"kb_used_meta":279678,"kb_avail":90381688,"statfs":{"total":95995035648,"available":92550848512,"internally_reserved":0,"allocated":3157786624,"data_stored":3156431579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":286390731},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495946,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":473668,"kb_used_data":320316,"kb_used_omap":1,"kb_used_meta":153342,"kb_avail":93271484,"statfs":{"total":95995035648,"available":95509999616,"internally_reserved":0,"allocated":328003584,"data_stored":326652339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":157022663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,1,0,1,2,3,8,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426532,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969016,"kb_used_data":1913904,"kb_used_omap":8,"kb_used_meta":55095,"kb_avail":91776136,"statfs":{"total":95995035648,"available":93978763264,"internally_reserved":0,"allocated":1959837696,"data_stored":1958493917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":56418186},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019224,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2202664,"kb_used_data":1914784,"kb_used_omap":1,"kb_used_meta":287870,"kb_avail":91542488,"statfs":{"total":95995035648,"available":93739507712,"internally_reserved":0,"allocated":1960738816,"data_stored":1959410820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":294779334},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435409,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4092488,"kb_used_data":3830396,"kb_used_omap":4,"kb_used_meta":262075,"kb_avail":89652664,"statfs":{"total":95995035648,"available":91804327936,"internally_reserved":0,"allocated":3922325504,"data_stored":3920978017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":268365065},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903261,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3472328,"kb_used_data":3362816,"kb_used_omap":10,"kb_used_meta":109493,"kb_avail":90272824,"statfs":{"total":95995035648,"available":92439371776,"internally_reserved":0,"allocated":3443523584,"data_stored":3442176033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":112121360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762462,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2605428,"kb_used_data":2348844,"kb_used_omap":7,"kb_used_meta":256568,"kb_avail":91139724,"statfs":{"total":95995035648,"available":93327077376,"internally_reserved":0,"allocated":2405216256,"data_stored":2403875177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":262725986},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925988,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1870524,"kb_used_data":1559796,"kb_used_omap":7,"kb_used_meta":310712,"kb_avail":91874628,"statfs":{"total":95995035648,"available":94079619072,"internally_reserved":0,"allocated":1597231104,"data_stored":1595882804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":318169231},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1587417088,"data_stored":1587413016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2397507584,"data_stored":2397503512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3437559808,"data_stored":3437559808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3910467584,"data_stored":3910467584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1955135488,"data_stored":1955135488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1955401728,"data_stored":1955397656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":323289088,"data_stored":323289088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3144810496,"data_stored":3144810496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:20:54.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:20:54.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:54.660 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:54.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:54.761 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:54.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:54.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:54.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:54.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:54.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:54.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:55.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:55.032 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:55.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:20:55.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:55.156 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:55.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:55.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:55.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:55.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:55.404 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:55.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:55.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:55.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:55.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:55.656 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:55.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:55.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:55.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:55.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:55.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:55.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:56.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:56.029 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:56.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:55 smithi027 ceph-mon[93921]: pgmap v740: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/287244 objects degraded (10.480%); 17868/287244 objects misplaced (6.220%); 461 KiB/s, 7 objects/s recovering 2024-08-29T15:20:56.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:55 smithi027 ceph-mon[98970]: pgmap v740: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/287244 objects degraded (10.480%); 17868/287244 objects misplaced (6.220%); 461 KiB/s, 7 objects/s recovering 2024-08-29T15:20:56.072 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:56.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:56.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:56.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:55 smithi171 ceph-mon[90031]: pgmap v740: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/287244 objects degraded (10.480%); 17868/287244 objects misplaced (6.220%); 461 KiB/s, 7 objects/s recovering 2024-08-29T15:20:56.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:56.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:56.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:56.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:56.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:56.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:56.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:56.636 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:56.652 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:56.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:56.776 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:56.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:56.884 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:56 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/287244 objects degraded (10.480%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:56.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:56 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/287244 objects degraded (10.480%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:56.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:56.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:57.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:57.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:57.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:57.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:57.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:57.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:56 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/287244 objects degraded (10.480%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:20:57.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:57.374 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:57.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:57.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:57.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:57.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:57.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:57.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:57.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:57.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:57.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:57.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:57.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:20:57.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:57.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:57.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:58.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:58.022 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:58.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:57 smithi027 ceph-mon[93921]: pgmap v741: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/286497 objects degraded (10.507%); 17828/286497 objects misplaced (6.223%); 461 KiB/s, 7 objects/s recovering 2024-08-29T15:20:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:20:58.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:57 smithi027 ceph-mon[98970]: pgmap v741: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/286497 objects degraded (10.507%); 17828/286497 objects misplaced (6.223%); 461 KiB/s, 7 objects/s recovering 2024-08-29T15:20:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:20:58.123 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:58.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.148 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:58.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:58.174 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:57 smithi171 ceph-mon[90031]: pgmap v741: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/286497 objects degraded (10.507%); 17828/286497 objects misplaced (6.223%); 461 KiB/s, 7 objects/s recovering 2024-08-29T15:20:58.174 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:20:58.249 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:58.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:58.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:58.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.399 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:58.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:58.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:20:58.523 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:58.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:20:58.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:58.629 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:20:58.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:58.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:20:58.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:58.775 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:58.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:20:58.899 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:58.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:58.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:20:59.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:59.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:59.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:20:59] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.51.0" 2024-08-29T15:20:59.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:20:59.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:20:59.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:20:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:20:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:59.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:20:59.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:59.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:59.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.421 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:59.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:20:59.422 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:20:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:20:59.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:20:59.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:20:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:20:59.565 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:20:59.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:20:59.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:20:59.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:20:59.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:59.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:20:59.857 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:20:59.857 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/285831 objects degraded (10.531%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1041,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":244,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":95277,"data_bytes":6246963197,"bytes_used":20424040448,"bytes_avail":747536244736,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":285831,"degraded_ratio":0.10531397923948066,"misplaced_objects":17759,"misplaced_total":285831,"misplaced_ratio":0.062131119437709693,"recovering_objects_per_sec":7,"recovering_bytes_per_sec":478264,"recovering_keys_per_sec":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":118},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:20:59.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:20:59.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:20:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:20:59.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:00.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:00.028 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:00.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:00.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:00.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:00.233 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:00 smithi171 ceph-mon[90031]: pgmap v742: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/285831 objects degraded (10.531%); 17759/285831 objects misplaced (6.213%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:00.234 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/334049260' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:21:00.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:00.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:00.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:00.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:00.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:00.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:00.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:00.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:00 smithi027 ceph-mon[93921]: pgmap v742: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/285831 objects degraded (10.531%); 17759/285831 objects misplaced (6.213%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:00.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/334049260' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:21:00.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:00 smithi027 ceph-mon[98970]: pgmap v742: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/285831 objects degraded (10.531%); 17759/285831 objects misplaced (6.213%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:00.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/334049260' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:21:00.613 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:21:00.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:21:00.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:00.639 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:00.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:00.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:00.766 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:00.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:00.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:00.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:00.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:00.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:01.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:01.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:01.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:01.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:01.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:01.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:01.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:01.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:01.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:01.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:01.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:01.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:01.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:01.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:01.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(6.1 218/217 [PushOp(6:82846163:::benchmark_data_smithi027_169445_object87450:head, version: 183'5518, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:82846163:::benchmark_data_smithi027_169445_object87450:head@183'5518, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:34.978950+0000", 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 146.12872328200001, 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010037341, 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.687 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.978950+0000", 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.978950+0000", 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.978951+0000", 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5600000000000001e-07 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.978988+0000", 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7209999999999998e-05 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.979009+0000", 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1520999999999999e-05 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.979028+0000", 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9062000000000001e-05 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.979389+0000", 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00036041099999999999 2024-08-29T15:21:01.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.979402+0000", 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3254999999999999e-05 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:34.988988+0000", 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095855260000000005 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.691 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144449 6.8 e223/218 6:17168afb:::benchmark_data_smithi027_169445_object19022:head v 223'9041, mlcod=181'5379)", 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:54.893576+0000", 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 126.214097955, 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019573889000000001, 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893576+0000", 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893576+0000", 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893577+0000", 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5300000000000002e-07 2024-08-29T15:21:01.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893587+0000", 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0383e-05 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893601+0000", 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3740000000000001e-05 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893609+0000", 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4679999999999992e-06 2024-08-29T15:21:01.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893686+0000", 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6874999999999998e-05 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.893710+0000", 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3533999999999999e-05 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.913115+0000", 2024-08-29T15:21:01.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019405338000000001 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.913149+0000", 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4598000000000003e-05 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144450 6.0 e223/218 6:0474b498:::benchmark_data_smithi027_169445_object19023:head v 223'9084, mlcod=181'5490)", 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:54.895749+0000", 2024-08-29T15:21:01.696 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 126.21192444099999, 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017362994999999999, 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895749+0000", 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895749+0000", 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895750+0000", 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.079e-06 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895758+0000", 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0900000000000005e-06 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895770+0000", 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2089999999999999e-05 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895778+0000", 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8269999999999998e-06 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895909+0000", 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000130445 2024-08-29T15:21:01.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.895935+0000", 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5987999999999999e-05 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.913059+0000", 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017124385999999998 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:54.913112+0000", 2024-08-29T15:21:01.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3090000000000002e-05 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144632 6.e e223/217 6:77447d2f:::benchmark_data_smithi027_169445_object19205:head v 223'9110, mlcod=181'5430)", 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:55.957439+0000", 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 125.150234258, 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01369048, 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.701 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957439+0000", 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957439+0000", 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957440+0000", 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2099999999999995e-07 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957448+0000", 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0600000000000008e-06 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957462+0000", 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4063000000000001e-05 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957470+0000", 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0819999999999999e-06 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957537+0000", 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6352000000000004e-05 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.957555+0000", 2024-08-29T15:21:01.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8610999999999999e-05 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.971091+0000", 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013536166000000001 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:55.971130+0000", 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8325000000000003e-05 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.705 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144717 6.0 e223/218 6:00ffb3a1:::benchmark_data_smithi027_169445_object19290:head v 223'9106, mlcod=181'5490)", 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:57.025311+0000", 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 124.08236252499999, 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01193761, 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025311+0000", 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025311+0000", 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025312+0000", 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.164e-06 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025321+0000", 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7029999999999995e-06 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025335+0000", 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4412e-05 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025344+0000", 2024-08-29T15:21:01.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3100000000000001e-06 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025417+0000", 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3578000000000006e-05 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.025441+0000", 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3451999999999998e-05 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.037217+0000", 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011776316 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.037249+0000", 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1674999999999997e-05 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144462 6.3 e223/218 6:c5ad7939:::benchmark_data_smithi027_169445_object19035:head v 223'9013, mlcod=0'0)", 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:57.557530+0000", 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 123.550143737, 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010358368, 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557530+0000", 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557530+0000", 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557530+0000", 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6499999999999999e-07 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557547+0000", 2024-08-29T15:21:01.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6820000000000002e-05 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557557+0000", 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0115999999999999e-05 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557562+0000", 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0880000000000002e-06 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557610+0000", 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7763999999999999e-05 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.557634+0000", 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4219999999999999e-05 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.567849+0000", 2024-08-29T15:21:01.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010214875999999999 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.567888+0000", 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8819000000000002e-05 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144805 6.e e223/217 6:7ba73355:::benchmark_data_smithi027_169445_object19378:head v 223'9124, mlcod=181'5430)", 2024-08-29T15:21:01.715 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:57.765199+0000", 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 123.34247487899999, 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010482116, 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765199+0000", 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765199+0000", 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765200+0000", 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4500000000000001e-06 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765208+0000", 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5310000000000004e-06 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765227+0000", 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9191999999999999e-05 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765236+0000", 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8899999999999996e-06 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765293+0000", 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7144e-05 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.765316+0000", 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3377999999999999e-05 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.775623+0000", 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01030669 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.775681+0000", 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7840999999999997e-05 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144808 6.0 e223/218 6:037462f8:::benchmark_data_smithi027_169445_object19381:head v 223'9116, mlcod=181'5490)", 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:57.773301+0000", 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 123.334372828, 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011368478, 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.720 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773301+0000", 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773301+0000", 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773302+0000", 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.037e-06 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773310+0000", 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8469999999999996e-06 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773327+0000", 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7351e-05 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773336+0000", 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7080000000000003e-06 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773465+0000", 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129195 2024-08-29T15:21:01.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.773487+0000", 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2133999999999998e-05 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.784639+0000", 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011151774 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:57.784669+0000", 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0431999999999999e-05 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:144938 6.f e223/217 6:f3c71cef:::benchmark_data_smithi027_169445_object19511:head v 223'9135, mlcod=181'5456)", 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:18:59.091327+0000", 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.01634628399999, 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012698884000000001, 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.725 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091327+0000", 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091327+0000", 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091328+0000", 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.37e-07 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091334+0000", 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9540000000000001e-06 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091346+0000", 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1691e-05 2024-08-29T15:21:01.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091352+0000", 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7360000000000004e-06 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091400+0000", 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8223999999999998e-05 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.091418+0000", 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7584e-05 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.103965+0000", 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012546816000000001 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:18:59.104026+0000", 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1241999999999994e-05 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.729 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:145239 6.1 e225/217 6:8dafc913:::benchmark_data_smithi027_169445_object19812:head v 225'9057, mlcod=185'5683)", 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:19:01.883806+0000", 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 119.22386770200001, 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019690684, 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883806+0000", 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883806+0000", 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883807+0000", 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.159e-06 2024-08-29T15:21:01.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883816+0000", 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6729999999999998e-06 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883839+0000", 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3794999999999998e-05 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883849+0000", 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4250000000000004e-06 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883928+0000", 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9283000000000006e-05 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.883956+0000", 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7548000000000001e-05 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.903435+0000", 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019479241000000001 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.903497+0000", 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.156e-05 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:145266 6.2 e225/224 6:4cfcef6f:::benchmark_data_smithi027_169445_object19839:head v 225'8966, mlcod=225'8964)", 2024-08-29T15:21:01.734 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:19:01.983326+0000", 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 119.12434706099999, 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014924048000000001, 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983326+0000", 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983326+0000", 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983327+0000", 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9599999999999998e-07 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983335+0000", 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5809999999999999e-06 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983350+0000", 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5333e-05 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983358+0000", 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7570000000000004e-06 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983424+0000", 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5747999999999995e-05 2024-08-29T15:21:01.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.983451+0000", 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7651e-05 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.998209+0000", 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014757977 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:01.998251+0000", 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1105e-05 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:145895 6.0 e227/218 6:009adf15:::benchmark_data_smithi027_169445_object20468:head v 227'9188, mlcod=181'5490)", 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:19:11.169629+0000", 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 109.93804458699999, 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011622954, 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.739 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169629+0000", 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169629+0000", 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169630+0000", 2024-08-29T15:21:01.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9400000000000005e-07 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169636+0000", 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.251e-06 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169649+0000", 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3669000000000001e-05 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169656+0000", 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8870000000000003e-06 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169743+0000", 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6706000000000006e-05 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.169774+0000", 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0963000000000002e-05 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.181210+0000", 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011436317999999999 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:11.181252+0000", 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1566000000000001e-05 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:148656 6.e e235/234 6:72a7dc70:::benchmark_data_smithi027_169445_object23229:head v 235'9366, mlcod=181'5430)", 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:19:48.149765+0000", 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 72.957908621000001, 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012017451, 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149765+0000", 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149765+0000", 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149766+0000", 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0300000000000001e-06 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149773+0000", 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.311e-06 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149788+0000", 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5079000000000001e-05 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149796+0000", 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4390000000000003e-06 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149873+0000", 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7616999999999999e-05 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.149915+0000", 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1306000000000001e-05 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.161744+0000", 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011829733 2024-08-29T15:21:01.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:48.161782+0000", 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7935999999999997e-05 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:149007 6.0 e235/218 6:04c4a7ee:::benchmark_data_smithi027_169445_object23580:head v 235'9395, mlcod=181'5490)", 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:19:52.326734+0000", 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 68.780939501999995, 2024-08-29T15:21:01.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011253000000000001, 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326734+0000", 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326734+0000", 2024-08-29T15:21:01.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326735+0000", 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6899999999999996e-07 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326743+0000", 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5939999999999996e-06 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326756+0000", 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3628999999999999e-05 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326764+0000", 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4560000000000003e-06 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326833+0000", 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9474999999999995e-05 2024-08-29T15:21:01.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.326859+0000", 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5694000000000002e-05 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.337953+0000", 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011093743 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:52.337987+0000", 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4440000000000002e-05 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:149254 6.0 e235/218 6:0d3a6c04:::benchmark_data_smithi027_169445_object23827:head v 235'9407, mlcod=181'5490)", 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:19:53.960230+0000", 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 67.147443553000002, 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010213292000000001, 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.753 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.960230+0000", 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.960230+0000", 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.960231+0000", 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.124e-06 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.960240+0000", 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2909999999999996e-06 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.960258+0000", 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7278e-05 2024-08-29T15:21:01.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.960266+0000", 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6799999999999999e-06 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.963079+0000", 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028131229999999998 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.963113+0000", 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3272999999999998e-05 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.970391+0000", 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0072779840000000004 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:53.970443+0000", 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2539e-05 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.757 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:149661 6.8 e236/218 6:1f471517:::benchmark_data_smithi027_169445_object24234:head v 236'9354, mlcod=181'5379)", 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:19:59.425544+0000", 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 61.682129578000001, 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012784138, 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425544+0000", 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425544+0000", 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425545+0000", 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3179999999999999e-06 2024-08-29T15:21:01.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425554+0000", 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6819999999999992e-06 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425575+0000", 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1220999999999999e-05 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425585+0000", 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6460000000000005e-06 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425711+0000", 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012657 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.425738+0000", 2024-08-29T15:21:01.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6514000000000001e-05 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.438268+0000", 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012529784 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:19:59.438328+0000", 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0402999999999997e-05 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.762 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:152702 6.1 e241/217 6:89cd13f2:::benchmark_data_smithi027_169445_object27275:head v 241'9540, mlcod=185'5683)", 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:20:30.079445+0000", 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 31.028228273, 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099454169999999998, 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079445+0000", 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079445+0000", 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079446+0000", 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.85e-07 2024-08-29T15:21:01.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079453+0000", 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4629999999999998e-06 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079465+0000", 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2869e-05 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079474+0000", 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5949999999999999e-06 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079544+0000", 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9952999999999999e-05 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.079577+0000", 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3331000000000003e-05 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.089355+0000", 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0097774790000000004 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.089391+0000", 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5942000000000002e-05 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.767 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:152711 6.e e241/234 6:7e045d56:::benchmark_data_smithi027_169445_object27284:head v 241'9621, mlcod=181'5430)", 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:20:30.095024+0000", 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 31.012649144000001, 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011003921, 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095024+0000", 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095024+0000", 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095025+0000", 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4700000000000001e-07 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095031+0000", 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0360000000000003e-06 2024-08-29T15:21:01.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095045+0000", 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4138000000000001e-05 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095051+0000", 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0809999999999999e-06 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095108+0000", 2024-08-29T15:21:01.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6833000000000003e-05 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.095130+0000", 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2126000000000001e-05 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.105974+0000", 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010843439 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:30.106028+0000", 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4521000000000002e-05 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:154248 6.0 e243/218 6:0eaee86a:::benchmark_data_smithi027_169445_object28821:head v 243'9691, mlcod=181'5490)", 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:20:44.341496+0000", 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 16.766177069000001, 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096511840000000002, 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.772 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341496+0000", 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341496+0000", 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341497+0000", 2024-08-29T15:21:01.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6499999999999998e-07 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341503+0000", 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0610000000000001e-06 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341517+0000", 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3844e-05 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341525+0000", 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6860000000000006e-06 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341582+0000", 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7021999999999997e-05 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.341605+0000", 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2684000000000002e-05 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.351105+0000", 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095006740000000006 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:44.351148+0000", 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2348000000000002e-05 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:155872 6.e e244/234 6:73f77a08:::benchmark_data_smithi027_169445_object30445:head v 244'9818, mlcod=181'5430)", 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:20:58.750984+0000", 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 2.3566892319999999, 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010667912999999999, 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.750984+0000", 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.750984+0000", 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.750985+0000", 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0800000000000003e-07 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.750991+0000", 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.553e-06 2024-08-29T15:21:01.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.751001+0000", 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414000000000001e-05 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.751009+0000", 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7300000000000005e-06 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.751086+0000", 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7058999999999995e-05 2024-08-29T15:21:01.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.751113+0000", 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7002999999999999e-05 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.761610+0000", 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010497348 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:20:58.761652+0000", 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1897999999999998e-05 2024-08-29T15:21:01.780 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.781 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.781 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.781 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:21:01.781 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:21:01.781 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:21:01.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:01.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:01.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:01.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:01.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:01.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:02.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:02.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:02.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:02.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:02.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:02.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:02.303 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:02.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:02.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:02 smithi027 ceph-mon[98970]: pgmap v743: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 30102/285441 objects degraded (10.546%); 17759/285441 objects misplaced (6.222%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:02.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/285441 objects degraded (10.546%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:02.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:02 smithi027 ceph-mon[93921]: pgmap v743: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 30102/285441 objects degraded (10.546%); 17759/285441 objects misplaced (6.222%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:02.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/285441 objects degraded (10.546%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:02.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:02.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:02.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:02 smithi171 ceph-mon[90031]: pgmap v743: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 30102/285441 objects degraded (10.546%); 17759/285441 objects misplaced (6.222%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/285441 objects degraded (10.546%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:02.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:02.555 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:02.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:02.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:02.680 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:02.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:02.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:02.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:02.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:02.933 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:02.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:02.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:03.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:03.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:03.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:03.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:03.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:03.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:03.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:21:03.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:03.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:03.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:03.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:03.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:03.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:03.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:03.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:03.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:03.680 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:03.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:03.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:03.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:03.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:03.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:03.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:03.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:04.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:04.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:04.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:04.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:04.278 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:04.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:04 smithi027 ceph-mon[93921]: pgmap v744: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 30102/284274 objects degraded (10.589%); 17695/284274 objects misplaced (6.225%); 397 KiB/s, 6 objects/s recovering 2024-08-29T15:21:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:04 smithi027 ceph-mon[98970]: pgmap v744: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 30102/284274 objects degraded (10.589%); 17695/284274 objects misplaced (6.225%); 397 KiB/s, 6 objects/s recovering 2024-08-29T15:21:04.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:04.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:04 smithi171 ceph-mon[90031]: pgmap v744: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 30102/284274 objects degraded (10.589%); 17695/284274 objects misplaced (6.225%); 397 KiB/s, 6 objects/s recovering 2024-08-29T15:21:04.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:04.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:04.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:04.652 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:04.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:04.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:04.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:04.799 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:04.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:04.901 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:04.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:04.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:04.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:05.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:05.053 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:05.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:05.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:05.278 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:05.299 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:05.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:05.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:05.425 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:05.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:05.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:05.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:05.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:05.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:05.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:05.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:05.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:05.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:05.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:21:05.973 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:21:05.981 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:05.983 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:21:05.983 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:21:05.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:05.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:05.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:06.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:21:06.010 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:21:06.028 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:21:06.028 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:21:06.047 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:21:06.047 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:21:06.064 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:21:06.065 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:21:06.082 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:21:06.082 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:21:06.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:06.100 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:21:06.100 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:21:06.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:06.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:06.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:06.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:06.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:06.370 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:06.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:06.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:06 smithi171 ceph-mon[90031]: pgmap v745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 30102/283944 objects degraded (10.601%); 17662/283944 objects misplaced (6.220%); 560 KiB/s, 8 objects/s recovering 2024-08-29T15:21:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/720553270' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:06.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:06 smithi027 ceph-mon[93921]: pgmap v745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 30102/283944 objects degraded (10.601%); 17662/283944 objects misplaced (6.220%); 560 KiB/s, 8 objects/s recovering 2024-08-29T15:21:06.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/720553270' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:06.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:06 smithi027 ceph-mon[98970]: pgmap v745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 30102/283944 objects degraded (10.601%); 17662/283944 objects misplaced (6.220%); 560 KiB/s, 8 objects/s recovering 2024-08-29T15:21:06.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/720553270' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:06.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:06.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:06.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:06.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:06.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:06.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:06.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:06.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:06.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:06.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:06.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:07.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:07.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:07.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:07.244 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:07.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:07.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:07.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:07.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:07.387 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/283944 objects degraded (10.601%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:07.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/283944 objects degraded (10.601%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:07.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:07.492 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:07.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/283944 objects degraded (10.601%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:07.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:07.595 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:07.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:07.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:07.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:07.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:07.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:07.973 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:07.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:07.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:08.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:08.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:08.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:08.196 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:21:08.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:08.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:08.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:08.313 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:08 smithi027 ceph-mon[98970]: pgmap v746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 30102/283065 objects degraded (10.634%); 17618/283065 objects misplaced (6.224%); 373 KiB/s, 5 objects/s recovering 2024-08-29T15:21:08.313 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3980565217' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:08.313 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:08 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:08.314 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:08 smithi027 ceph-mon[93921]: pgmap v746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 30102/283065 objects degraded (10.634%); 17618/283065 objects misplaced (6.224%); 373 KiB/s, 5 objects/s recovering 2024-08-29T15:21:08.314 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3980565217' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:08.314 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:08 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:08.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:08.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:08.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:08.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:08.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:08 smithi171 ceph-mon[90031]: pgmap v746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 30102/283065 objects degraded (10.634%); 17618/283065 objects misplaced (6.224%); 373 KiB/s, 5 objects/s recovering 2024-08-29T15:21:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3980565217' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:08 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:08.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:08.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:08.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:08.740 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:08.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:08.864 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:08.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:08.989 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:08.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:08.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:09.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:09.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.115 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:09.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:09.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:21:09.133 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:21:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:21:09] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.51.0" 2024-08-29T15:21:09.216 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:09.239 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:09.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:09.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:09.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:09.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:09.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:09.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:09 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:21:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:09 smithi171 ceph-mon[90031]: osdmap e245: 8 total, 8 up, 8 in 2024-08-29T15:21:09.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:09 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:09.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:21:09.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:09 smithi027 ceph-mon[98970]: osdmap e245: 8 total, 8 up, 8 in 2024-08-29T15:21:09.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:09 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:09.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:21:09.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:09 smithi027 ceph-mon[93921]: osdmap e245: 8 total, 8 up, 8 in 2024-08-29T15:21:09.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:09.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:09.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:09.721 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:09.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:09.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:09.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:09.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:09.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:09.944 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:09.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:09.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:09.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:10.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:10.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:10.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:10.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:10.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.242 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:10.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:10.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:10.365 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:10.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:10.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:10.490 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:10 smithi171 ceph-mon[90031]: pgmap v748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 30102/282303 objects degraded (10.663%); 17558/282303 objects misplaced (6.220%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:10 smithi027 ceph-mon[93921]: pgmap v748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 30102/282303 objects degraded (10.663%); 17558/282303 objects misplaced (6.220%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:10 smithi027 ceph-mon[98970]: pgmap v748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 30102/282303 objects degraded (10.663%); 17558/282303 objects misplaced (6.220%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:10.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:10.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:10.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:10.737 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:10.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:10.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:10.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:10.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:21:10.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:10.984 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:10.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:10.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:11.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:11.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:11.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:11.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:11.235 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:11.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:11.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:11.359 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:11.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:11.438 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:21:11.438 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:21:11.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:11.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:11.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:11.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:11.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:11.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:11.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:11.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:11.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:11.935 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:21:11.936 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:21:11.959 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:11.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:11.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:12.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:12.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:12.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:12.208 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:12.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:12.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:12.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:12.332 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:12.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:12.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:12.455 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:12.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:12.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:12.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:12 smithi171 ceph-mon[90031]: pgmap v749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/281877 objects degraded (10.679%); 17558/281877 objects misplaced (6.229%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/281877 objects degraded (10.679%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:12.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:12 smithi027 ceph-mon[98970]: pgmap v749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/281877 objects degraded (10.679%); 17558/281877 objects misplaced (6.229%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:12.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/281877 objects degraded (10.679%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:12.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:12 smithi027 ceph-mon[93921]: pgmap v749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/281877 objects degraded (10.679%); 17558/281877 objects misplaced (6.229%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/281877 objects degraded (10.679%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:12.579 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:12.600 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:12.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:12.694 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":749,"stamp":"2024-08-29T15:21:10.551484+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6160586749,"num_objects":93959,"num_object_clones":0,"num_object_copies":281877,"num_objects_missing_on_primary":6854,"num_objects_missing":6854,"num_objects_degraded":30102,"num_objects_misplaced":17558,"num_objects_unfound":0,"num_objects_dirty":93959,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":158042,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14612,"num_bytes_recovered":836439245,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122278,"ondisk_log_size":122278,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19722604,"kb_used_data":17956648,"kb_used_omap":43,"kb_used_meta":1765844,"kb_avail":730238612,"statfs":{"total":767960285184,"available":747764338688,"internally_reserved":0,"allocated":18387607552,"data_stored":18376783886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1808224748},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,0,0,0,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-77856768,"num_objects":-1188,"num_object_clones":0,"num_object_copies":-3564,"num_objects_missing_on_primary":-70,"num_objects_missing":-70,"num_objects_degraded":0,"num_objects_misplaced":-201,"num_objects_unfound":0,"num_objects_dirty":-1188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1188,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":71,"num_bytes_recovered":4653056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002696"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":372,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785749+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:20:50.785749+0000","last_peered":"2024-08-29T15:20:50.785749+0000","last_clean":"2024-08-29T15:20:50.785749+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:20:50.785749+0000","last_undegraded":"2024-08-29T15:20:50.785749+0000","last_fullsized":"2024-08-29T15:20:50.785749+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":21,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:50.783912+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:20:50.783912+0000","last_peered":"2024-08-29T15:20:50.783912+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:20:50.783912+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:20:50.783912+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748322+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:20:39.748322+0000","last_peered":"2024-08-29T15:20:39.748322+0000","last_clean":"2024-08-29T15:20:39.748322+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:20:39.748322+0000","last_undegraded":"2024-08-29T15:20:39.748322+0000","last_fullsized":"2024-08-29T15:20:39.748322+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":339,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.511015+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:21:08.511015+0000","last_peered":"2024-08-29T15:21:08.511015+0000","last_clean":"2024-08-29T15:21:08.511015+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:21:08.511015+0000","last_undegraded":"2024-08-29T15:21:08.511015+0000","last_fullsized":"2024-08-29T15:21:08.511015+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":339,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508057+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:21:08.508057+0000","last_peered":"2024-08-29T15:21:08.508057+0000","last_clean":"2024-08-29T15:21:08.508057+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:21:08.508057+0000","last_undegraded":"2024-08-29T15:21:08.508057+0000","last_fullsized":"2024-08-29T15:21:08.508057+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784701+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:20:50.784701+0000","last_peered":"2024-08-29T15:20:50.784701+0000","last_clean":"2024-08-29T15:20:50.784701+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:20:50.784701+0000","last_undegraded":"2024-08-29T15:20:50.784701+0000","last_fullsized":"2024-08-29T15:20:50.784701+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":149,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.788738+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:20:50.788738+0000","last_peered":"2024-08-29T15:20:50.788738+0000","last_clean":"2024-08-29T15:20:50.788738+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:20:50.788738+0000","last_undegraded":"2024-08-29T15:20:50.788738+0000","last_fullsized":"2024-08-29T15:20:50.788738+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":111,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.757222+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:20:39.757222+0000","last_peered":"2024-08-29T15:20:39.757222+0000","last_clean":"2024-08-29T15:20:39.757222+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:20:39.757222+0000","last_undegraded":"2024-08-29T15:20:39.757222+0000","last_fullsized":"2024-08-29T15:20:39.757222+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"244'9960","reported_seq":17981,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:05.872909+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:21:05.872909+0000","last_peered":"2024-08-29T15:21:05.872909+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:21:05.872909+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9960,"log_dups_size":0,"ondisk_log_size":9960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390070272,"num_objects":5952,"num_object_clones":0,"num_object_copies":17856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9960,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":3027}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":20,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:50.784762+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:20:50.784762+0000","last_peered":"2024-08-29T15:20:50.784762+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:20:50.784762+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:20:50.784762+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.747575+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:20:39.747575+0000","last_peered":"2024-08-29T15:20:39.747575+0000","last_clean":"2024-08-29T15:20:39.747575+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:20:39.747575+0000","last_undegraded":"2024-08-29T15:20:39.747575+0000","last_fullsized":"2024-08-29T15:20:39.747575+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"244'9722","reported_seq":9887,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:21:06.325484+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:21:06.325484+0000","last_peered":"2024-08-29T15:21:06.325484+0000","last_clean":"2024-08-29T15:21:06.325484+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:21:06.325484+0000","last_undegraded":"2024-08-29T15:21:06.325484+0000","last_fullsized":"2024-08-29T15:21:06.325484+0000","mapping_epoch":168,"log_start":"178'3028","ondisk_log_start":"178'3028","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381943808,"num_objects":5828,"num_object_clones":0,"num_object_copies":17484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9722,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":20,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:50.784987+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:20:50.784987+0000","last_peered":"2024-08-29T15:20:50.784987+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:20:50.784987+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:20:50.784987+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784453+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:20:50.784453+0000","last_peered":"2024-08-29T15:20:50.784453+0000","last_clean":"2024-08-29T15:20:50.784453+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:20:50.784453+0000","last_undegraded":"2024-08-29T15:20:50.784453+0000","last_fullsized":"2024-08-29T15:20:50.784453+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"244'9831","reported_seq":9966,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:06.601526+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:21:06.601526+0000","last_peered":"2024-08-29T15:21:06.601526+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:21:06.601526+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'3074","ondisk_log_start":"178'3074","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385417216,"num_objects":5881,"num_object_clones":0,"num_object_copies":17643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9831,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3404}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":156,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.507557+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:21:08.507557+0000","last_peered":"2024-08-29T15:21:08.507557+0000","last_clean":"2024-08-29T15:21:08.507557+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:21:08.507557+0000","last_undegraded":"2024-08-29T15:21:08.507557+0000","last_fullsized":"2024-08-29T15:21:08.507557+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":66,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.761138+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:20:39.761138+0000","last_peered":"2024-08-29T15:20:39.761138+0000","last_clean":"2024-08-29T15:20:39.761138+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:20:39.761138+0000","last_undegraded":"2024-08-29T15:20:39.761138+0000","last_fullsized":"2024-08-29T15:20:39.761138+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"245'9938","reported_seq":15325,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:08.953886+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:21:08.953886+0000","last_peered":"2024-08-29T15:21:08.953886+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:21:08.953886+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'1762","ondisk_log_start":"176'1762","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8176,"log_dups_size":0,"ondisk_log_size":8176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382730240,"num_objects":5840,"num_object_clones":0,"num_object_copies":17520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9938,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5840}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":128,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748877+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:20:39.748877+0000","last_peered":"2024-08-29T15:20:39.748877+0000","last_clean":"2024-08-29T15:20:39.748877+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:20:39.748877+0000","last_undegraded":"2024-08-29T15:20:39.748877+0000","last_fullsized":"2024-08-29T15:20:39.748877+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":354,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508093+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:21:08.508093+0000","last_peered":"2024-08-29T15:21:08.508093+0000","last_clean":"2024-08-29T15:21:08.508093+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:21:08.508093+0000","last_undegraded":"2024-08-29T15:21:08.508093+0000","last_fullsized":"2024-08-29T15:21:08.508093+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"245'9897","reported_seq":10027,"reported_epoch":245,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:08.542370+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:21:08.542370+0000","last_peered":"2024-08-29T15:21:08.542370+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:21:08.542370+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:21:08.542370+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387776512,"num_objects":5917,"num_object_clones":0,"num_object_copies":17751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9897,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3440},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":346,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785315+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:20:50.785315+0000","last_peered":"2024-08-29T15:20:50.785315+0000","last_clean":"2024-08-29T15:20:50.785315+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:20:50.785315+0000","last_undegraded":"2024-08-29T15:20:50.785315+0000","last_fullsized":"2024-08-29T15:20:50.785315+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.783728+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:20:50.783728+0000","last_peered":"2024-08-29T15:20:50.783728+0000","last_clean":"2024-08-29T15:20:50.783728+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:20:50.783728+0000","last_undegraded":"2024-08-29T15:20:50.783728+0000","last_fullsized":"2024-08-29T15:20:50.783728+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"244'9872","reported_seq":10054,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:06.469310+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:21:06.469310+0000","last_peered":"2024-08-29T15:21:06.469310+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:21:06.469310+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:21:06.469310+0000","mapping_epoch":234,"log_start":"178'3168","ondisk_log_start":"178'3168","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391249920,"num_objects":5970,"num_object_clones":0,"num_object_copies":17910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9872,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3505},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":84,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.761069+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:20:39.761069+0000","last_peered":"2024-08-29T15:20:39.761069+0000","last_clean":"2024-08-29T15:20:39.761069+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:20:39.761069+0000","last_undegraded":"2024-08-29T15:20:39.761069+0000","last_fullsized":"2024-08-29T15:20:39.761069+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784758+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:20:50.784758+0000","last_peered":"2024-08-29T15:20:50.784758+0000","last_clean":"2024-08-29T15:20:50.784758+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:20:50.784758+0000","last_undegraded":"2024-08-29T15:20:50.784758+0000","last_fullsized":"2024-08-29T15:20:50.784758+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"244'9740","reported_seq":10106,"reported_epoch":244,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:21:06.644114+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:21:06.644114+0000","last_peered":"2024-08-29T15:21:06.644114+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:21:06.644114+0000","last_undegraded":"2024-08-29T15:21:06.644114+0000","last_fullsized":"2024-08-29T15:21:06.644114+0000","mapping_epoch":202,"log_start":"178'3067","ondisk_log_start":"178'3067","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387842048,"num_objects":5918,"num_object_clones":0,"num_object_copies":17754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5918,"num_objects_unfound":0,"num_objects_dirty":5918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9740,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5918}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784533+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:20:50.784533+0000","last_peered":"2024-08-29T15:20:50.784533+0000","last_clean":"2024-08-29T15:20:50.784533+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:20:50.784533+0000","last_undegraded":"2024-08-29T15:20:50.784533+0000","last_fullsized":"2024-08-29T15:20:50.784533+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":228,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785190+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:20:50.785190+0000","last_peered":"2024-08-29T15:20:50.785190+0000","last_clean":"2024-08-29T15:20:50.785190+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:20:50.785190+0000","last_undegraded":"2024-08-29T15:20:50.785190+0000","last_fullsized":"2024-08-29T15:20:50.785190+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"244'9837","reported_seq":9960,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:21:05.626064+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:21:05.626064+0000","last_peered":"2024-08-29T15:21:05.626064+0000","last_clean":"2024-08-29T15:21:05.626064+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:21:05.626064+0000","last_undegraded":"2024-08-29T15:21:05.626064+0000","last_fullsized":"2024-08-29T15:21:05.626064+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8559,"log_dups_size":0,"ondisk_log_size":8559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388104216,"num_objects":5923,"num_object_clones":0,"num_object_copies":17769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5923,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9837,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":349,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785531+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:20:50.785531+0000","last_peered":"2024-08-29T15:20:50.785531+0000","last_clean":"2024-08-29T15:20:50.785531+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:20:50.785531+0000","last_undegraded":"2024-08-29T15:20:50.785531+0000","last_fullsized":"2024-08-29T15:20:50.785531+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":341,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.788785+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:20:50.788785+0000","last_peered":"2024-08-29T15:20:50.788785+0000","last_clean":"2024-08-29T15:20:50.788785+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:20:50.788785+0000","last_undegraded":"2024-08-29T15:20:50.788785+0000","last_fullsized":"2024-08-29T15:20:50.788785+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"244'9504","reported_seq":9675,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:21:05.758148+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:21:05.758148+0000","last_peered":"2024-08-29T15:21:05.758148+0000","last_clean":"2024-08-29T15:21:05.758148+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:21:05.758148+0000","last_undegraded":"2024-08-29T15:21:05.758148+0000","last_fullsized":"2024-08-29T15:21:05.758148+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9504,"log_dups_size":0,"ondisk_log_size":9504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373817344,"num_objects":5704,"num_object_clones":0,"num_object_copies":17112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9504,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":442,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.507811+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:21:08.507811+0000","last_peered":"2024-08-29T15:21:08.507811+0000","last_clean":"2024-08-29T15:21:08.507811+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:21:08.507811+0000","last_undegraded":"2024-08-29T15:21:08.507811+0000","last_fullsized":"2024-08-29T15:21:08.507811+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":712,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.747841+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:20:39.747841+0000","last_peered":"2024-08-29T15:20:39.747841+0000","last_clean":"2024-08-29T15:20:39.747841+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:20:39.747841+0000","last_undegraded":"2024-08-29T15:20:39.747841+0000","last_fullsized":"2024-08-29T15:20:39.747841+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":220,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748953+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:20:39.748953+0000","last_peered":"2024-08-29T15:20:39.748953+0000","last_clean":"2024-08-29T15:20:39.748953+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:20:39.748953+0000","last_undegraded":"2024-08-29T15:20:39.748953+0000","last_fullsized":"2024-08-29T15:20:39.748953+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":82,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.758240+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:20:39.758240+0000","last_peered":"2024-08-29T15:20:39.758240+0000","last_clean":"2024-08-29T15:20:39.758240+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:20:39.758240+0000","last_undegraded":"2024-08-29T15:20:39.758240+0000","last_fullsized":"2024-08-29T15:20:39.758240+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"244'9869","reported_seq":10232,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:21:05.616584+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:21:05.616584+0000","last_peered":"2024-08-29T15:21:05.616584+0000","last_clean":"2024-08-29T15:21:05.616584+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:21:05.616584+0000","last_undegraded":"2024-08-29T15:21:05.616584+0000","last_fullsized":"2024-08-29T15:21:05.616584+0000","mapping_epoch":201,"log_start":"178'2853","ondisk_log_start":"178'2853","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380960768,"num_objects":5813,"num_object_clones":0,"num_object_copies":17439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9869,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":814,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.787534+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:20:50.787534+0000","last_peered":"2024-08-29T15:20:50.787534+0000","last_clean":"2024-08-29T15:20:50.787534+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:20:50.787534+0000","last_undegraded":"2024-08-29T15:20:50.787534+0000","last_fullsized":"2024-08-29T15:20:50.787534+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":354,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508260+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:21:08.508260+0000","last_peered":"2024-08-29T15:21:08.508260+0000","last_clean":"2024-08-29T15:21:08.508260+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:21:08.508260+0000","last_undegraded":"2024-08-29T15:21:08.508260+0000","last_fullsized":"2024-08-29T15:21:08.508260+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":362,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508299+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:21:08.508299+0000","last_peered":"2024-08-29T15:21:08.508299+0000","last_clean":"2024-08-29T15:21:08.508299+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:21:08.508299+0000","last_undegraded":"2024-08-29T15:21:08.508299+0000","last_fullsized":"2024-08-29T15:21:08.508299+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"244'9746","reported_seq":15533,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:06.639877+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:21:06.639877+0000","last_peered":"2024-08-29T15:21:06.639877+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:21:06.639877+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'3030","ondisk_log_start":"178'3030","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389545984,"num_objects":5944,"num_object_clones":0,"num_object_copies":17832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9746,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1510},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":623,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785117+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:20:50.785117+0000","last_peered":"2024-08-29T15:20:50.785117+0000","last_clean":"2024-08-29T15:20:50.785117+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:20:50.785117+0000","last_undegraded":"2024-08-29T15:20:50.785117+0000","last_fullsized":"2024-08-29T15:20:50.785117+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":73,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.757000+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:20:39.757000+0000","last_peered":"2024-08-29T15:20:39.757000+0000","last_clean":"2024-08-29T15:20:39.757000+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:20:39.757000+0000","last_undegraded":"2024-08-29T15:20:39.757000+0000","last_fullsized":"2024-08-29T15:20:39.757000+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":387,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.788657+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:20:50.788657+0000","last_peered":"2024-08-29T15:20:50.788657+0000","last_clean":"2024-08-29T15:20:50.788657+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:20:50.788657+0000","last_undegraded":"2024-08-29T15:20:50.788657+0000","last_fullsized":"2024-08-29T15:20:50.788657+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"244'9820","reported_seq":9995,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:21:05.876786+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:21:05.876786+0000","last_peered":"2024-08-29T15:21:05.876786+0000","last_clean":"2024-08-29T15:21:05.876786+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:21:05.876786+0000","last_undegraded":"2024-08-29T15:21:05.876786+0000","last_fullsized":"2024-08-29T15:21:05.876786+0000","mapping_epoch":234,"log_start":"178'3097","ondisk_log_start":"178'3097","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6723,"log_dups_size":0,"ondisk_log_size":6723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386138112,"num_objects":5892,"num_object_clones":0,"num_object_copies":17676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9820,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":180,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748846+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:20:39.748846+0000","last_peered":"2024-08-29T15:20:39.748846+0000","last_clean":"2024-08-29T15:20:39.748846+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:20:39.748846+0000","last_undegraded":"2024-08-29T15:20:39.748846+0000","last_fullsized":"2024-08-29T15:20:39.748846+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.783795+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:20:50.783795+0000","last_peered":"2024-08-29T15:20:50.783795+0000","last_clean":"2024-08-29T15:20:50.783795+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:20:50.783795+0000","last_undegraded":"2024-08-29T15:20:50.783795+0000","last_fullsized":"2024-08-29T15:20:50.783795+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748468+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:20:39.748468+0000","last_peered":"2024-08-29T15:20:39.748468+0000","last_clean":"2024-08-29T15:20:39.748468+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:20:39.748468+0000","last_undegraded":"2024-08-29T15:20:39.748468+0000","last_fullsized":"2024-08-29T15:20:39.748468+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"245'9878","reported_seq":10053,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:08.565165+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:21:08.565165+0000","last_peered":"2024-08-29T15:21:08.565165+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:21:08.565165+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"176'1742","ondisk_log_start":"176'1742","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8136,"log_dups_size":0,"ondisk_log_size":8136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389808128,"num_objects":5948,"num_object_clones":0,"num_object_copies":17844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9878,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3525}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":112,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748257+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:20:39.748257+0000","last_peered":"2024-08-29T15:20:39.748257+0000","last_clean":"2024-08-29T15:20:39.748257+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:20:39.748257+0000","last_undegraded":"2024-08-29T15:20:39.748257+0000","last_fullsized":"2024-08-29T15:20:39.748257+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":779,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.788725+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:20:50.788725+0000","last_peered":"2024-08-29T15:20:50.788725+0000","last_clean":"2024-08-29T15:20:50.788725+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:20:50.788725+0000","last_undegraded":"2024-08-29T15:20:50.788725+0000","last_fullsized":"2024-08-29T15:20:50.788725+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":356,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.787993+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:20:50.787993+0000","last_peered":"2024-08-29T15:20:50.787993+0000","last_clean":"2024-08-29T15:20:50.787993+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:20:50.787993+0000","last_undegraded":"2024-08-29T15:20:50.787993+0000","last_fullsized":"2024-08-29T15:20:50.787993+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"244'9789","reported_seq":10518,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:05.608356+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:21:05.608356+0000","last_peered":"2024-08-29T15:21:05.608356+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:21:05.608356+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"177'2780","ondisk_log_start":"177'2780","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7009,"log_dups_size":0,"ondisk_log_size":7009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385286144,"num_objects":5879,"num_object_clones":0,"num_object_copies":17637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5843,"num_objects_unfound":0,"num_objects_dirty":5879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9789,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3728}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":361,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.788633+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:20:50.788633+0000","last_peered":"2024-08-29T15:20:50.788633+0000","last_clean":"2024-08-29T15:20:50.788633+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:20:50.788633+0000","last_undegraded":"2024-08-29T15:20:50.788633+0000","last_fullsized":"2024-08-29T15:20:50.788633+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":670,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:50.826734+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:20:50.826734+0000","last_peered":"2024-08-29T15:20:50.826734+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:20:50.826734+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:20:50.826734+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":354,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508005+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:21:08.508005+0000","last_peered":"2024-08-29T15:21:08.508005+0000","last_clean":"2024-08-29T15:21:08.508005+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:21:08.508005+0000","last_undegraded":"2024-08-29T15:21:08.508005+0000","last_fullsized":"2024-08-29T15:21:08.508005+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"245'9718","reported_seq":16846,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:09.460538+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:21:09.460538+0000","last_peered":"2024-08-29T15:21:09.460538+0000","last_clean":"2024-08-29T15:21:09.460538+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:21:09.460538+0000","last_undegraded":"2024-08-29T15:21:09.460538+0000","last_fullsized":"2024-08-29T15:21:09.460538+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7319,"log_dups_size":0,"ondisk_log_size":7319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374210560,"num_objects":5710,"num_object_clones":0,"num_object_copies":17130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9718,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":346,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.786558+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:20:50.786558+0000","last_peered":"2024-08-29T15:20:50.786558+0000","last_clean":"2024-08-29T15:20:50.786558+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:20:50.786558+0000","last_undegraded":"2024-08-29T15:20:50.786558+0000","last_fullsized":"2024-08-29T15:20:50.786558+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":969,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:20:50.791213+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:20:50.791213+0000","last_peered":"2024-08-29T15:20:50.791213+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:20:50.791213+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":344,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785160+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:20:50.785160+0000","last_peered":"2024-08-29T15:20:50.785160+0000","last_clean":"2024-08-29T15:20:50.785160+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:20:50.785160+0000","last_undegraded":"2024-08-29T15:20:50.785160+0000","last_fullsized":"2024-08-29T15:20:50.785160+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"244'9798","reported_seq":11692,"reported_epoch":244,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:08.310524+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:21:08.310524+0000","last_peered":"2024-08-29T15:21:08.310524+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:21:08.310524+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"176'2212","ondisk_log_start":"176'2212","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380895232,"num_objects":5812,"num_object_clones":0,"num_object_copies":17436,"num_objects_missing_on_primary":6846,"num_objects_missing":6846,"num_objects_degraded":2313,"num_objects_misplaced":5797,"num_objects_unfound":0,"num_objects_dirty":5812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9798,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":876,"num_bytes_recovered":57278464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":3426},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":2}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":344,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.507940+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:21:08.507940+0000","last_peered":"2024-08-29T15:21:08.507940+0000","last_clean":"2024-08-29T15:21:08.507940+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:21:08.507940+0000","last_undegraded":"2024-08-29T15:21:08.507940+0000","last_fullsized":"2024-08-29T15:21:08.507940+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":93,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.757743+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:20:39.757743+0000","last_peered":"2024-08-29T15:20:39.757743+0000","last_clean":"2024-08-29T15:20:39.757743+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:20:39.757743+0000","last_undegraded":"2024-08-29T15:20:39.757743+0000","last_fullsized":"2024-08-29T15:20:39.757743+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784617+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:20:50.784617+0000","last_peered":"2024-08-29T15:20:50.784617+0000","last_clean":"2024-08-29T15:20:50.784617+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:20:50.784617+0000","last_undegraded":"2024-08-29T15:20:50.784617+0000","last_fullsized":"2024-08-29T15:20:50.784617+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784065+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:20:50.784065+0000","last_peered":"2024-08-29T15:20:50.784065+0000","last_clean":"2024-08-29T15:20:50.784065+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:20:50.784065+0000","last_undegraded":"2024-08-29T15:20:50.784065+0000","last_fullsized":"2024-08-29T15:20:50.784065+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":20,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:50.784523+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:20:50.784523+0000","last_peered":"2024-08-29T15:20:50.784523+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:20:50.784523+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:20:50.784523+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":350,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.787861+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:20:50.787861+0000","last_peered":"2024-08-29T15:20:50.787861+0000","last_clean":"2024-08-29T15:20:50.787861+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:20:50.787861+0000","last_undegraded":"2024-08-29T15:20:50.787861+0000","last_fullsized":"2024-08-29T15:20:50.787861+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":101,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:50.787545+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:20:50.787545+0000","last_peered":"2024-08-29T15:20:50.787545+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:20:50.787545+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:20:50.787545+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748140+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:20:39.748140+0000","last_peered":"2024-08-29T15:20:39.748140+0000","last_clean":"2024-08-29T15:20:39.748140+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:20:39.748140+0000","last_undegraded":"2024-08-29T15:20:39.748140+0000","last_fullsized":"2024-08-29T15:20:39.748140+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":310,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784847+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:20:50.784847+0000","last_peered":"2024-08-29T15:20:50.784847+0000","last_clean":"2024-08-29T15:20:50.784847+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:20:50.784847+0000","last_undegraded":"2024-08-29T15:20:50.784847+0000","last_fullsized":"2024-08-29T15:20:50.784847+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":365,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508228+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:21:08.508228+0000","last_peered":"2024-08-29T15:21:08.508228+0000","last_clean":"2024-08-29T15:21:08.508228+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:21:08.508228+0000","last_undegraded":"2024-08-29T15:21:08.508228+0000","last_fullsized":"2024-08-29T15:21:08.508228+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":362,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784946+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:20:50.784946+0000","last_peered":"2024-08-29T15:20:50.784946+0000","last_clean":"2024-08-29T15:20:50.784946+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:20:50.784946+0000","last_undegraded":"2024-08-29T15:20:50.784946+0000","last_fullsized":"2024-08-29T15:20:50.784946+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":82,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.828878+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:20:50.828878+0000","last_peered":"2024-08-29T15:20:50.828878+0000","last_clean":"2024-08-29T15:20:50.828878+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:20:50.828878+0000","last_undegraded":"2024-08-29T15:20:50.828878+0000","last_fullsized":"2024-08-29T15:20:50.828878+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":220,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.507739+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:21:08.507739+0000","last_peered":"2024-08-29T15:21:08.507739+0000","last_clean":"2024-08-29T15:21:08.507739+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:21:08.507739+0000","last_undegraded":"2024-08-29T15:21:08.507739+0000","last_fullsized":"2024-08-29T15:21:08.507739+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":340,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508209+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:21:08.508209+0000","last_peered":"2024-08-29T15:21:08.508209+0000","last_clean":"2024-08-29T15:21:08.508209+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:21:08.508209+0000","last_undegraded":"2024-08-29T15:21:08.508209+0000","last_fullsized":"2024-08-29T15:21:08.508209+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":342,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.792107+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:20:50.792107+0000","last_peered":"2024-08-29T15:20:50.792107+0000","last_clean":"2024-08-29T15:20:50.792107+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:20:50.792107+0000","last_undegraded":"2024-08-29T15:20:50.792107+0000","last_fullsized":"2024-08-29T15:20:50.792107+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":346,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.787392+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:20:50.787392+0000","last_peered":"2024-08-29T15:20:50.787392+0000","last_clean":"2024-08-29T15:20:50.787392+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:20:50.787392+0000","last_undegraded":"2024-08-29T15:20:50.787392+0000","last_fullsized":"2024-08-29T15:20:50.787392+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784386+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:20:50.784386+0000","last_peered":"2024-08-29T15:20:50.784386+0000","last_clean":"2024-08-29T15:20:50.784386+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:20:50.784386+0000","last_undegraded":"2024-08-29T15:20:50.784386+0000","last_fullsized":"2024-08-29T15:20:50.784386+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":366,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.788832+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:20:50.788832+0000","last_peered":"2024-08-29T15:20:50.788832+0000","last_clean":"2024-08-29T15:20:50.788832+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:20:50.788832+0000","last_undegraded":"2024-08-29T15:20:50.788832+0000","last_fullsized":"2024-08-29T15:20:50.788832+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":332,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.507915+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:21:08.507915+0000","last_peered":"2024-08-29T15:21:08.507915+0000","last_clean":"2024-08-29T15:21:08.507915+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:21:08.507915+0000","last_undegraded":"2024-08-29T15:21:08.507915+0000","last_fullsized":"2024-08-29T15:21:08.507915+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.783712+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:20:50.783712+0000","last_peered":"2024-08-29T15:20:50.783712+0000","last_clean":"2024-08-29T15:20:50.783712+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:20:50.783712+0000","last_undegraded":"2024-08-29T15:20:50.783712+0000","last_fullsized":"2024-08-29T15:20:50.783712+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":134,"reported_epoch":243,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:20:39.748620+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:20:39.748620+0000","last_peered":"2024-08-29T15:20:39.748620+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:20:39.748620+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:20:39.748620+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.784461+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:20:50.784461+0000","last_peered":"2024-08-29T15:20:50.784461+0000","last_clean":"2024-08-29T15:20:50.784461+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:20:50.784461+0000","last_undegraded":"2024-08-29T15:20:50.784461+0000","last_fullsized":"2024-08-29T15:20:50.784461+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":140,"reported_epoch":243,"state":"active+clean","last_fresh":"2024-08-29T15:20:39.748594+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:20:39.748594+0000","last_peered":"2024-08-29T15:20:39.748594+0000","last_clean":"2024-08-29T15:20:39.748594+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:20:39.748594+0000","last_undegraded":"2024-08-29T15:20:39.748594+0000","last_fullsized":"2024-08-29T15:20:39.748594+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":378,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.785241+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:20:50.785241+0000","last_peered":"2024-08-29T15:20:50.785241+0000","last_clean":"2024-08-29T15:20:50.785241+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:20:50.785241+0000","last_undegraded":"2024-08-29T15:20:50.785241+0000","last_fullsized":"2024-08-29T15:20:50.785241+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":379,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.786008+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:20:50.786008+0000","last_peered":"2024-08-29T15:20:50.786008+0000","last_clean":"2024-08-29T15:20:50.786008+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:20:50.786008+0000","last_undegraded":"2024-08-29T15:20:50.786008+0000","last_fullsized":"2024-08-29T15:20:50.786008+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":244,"state":"active+clean","last_fresh":"2024-08-29T15:20:50.783553+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:20:50.783553+0000","last_peered":"2024-08-29T15:20:50.783553+0000","last_clean":"2024-08-29T15:20:50.783553+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:20:50.783553+0000","last_undegraded":"2024-08-29T15:20:50.783553+0000","last_fullsized":"2024-08-29T15:20:50.783553+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":388,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.508126+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:21:08.508126+0000","last_peered":"2024-08-29T15:21:08.508126+0000","last_clean":"2024-08-29T15:21:08.508126+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:21:08.508126+0000","last_undegraded":"2024-08-29T15:21:08.508126+0000","last_fullsized":"2024-08-29T15:21:08.508126+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6155796504,"num_objects":93931,"num_object_clones":0,"num_object_copies":281793,"num_objects_missing_on_primary":6846,"num_objects_missing":6846,"num_objects_degraded":30073,"num_objects_misplaced":17558,"num_objects_unfound":0,"num_objects_dirty":93931,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156919,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14503,"num_bytes_recovered":830472192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18338492416,"data_stored":18338480200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121096,"ondisk_log_size":121096,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234318,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3314152,"kb_used_data":3008224,"kb_used_omap":1,"kb_used_meta":305918,"kb_avail":90431000,"statfs":{"total":95995035648,"available":92601344000,"internally_reserved":0,"allocated":3080421376,"data_stored":3079051035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":313260491},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495950,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":457620,"kb_used_data":300172,"kb_used_omap":1,"kb_used_meta":157438,"kb_avail":93287532,"statfs":{"total":95995035648,"available":95526432768,"internally_reserved":0,"allocated":307376128,"data_stored":306017235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":161216967},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426535,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1939144,"kb_used_data":1879936,"kb_used_omap":8,"kb_used_meta":59191,"kb_avail":91806008,"statfs":{"total":95995035648,"available":94009352192,"internally_reserved":0,"allocated":1925054464,"data_stored":1923703037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":60612490},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019228,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2160460,"kb_used_data":1868480,"kb_used_omap":1,"kb_used_meta":291966,"kb_avail":91584692,"statfs":{"total":95995035648,"available":93782724608,"internally_reserved":0,"allocated":1913323520,"data_stored":1911980228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":298973638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435412,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4024580,"kb_used_data":3758332,"kb_used_omap":4,"kb_used_meta":266235,"kb_avail":89720572,"statfs":{"total":95995035648,"available":91873865728,"internally_reserved":0,"allocated":3848531968,"data_stored":3847184481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":272624905},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903264,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3421320,"kb_used_data":3307648,"kb_used_omap":10,"kb_used_meta":113653,"kb_avail":90323832,"statfs":{"total":95995035648,"available":92491603968,"internally_reserved":0,"allocated":3387031552,"data_stored":3385684001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":116381200},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762465,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2573316,"kb_used_data":2312572,"kb_used_omap":7,"kb_used_meta":260728,"kb_avail":91171836,"statfs":{"total":95995035648,"available":93359960064,"internally_reserved":0,"allocated":2368073728,"data_stored":2366725001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":266985826},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925992,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1832012,"kb_used_data":1521284,"kb_used_omap":7,"kb_used_meta":310712,"kb_avail":91913140,"statfs":{"total":95995035648,"available":94119055360,"internally_reserved":0,"allocated":1557794816,"data_stored":1556438868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":318169231},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1549209600,"data_stored":1549205528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2359300096,"data_stored":2359296024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3381526528,"data_stored":3381526528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3842572288,"data_stored":3842572288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1909653504,"data_stored":1909653504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1921060864,"data_stored":1921056792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":303300608,"data_stored":303300608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3071868928,"data_stored":3071868928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:21:12.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:12.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:12.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:12.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:12.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:12.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:13.002 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:13.024 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:13.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:13.147 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:13.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:13.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:13.272 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:13.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:13.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:13.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:13.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:13.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:13 smithi171 ceph-mon[90031]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:13.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:13.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:13 smithi027 ceph-mon[98970]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:13.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:13 smithi027 ceph-mon[93921]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:13.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:21:13.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:13.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:13.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:13.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:13.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:13.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:13.897 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:13.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:13.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:13.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.023 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:14.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:14.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:14.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:14.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:14.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:14.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:14 smithi027 ceph-mon[98970]: pgmap v750: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/280947 objects degraded (10.714%); 17499/280947 objects misplaced (6.229%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:14 smithi027 ceph-mon[93921]: pgmap v750: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/280947 objects degraded (10.714%); 17499/280947 objects misplaced (6.229%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:14.373 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:14.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:14.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:14 smithi171 ceph-mon[90031]: pgmap v750: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/280947 objects degraded (10.714%); 17499/280947 objects misplaced (6.229%); 454 KiB/s, 7 objects/s recovering 2024-08-29T15:21:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:14.520 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:14.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:14.645 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:14.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:14.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:14.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:14.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:14.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:14.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:14.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:14.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:14.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:15.000 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:15.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:15.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:15.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:15.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:15.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:15.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:15.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:15.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:15.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:15.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:15.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:15.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:15.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:15.654 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:15.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:15.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:21:15.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:15.758 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:21:15.758 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:21:15.778 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:21:15.778 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:21:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:15.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:15.794 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:21:15.795 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:21:15.812 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:21:15.812 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:21:15.844 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:21:15.845 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:21:15.867 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:21:15.868 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:21:15.880 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:15.888 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:21:15.888 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:21:15.902 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:15.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:15.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:15.906 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:21:15.906 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:21:16.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:16.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:16.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:16.127 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:16.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:16.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:16.275 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:16.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:16.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:16.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:16.489 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:16 smithi027 ceph-mon[93921]: pgmap v751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/280635 objects degraded (10.726%); 17465/280635 objects misplaced (6.223%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:21:16.489 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:16 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2833813230' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:16.490 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:16 smithi027 ceph-mon[98970]: pgmap v751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/280635 objects degraded (10.726%); 17465/280635 objects misplaced (6.223%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:21:16.490 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:16 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2833813230' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:16.503 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:16 smithi171 ceph-mon[90031]: pgmap v751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 110 op/s; 30102/280635 objects degraded (10.726%); 17465/280635 objects misplaced (6.223%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:21:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:16 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2833813230' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:16.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:16.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:16.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:16.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:16.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:16.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:16.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:16.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:16.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:17.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:17.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:17.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:17.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:17.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:17.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:17.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:17.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:17.467 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/280635 objects degraded (10.726%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:17.467 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/280635 objects degraded (10.726%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:17.503 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/280635 objects degraded (10.726%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:17.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:17.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:17.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:17.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:17.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:17.751 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:17.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:17.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:17.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:17.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:18.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:18.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:18.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:18.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:18.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:18.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:18.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:18.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:18.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:18.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:18.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:18.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:18 smithi027 ceph-mon[98970]: pgmap v752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 30102/279897 objects degraded (10.755%); 17428/279897 objects misplaced (6.227%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:21:18.513 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:18 smithi027 ceph-mon[93921]: pgmap v752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 30102/279897 objects degraded (10.755%); 17428/279897 objects misplaced (6.227%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:21:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:18 smithi171 ceph-mon[90031]: pgmap v752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 105 op/s; 30102/279897 objects degraded (10.755%); 17428/279897 objects misplaced (6.227%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:21:18.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:18.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:18.607 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:18.607 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:18.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:18.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:18.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:18.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:18.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:18.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:18.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:18.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:19.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:19.051 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:19.065 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:21:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:21:19] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-08-29T15:21:19.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:19.175 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:19.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:19.226 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:21:19.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:19.299 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/756969422' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:19 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/756969422' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:19.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:19 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:19.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:19.425 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/756969422' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:19 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:19.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:19.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:19.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:19.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:19.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:19.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:19.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:19.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:19.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:19.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:21:19.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:19.926 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:19.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:19.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:20.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:20.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:20.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:20.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:20.175 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:20.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:20.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:20.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:20.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:20.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:20.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:20.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:20.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[93921]: pgmap v753: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 30102/279153 objects degraded (10.783%); 17381/279153 objects misplaced (6.226%); 488 KiB/s, 7 objects/s recovering 2024-08-29T15:21:20.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:20.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:21:20.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[93921]: osdmap e246: 8 total, 8 up, 8 in 2024-08-29T15:21:20.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[98970]: pgmap v753: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 30102/279153 objects degraded (10.783%); 17381/279153 objects misplaced (6.226%); 488 KiB/s, 7 objects/s recovering 2024-08-29T15:21:20.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:20.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:21:20.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:20 smithi027 ceph-mon[98970]: osdmap e246: 8 total, 8 up, 8 in 2024-08-29T15:21:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:20 smithi171 ceph-mon[90031]: pgmap v753: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 104 op/s; 30102/279153 objects degraded (10.783%); 17381/279153 objects misplaced (6.226%); 488 KiB/s, 7 objects/s recovering 2024-08-29T15:21:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:20 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:20 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:21:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:20 smithi171 ceph-mon[90031]: osdmap e246: 8 total, 8 up, 8 in 2024-08-29T15:21:20.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:20.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:20.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:20.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:20.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:20.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:20.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:20.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:20.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:20.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:21.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:21.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:21.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:21.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:21.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:21.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:21.316 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:21.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:21.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:21.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:21.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:21.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:21.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:21.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:21.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:21.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:21.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:21.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:21.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:22.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:22.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:22.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:22.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:22.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:22.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:21:22.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:22.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:22.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:22.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:22.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:22.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:22 smithi171 ceph-mon[90031]: pgmap v755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30102/278595 objects degraded (10.805%); 17338/278595 objects misplaced (6.223%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:22.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/278595 objects degraded (10.805%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:22.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:22.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:22.544 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:22.553 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[98970]: pgmap v755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30102/278595 objects degraded (10.805%); 17338/278595 objects misplaced (6.223%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:22.553 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/278595 objects degraded (10.805%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:22.554 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:22.554 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:22.554 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[93921]: pgmap v755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 109 op/s; 30102/278595 objects degraded (10.805%); 17338/278595 objects misplaced (6.223%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:22.554 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/278595 objects degraded (10.805%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:22.554 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:22.554 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:22.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:22.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:22.612 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:22.656 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:22.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:22.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:22.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:22.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:22.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:22.939 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:22.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:23.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:23.064 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:23.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:23.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:23.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:23.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:23.213 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:21:23.213 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:21:23.213 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:21:23.213 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:21:23.214 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:21:23.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:23.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:23.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:23.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:23.438 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:23.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:23.447 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:21:23.447 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:21:23.447 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:23.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:21:23.448 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:21:23.448 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:21:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:21:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:23.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:23.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:23.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:23.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:23.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:23.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:23.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:23.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:23.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:24.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:24.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:24.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:21:24.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:24.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:24.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:24.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:24.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:24 smithi027 ceph-mon[98970]: pgmap v756: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 30102/277287 objects degraded (10.856%); 17263/277287 objects misplaced (6.226%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:24.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:24 smithi027 ceph-mon[93921]: pgmap v756: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 30102/277287 objects degraded (10.856%); 17263/277287 objects misplaced (6.226%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:24.444 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:24.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:24.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:24 smithi171 ceph-mon[90031]: pgmap v756: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 121 op/s; 30102/277287 objects degraded (10.856%); 17263/277287 objects misplaced (6.226%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:24.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:24.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:24.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:24.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:24.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:24.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:24.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:24.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:24.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:24.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:24.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:24.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:25.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:25.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:25.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:25.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:25.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:25.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:25.339 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:21:25.340 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:21:25.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:25.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:25.468 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:25.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:25.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:25.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:25.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:25.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:25.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:25.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:25.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:25.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:25.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:25.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:25.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:26.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:26.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:26.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:26.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:26.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:26.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:26.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:26.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:26.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:26 smithi171 ceph-mon[90031]: pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/276924 objects degraded (10.870%); 17226/276924 objects misplaced (6.220%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:26 smithi171 ceph-mon[90031]: from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:26.563 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:26 smithi027 ceph-mon[93921]: pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/276924 objects degraded (10.870%); 17226/276924 objects misplaced (6.220%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:26.563 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:26 smithi027 ceph-mon[93921]: from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:26.564 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:26 smithi027 ceph-mon[98970]: pgmap v757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/276924 objects degraded (10.870%); 17226/276924 objects misplaced (6.220%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:26.564 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:26 smithi027 ceph-mon[98970]: from='client.45413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:26.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:26.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:26.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:26.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:26.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:26.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:26.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:26.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:26.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:26.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:26.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:26.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:27.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:27.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:27.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:27.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:27.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:27.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/276924 objects degraded (10.870%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:27.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/276924 objects degraded (10.870%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:27.337 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:27.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:27.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:27.503 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/276924 objects degraded (10.870%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:27.522 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":757,"stamp":"2024-08-29T15:21:24.555344+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6052386813,"num_objects":92308,"num_object_clones":0,"num_object_copies":276924,"num_objects_missing_on_primary":6740,"num_objects_missing":6740,"num_objects_degraded":30102,"num_objects_misplaced":17226,"num_objects_unfound":0,"num_objects_dirty":92308,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":159693,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14725,"num_bytes_recovered":843844813,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122629,"ondisk_log_size":122629,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19433224,"kb_used_data":17667320,"kb_used_omap":43,"kb_used_meta":1765780,"kb_avail":730527992,"statfs":{"total":767960285184,"available":748060663808,"internally_reserved":0,"allocated":18091335680,"data_stored":18080412590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1808159212},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1,1,1,0,3,4,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-81068032,"num_objects":-1237,"num_object_clones":0,"num_object_copies":-3711,"num_objects_missing_on_primary":-80,"num_objects_missing":-80,"num_objects_degraded":0,"num_objects_misplaced":-239,"num_objects_unfound":0,"num_objects_dirty":-1237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1237,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002680"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":376,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223470+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:21:19.223470+0000","last_peered":"2024-08-29T15:21:19.223470+0000","last_clean":"2024-08-29T15:21:19.223470+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:21:19.223470+0000","last_undegraded":"2024-08-29T15:21:19.223470+0000","last_fullsized":"2024-08-29T15:21:19.223470+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":22,"reported_epoch":245,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:08.198831+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:21:08.198831+0000","last_peered":"2024-08-29T15:21:08.198831+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:21:08.198831+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:21:08.198831+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":112,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196636+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:21:08.196636+0000","last_peered":"2024-08-29T15:21:08.196636+0000","last_clean":"2024-08-29T15:21:08.196636+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:21:08.196636+0000","last_undegraded":"2024-08-29T15:21:08.196636+0000","last_fullsized":"2024-08-29T15:21:08.196636+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":342,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302073+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:21:19.302073+0000","last_peered":"2024-08-29T15:21:19.302073+0000","last_clean":"2024-08-29T15:21:19.302073+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:21:19.302073+0000","last_undegraded":"2024-08-29T15:21:19.302073+0000","last_fullsized":"2024-08-29T15:21:19.302073+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":341,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302445+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:21:19.302445+0000","last_peered":"2024-08-29T15:21:19.302445+0000","last_clean":"2024-08-29T15:21:19.302445+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:21:19.302445+0000","last_undegraded":"2024-08-29T15:21:19.302445+0000","last_fullsized":"2024-08-29T15:21:19.302445+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198503+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:21:08.198503+0000","last_peered":"2024-08-29T15:21:08.198503+0000","last_clean":"2024-08-29T15:21:08.198503+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:21:08.198503+0000","last_undegraded":"2024-08-29T15:21:08.198503+0000","last_fullsized":"2024-08-29T15:21:08.198503+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":153,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226364+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:21:19.226364+0000","last_peered":"2024-08-29T15:21:19.226364+0000","last_clean":"2024-08-29T15:21:19.226364+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:21:19.226364+0000","last_undegraded":"2024-08-29T15:21:19.226364+0000","last_fullsized":"2024-08-29T15:21:19.226364+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":113,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.345695+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:21:08.345695+0000","last_peered":"2024-08-29T15:21:08.345695+0000","last_clean":"2024-08-29T15:21:08.345695+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:21:08.345695+0000","last_undegraded":"2024-08-29T15:21:08.345695+0000","last_fullsized":"2024-08-29T15:21:08.345695+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"246'10046","reported_seq":18069,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:21.058674+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:21:21.058674+0000","last_peered":"2024-08-29T15:21:21.058674+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:21:21.058674+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384434176,"num_objects":5866,"num_object_clones":0,"num_object_copies":17598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10046,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2941}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":21,"reported_epoch":245,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:08.199216+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:21:08.199216+0000","last_peered":"2024-08-29T15:21:08.199216+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:21:08.199216+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:21:08.199216+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":130,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196390+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:21:08.196390+0000","last_peered":"2024-08-29T15:21:08.196390+0000","last_clean":"2024-08-29T15:21:08.196390+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:21:08.196390+0000","last_undegraded":"2024-08-29T15:21:08.196390+0000","last_fullsized":"2024-08-29T15:21:08.196390+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"246'9832","reported_seq":10001,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:21.755936+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:21:21.755936+0000","last_peered":"2024-08-29T15:21:21.755936+0000","last_clean":"2024-08-29T15:21:21.755936+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:21:21.755936+0000","last_undegraded":"2024-08-29T15:21:21.755936+0000","last_fullsized":"2024-08-29T15:21:21.755936+0000","mapping_epoch":168,"log_start":"178'3128","ondisk_log_start":"178'3128","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374734848,"num_objects":5718,"num_object_clones":0,"num_object_copies":17154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9832,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":21,"reported_epoch":245,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:08.201564+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:21:08.201564+0000","last_peered":"2024-08-29T15:21:08.201564+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:21:08.201564+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:21:08.201564+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198401+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:21:08.198401+0000","last_peered":"2024-08-29T15:21:08.198401+0000","last_clean":"2024-08-29T15:21:08.198401+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:21:08.198401+0000","last_undegraded":"2024-08-29T15:21:08.198401+0000","last_fullsized":"2024-08-29T15:21:08.198401+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"246'9944","reported_seq":10083,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:21.282453+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:21:21.282453+0000","last_peered":"2024-08-29T15:21:21.282453+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:21:21.282453+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'3274","ondisk_log_start":"178'3274","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378011648,"num_objects":5768,"num_object_clones":0,"num_object_copies":17304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9944,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3291}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":158,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.305123+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:21:19.305123+0000","last_peered":"2024-08-29T15:21:19.305123+0000","last_clean":"2024-08-29T15:21:19.305123+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:21:19.305123+0000","last_undegraded":"2024-08-29T15:21:19.305123+0000","last_fullsized":"2024-08-29T15:21:19.305123+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.224427+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:21:19.224427+0000","last_peered":"2024-08-29T15:21:19.224427+0000","last_clean":"2024-08-29T15:21:19.224427+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:21:19.224427+0000","last_undegraded":"2024-08-29T15:21:19.224427+0000","last_fullsized":"2024-08-29T15:21:19.224427+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"246'10037","reported_seq":15426,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:23.966498+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:21:23.966498+0000","last_peered":"2024-08-29T15:21:23.966498+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:21:23.966498+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'1862","ondisk_log_start":"176'1862","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8175,"log_dups_size":0,"ondisk_log_size":8175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376242176,"num_objects":5741,"num_object_clones":0,"num_object_copies":17223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10037,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5741}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":130,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196967+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:21:08.196967+0000","last_peered":"2024-08-29T15:21:08.196967+0000","last_clean":"2024-08-29T15:21:08.196967+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:21:08.196967+0000","last_undegraded":"2024-08-29T15:21:08.196967+0000","last_fullsized":"2024-08-29T15:21:08.196967+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":356,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302278+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:21:19.302278+0000","last_peered":"2024-08-29T15:21:19.302278+0000","last_clean":"2024-08-29T15:21:19.302278+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:21:19.302278+0000","last_undegraded":"2024-08-29T15:21:19.302278+0000","last_fullsized":"2024-08-29T15:21:19.302278+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"246'9965","reported_seq":10096,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:19.458992+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:21:19.458992+0000","last_peered":"2024-08-29T15:21:19.458992+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:21:19.458992+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:21:19.458992+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7632,"log_dups_size":0,"ondisk_log_size":7632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383320064,"num_objects":5849,"num_object_clones":0,"num_object_copies":17547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9965,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3372},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":350,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223707+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:21:19.223707+0000","last_peered":"2024-08-29T15:21:19.223707+0000","last_clean":"2024-08-29T15:21:19.223707+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:21:19.223707+0000","last_undegraded":"2024-08-29T15:21:19.223707+0000","last_fullsized":"2024-08-29T15:21:19.223707+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.199042+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:21:08.199042+0000","last_peered":"2024-08-29T15:21:08.199042+0000","last_clean":"2024-08-29T15:21:08.199042+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:21:08.199042+0000","last_undegraded":"2024-08-29T15:21:08.199042+0000","last_fullsized":"2024-08-29T15:21:08.199042+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"246'10002","reported_seq":10188,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:21.739136+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:21:21.739136+0000","last_peered":"2024-08-29T15:21:21.739136+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:21:21.739136+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:21:21.739136+0000","mapping_epoch":234,"log_start":"178'3268","ondisk_log_start":"178'3268","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382730240,"num_objects":5840,"num_object_clones":0,"num_object_copies":17520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10002,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3375},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":87,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.224364+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:21:19.224364+0000","last_peered":"2024-08-29T15:21:19.224364+0000","last_clean":"2024-08-29T15:21:19.224364+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:21:19.224364+0000","last_undegraded":"2024-08-29T15:21:19.224364+0000","last_fullsized":"2024-08-29T15:21:19.224364+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.199157+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:21:08.199157+0000","last_peered":"2024-08-29T15:21:08.199157+0000","last_clean":"2024-08-29T15:21:08.199157+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:21:08.199157+0000","last_undegraded":"2024-08-29T15:21:08.199157+0000","last_fullsized":"2024-08-29T15:21:08.199157+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"246'9852","reported_seq":10222,"reported_epoch":246,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:21:21.759514+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:21:21.759514+0000","last_peered":"2024-08-29T15:21:21.759514+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:21:21.759514+0000","last_undegraded":"2024-08-29T15:21:21.759514+0000","last_fullsized":"2024-08-29T15:21:21.759514+0000","mapping_epoch":202,"log_start":"178'3167","ondisk_log_start":"178'3167","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380502016,"num_objects":5806,"num_object_clones":0,"num_object_copies":17418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5806,"num_objects_unfound":0,"num_objects_dirty":5806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9852,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5806}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198432+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:21:08.198432+0000","last_peered":"2024-08-29T15:21:08.198432+0000","last_clean":"2024-08-29T15:21:08.198432+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:21:08.198432+0000","last_undegraded":"2024-08-29T15:21:08.198432+0000","last_fullsized":"2024-08-29T15:21:08.198432+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":229,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.201613+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:21:08.201613+0000","last_peered":"2024-08-29T15:21:08.201613+0000","last_clean":"2024-08-29T15:21:08.201613+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:21:08.201613+0000","last_undegraded":"2024-08-29T15:21:08.201613+0000","last_fullsized":"2024-08-29T15:21:08.201613+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"246'9943","reported_seq":10068,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:20.909674+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:21:20.909674+0000","last_peered":"2024-08-29T15:21:20.909674+0000","last_clean":"2024-08-29T15:21:20.909674+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:21:20.909674+0000","last_undegraded":"2024-08-29T15:21:20.909674+0000","last_fullsized":"2024-08-29T15:21:20.909674+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8665,"log_dups_size":0,"ondisk_log_size":8665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381157400,"num_objects":5817,"num_object_clones":0,"num_object_copies":17451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5817,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9943,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":353,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.224561+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:21:19.224561+0000","last_peered":"2024-08-29T15:21:19.224561+0000","last_clean":"2024-08-29T15:21:19.224561+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:21:19.224561+0000","last_undegraded":"2024-08-29T15:21:19.224561+0000","last_fullsized":"2024-08-29T15:21:19.224561+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":346,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226481+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:21:19.226481+0000","last_peered":"2024-08-29T15:21:19.226481+0000","last_clean":"2024-08-29T15:21:19.226481+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:21:19.226481+0000","last_undegraded":"2024-08-29T15:21:19.226481+0000","last_fullsized":"2024-08-29T15:21:19.226481+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"246'9615","reported_seq":9788,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:20.742240+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:21:20.742240+0000","last_peered":"2024-08-29T15:21:20.742240+0000","last_clean":"2024-08-29T15:21:20.742240+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:21:20.742240+0000","last_undegraded":"2024-08-29T15:21:20.742240+0000","last_fullsized":"2024-08-29T15:21:20.742240+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9615,"log_dups_size":0,"ondisk_log_size":9615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366542848,"num_objects":5593,"num_object_clones":0,"num_object_copies":16779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9615,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":444,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302215+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:21:19.302215+0000","last_peered":"2024-08-29T15:21:19.302215+0000","last_clean":"2024-08-29T15:21:19.302215+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:21:19.302215+0000","last_undegraded":"2024-08-29T15:21:19.302215+0000","last_fullsized":"2024-08-29T15:21:19.302215+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":714,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196540+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:21:08.196540+0000","last_peered":"2024-08-29T15:21:08.196540+0000","last_clean":"2024-08-29T15:21:08.196540+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:21:08.196540+0000","last_undegraded":"2024-08-29T15:21:08.196540+0000","last_fullsized":"2024-08-29T15:21:08.196540+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":222,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196579+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:21:08.196579+0000","last_peered":"2024-08-29T15:21:08.196579+0000","last_clean":"2024-08-29T15:21:08.196579+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:21:08.196579+0000","last_undegraded":"2024-08-29T15:21:08.196579+0000","last_fullsized":"2024-08-29T15:21:08.196579+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":85,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:13.599984+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:21:13.599984+0000","last_peered":"2024-08-29T15:21:13.599984+0000","last_clean":"2024-08-29T15:21:13.599984+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:21:13.599984+0000","last_undegraded":"2024-08-29T15:21:13.599984+0000","last_fullsized":"2024-08-29T15:21:13.599984+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"246'9978","reported_seq":10345,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:20.290523+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:21:20.290523+0000","last_peered":"2024-08-29T15:21:20.290523+0000","last_clean":"2024-08-29T15:21:20.290523+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:21:20.290523+0000","last_undegraded":"2024-08-29T15:21:20.290523+0000","last_fullsized":"2024-08-29T15:21:20.290523+0000","mapping_epoch":201,"log_start":"178'2953","ondisk_log_start":"178'2953","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7025,"log_dups_size":0,"ondisk_log_size":7025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373817344,"num_objects":5704,"num_object_clones":0,"num_object_copies":17112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9978,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":818,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.225380+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:21:19.225380+0000","last_peered":"2024-08-29T15:21:19.225380+0000","last_clean":"2024-08-29T15:21:19.225380+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:21:19.225380+0000","last_undegraded":"2024-08-29T15:21:19.225380+0000","last_fullsized":"2024-08-29T15:21:19.225380+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":356,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.301927+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:21:19.301927+0000","last_peered":"2024-08-29T15:21:19.301927+0000","last_clean":"2024-08-29T15:21:19.301927+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:21:19.301927+0000","last_undegraded":"2024-08-29T15:21:19.301927+0000","last_fullsized":"2024-08-29T15:21:19.301927+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":364,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302545+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:21:19.302545+0000","last_peered":"2024-08-29T15:21:19.302545+0000","last_clean":"2024-08-29T15:21:19.302545+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:21:19.302545+0000","last_undegraded":"2024-08-29T15:21:19.302545+0000","last_fullsized":"2024-08-29T15:21:19.302545+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"246'9839","reported_seq":15630,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:21.431888+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:21:21.431888+0000","last_peered":"2024-08-29T15:21:21.431888+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:21:21.431888+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'3130","ondisk_log_start":"178'3130","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383451136,"num_objects":5851,"num_object_clones":0,"num_object_copies":17553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9839,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1417},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":627,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223351+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:21:19.223351+0000","last_peered":"2024-08-29T15:21:19.223351+0000","last_clean":"2024-08-29T15:21:19.223351+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:21:19.223351+0000","last_undegraded":"2024-08-29T15:21:19.223351+0000","last_fullsized":"2024-08-29T15:21:19.223351+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.345544+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:21:08.345544+0000","last_peered":"2024-08-29T15:21:08.345544+0000","last_clean":"2024-08-29T15:21:08.345544+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:21:08.345544+0000","last_undegraded":"2024-08-29T15:21:08.345544+0000","last_fullsized":"2024-08-29T15:21:08.345544+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":391,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226230+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:21:19.226230+0000","last_peered":"2024-08-29T15:21:19.226230+0000","last_clean":"2024-08-29T15:21:19.226230+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:21:19.226230+0000","last_undegraded":"2024-08-29T15:21:19.226230+0000","last_fullsized":"2024-08-29T15:21:19.226230+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"246'9923","reported_seq":10102,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:21.752031+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:21:21.752031+0000","last_peered":"2024-08-29T15:21:21.752031+0000","last_clean":"2024-08-29T15:21:21.752031+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:21:21.752031+0000","last_undegraded":"2024-08-29T15:21:21.752031+0000","last_fullsized":"2024-08-29T15:21:21.752031+0000","mapping_epoch":234,"log_start":"178'3197","ondisk_log_start":"178'3197","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379387904,"num_objects":5789,"num_object_clones":0,"num_object_copies":17367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9923,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":182,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196490+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:21:08.196490+0000","last_peered":"2024-08-29T15:21:08.196490+0000","last_clean":"2024-08-29T15:21:08.196490+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:21:08.196490+0000","last_undegraded":"2024-08-29T15:21:08.196490+0000","last_fullsized":"2024-08-29T15:21:08.196490+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198718+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:21:08.198718+0000","last_peered":"2024-08-29T15:21:08.198718+0000","last_clean":"2024-08-29T15:21:08.198718+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:21:08.198718+0000","last_undegraded":"2024-08-29T15:21:08.198718+0000","last_fullsized":"2024-08-29T15:21:08.198718+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":89,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196864+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:21:08.196864+0000","last_peered":"2024-08-29T15:21:08.196864+0000","last_clean":"2024-08-29T15:21:08.196864+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:21:08.196864+0000","last_undegraded":"2024-08-29T15:21:08.196864+0000","last_fullsized":"2024-08-29T15:21:08.196864+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"246'9991","reported_seq":10169,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:23.934178+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:21:23.934178+0000","last_peered":"2024-08-29T15:21:23.934178+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:21:23.934178+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"176'1842","ondisk_log_start":"176'1842","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8149,"log_dups_size":0,"ondisk_log_size":8149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382402560,"num_objects":5835,"num_object_clones":0,"num_object_copies":17505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9991,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3412}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":114,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196751+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:21:08.196751+0000","last_peered":"2024-08-29T15:21:08.196751+0000","last_clean":"2024-08-29T15:21:08.196751+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:21:08.196751+0000","last_undegraded":"2024-08-29T15:21:08.196751+0000","last_fullsized":"2024-08-29T15:21:08.196751+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":783,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226760+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:21:19.226760+0000","last_peered":"2024-08-29T15:21:19.226760+0000","last_clean":"2024-08-29T15:21:19.226760+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:21:19.226760+0000","last_undegraded":"2024-08-29T15:21:19.226760+0000","last_fullsized":"2024-08-29T15:21:19.226760+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":360,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.225863+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:21:19.225863+0000","last_peered":"2024-08-29T15:21:19.225863+0000","last_clean":"2024-08-29T15:21:19.225863+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:21:19.225863+0000","last_undegraded":"2024-08-29T15:21:19.225863+0000","last_fullsized":"2024-08-29T15:21:19.225863+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"246'9895","reported_seq":10628,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:20.267250+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:21:20.267250+0000","last_peered":"2024-08-29T15:21:20.267250+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:21:20.267250+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"178'2880","ondisk_log_start":"178'2880","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7015,"log_dups_size":0,"ondisk_log_size":7015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378339328,"num_objects":5773,"num_object_clones":0,"num_object_copies":17319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5737,"num_objects_unfound":0,"num_objects_dirty":5773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9895,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3622}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":365,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226672+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:21:19.226672+0000","last_peered":"2024-08-29T15:21:19.226672+0000","last_clean":"2024-08-29T15:21:19.226672+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:21:19.226672+0000","last_undegraded":"2024-08-29T15:21:19.226672+0000","last_fullsized":"2024-08-29T15:21:19.226672+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":671,"reported_epoch":245,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:08.433846+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:21:08.433846+0000","last_peered":"2024-08-29T15:21:08.433846+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:21:08.433846+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:21:08.433846+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":356,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302171+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:21:19.302171+0000","last_peered":"2024-08-29T15:21:19.302171+0000","last_clean":"2024-08-29T15:21:19.302171+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:21:19.302171+0000","last_undegraded":"2024-08-29T15:21:19.302171+0000","last_fullsized":"2024-08-29T15:21:19.302171+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"246'9795","reported_seq":16924,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.467315+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:21:19.467315+0000","last_peered":"2024-08-29T15:21:19.467315+0000","last_clean":"2024-08-29T15:21:19.467315+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:21:19.467315+0000","last_undegraded":"2024-08-29T15:21:19.467315+0000","last_fullsized":"2024-08-29T15:21:19.467315+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369164288,"num_objects":5633,"num_object_clones":0,"num_object_copies":16899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9795,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":350,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226238+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:21:19.226238+0000","last_peered":"2024-08-29T15:21:19.226238+0000","last_clean":"2024-08-29T15:21:19.226238+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:21:19.226238+0000","last_undegraded":"2024-08-29T15:21:19.226238+0000","last_fullsized":"2024-08-29T15:21:19.226238+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":973,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:19.224715+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:21:19.224715+0000","last_peered":"2024-08-29T15:21:19.224715+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:21:19.224715+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":348,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223213+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:21:19.223213+0000","last_peered":"2024-08-29T15:21:19.223213+0000","last_clean":"2024-08-29T15:21:19.223213+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:21:19.223213+0000","last_undegraded":"2024-08-29T15:21:19.223213+0000","last_fullsized":"2024-08-29T15:21:19.223213+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"246'9913","reported_seq":12036,"reported_epoch":246,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:23.620960+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:21:23.620960+0000","last_peered":"2024-08-29T15:21:23.620960+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:21:23.620960+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"177'2412","ondisk_log_start":"177'2412","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373358592,"num_objects":5697,"num_object_clones":0,"num_object_copies":17091,"num_objects_missing_on_primary":6732,"num_objects_missing":6732,"num_objects_degraded":2313,"num_objects_misplaced":5683,"num_objects_unfound":0,"num_objects_dirty":5697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9913,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":989,"num_bytes_recovered":64684032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":3312},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":346,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302348+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:21:19.302348+0000","last_peered":"2024-08-29T15:21:19.302348+0000","last_clean":"2024-08-29T15:21:19.302348+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:21:19.302348+0000","last_undegraded":"2024-08-29T15:21:19.302348+0000","last_fullsized":"2024-08-29T15:21:19.302348+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":96,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:18.623128+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:21:18.623128+0000","last_peered":"2024-08-29T15:21:18.623128+0000","last_clean":"2024-08-29T15:21:18.623128+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:21:18.623128+0000","last_undegraded":"2024-08-29T15:21:18.623128+0000","last_fullsized":"2024-08-29T15:21:18.623128+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198468+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:21:08.198468+0000","last_peered":"2024-08-29T15:21:08.198468+0000","last_clean":"2024-08-29T15:21:08.198468+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:21:08.198468+0000","last_undegraded":"2024-08-29T15:21:08.198468+0000","last_fullsized":"2024-08-29T15:21:08.198468+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198819+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:21:08.198819+0000","last_peered":"2024-08-29T15:21:08.198819+0000","last_clean":"2024-08-29T15:21:08.198819+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:21:08.198819+0000","last_undegraded":"2024-08-29T15:21:08.198819+0000","last_fullsized":"2024-08-29T15:21:08.198819+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":21,"reported_epoch":245,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:08.199091+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:21:08.199091+0000","last_peered":"2024-08-29T15:21:08.199091+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:21:08.199091+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:21:08.199091+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":354,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.225720+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:21:19.225720+0000","last_peered":"2024-08-29T15:21:19.225720+0000","last_clean":"2024-08-29T15:21:19.225720+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:21:19.225720+0000","last_undegraded":"2024-08-29T15:21:19.225720+0000","last_fullsized":"2024-08-29T15:21:19.225720+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":105,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:19.223717+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:21:19.223717+0000","last_peered":"2024-08-29T15:21:19.223717+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:21:19.223717+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:21:19.223717+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":112,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196695+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:21:08.196695+0000","last_peered":"2024-08-29T15:21:08.196695+0000","last_clean":"2024-08-29T15:21:08.196695+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:21:08.196695+0000","last_undegraded":"2024-08-29T15:21:08.196695+0000","last_fullsized":"2024-08-29T15:21:08.196695+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":314,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223162+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:21:19.223162+0000","last_peered":"2024-08-29T15:21:19.223162+0000","last_clean":"2024-08-29T15:21:19.223162+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:21:19.223162+0000","last_undegraded":"2024-08-29T15:21:19.223162+0000","last_fullsized":"2024-08-29T15:21:19.223162+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":367,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302545+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:21:19.302545+0000","last_peered":"2024-08-29T15:21:19.302545+0000","last_clean":"2024-08-29T15:21:19.302545+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:21:19.302545+0000","last_undegraded":"2024-08-29T15:21:19.302545+0000","last_fullsized":"2024-08-29T15:21:19.302545+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":366,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223170+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:21:19.223170+0000","last_peered":"2024-08-29T15:21:19.223170+0000","last_clean":"2024-08-29T15:21:19.223170+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:21:19.223170+0000","last_undegraded":"2024-08-29T15:21:19.223170+0000","last_fullsized":"2024-08-29T15:21:19.223170+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":84,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.346148+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:21:08.346148+0000","last_peered":"2024-08-29T15:21:08.346148+0000","last_clean":"2024-08-29T15:21:08.346148+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:21:08.346148+0000","last_undegraded":"2024-08-29T15:21:08.346148+0000","last_fullsized":"2024-08-29T15:21:08.346148+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":223,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.301973+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:21:19.301973+0000","last_peered":"2024-08-29T15:21:19.301973+0000","last_clean":"2024-08-29T15:21:19.301973+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:21:19.301973+0000","last_undegraded":"2024-08-29T15:21:19.301973+0000","last_fullsized":"2024-08-29T15:21:19.301973+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":343,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302522+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:21:19.302522+0000","last_peered":"2024-08-29T15:21:19.302522+0000","last_clean":"2024-08-29T15:21:19.302522+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:21:19.302522+0000","last_undegraded":"2024-08-29T15:21:19.302522+0000","last_fullsized":"2024-08-29T15:21:19.302522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":346,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226111+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:21:19.226111+0000","last_peered":"2024-08-29T15:21:19.226111+0000","last_clean":"2024-08-29T15:21:19.226111+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:21:19.226111+0000","last_undegraded":"2024-08-29T15:21:19.226111+0000","last_fullsized":"2024-08-29T15:21:19.226111+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":350,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223505+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:21:19.223505+0000","last_peered":"2024-08-29T15:21:19.223505+0000","last_clean":"2024-08-29T15:21:19.223505+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:21:19.223505+0000","last_undegraded":"2024-08-29T15:21:19.223505+0000","last_fullsized":"2024-08-29T15:21:19.223505+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198575+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:21:08.198575+0000","last_peered":"2024-08-29T15:21:08.198575+0000","last_clean":"2024-08-29T15:21:08.198575+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:21:08.198575+0000","last_undegraded":"2024-08-29T15:21:08.198575+0000","last_fullsized":"2024-08-29T15:21:08.198575+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":370,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226858+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:21:19.226858+0000","last_peered":"2024-08-29T15:21:19.226858+0000","last_clean":"2024-08-29T15:21:19.226858+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:21:19.226858+0000","last_undegraded":"2024-08-29T15:21:19.226858+0000","last_fullsized":"2024-08-29T15:21:19.226858+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":334,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302067+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:21:19.302067+0000","last_peered":"2024-08-29T15:21:19.302067+0000","last_clean":"2024-08-29T15:21:19.302067+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:21:19.302067+0000","last_undegraded":"2024-08-29T15:21:19.302067+0000","last_fullsized":"2024-08-29T15:21:19.302067+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198629+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:21:08.198629+0000","last_peered":"2024-08-29T15:21:08.198629+0000","last_clean":"2024-08-29T15:21:08.198629+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:21:08.198629+0000","last_undegraded":"2024-08-29T15:21:08.198629+0000","last_fullsized":"2024-08-29T15:21:08.198629+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":136,"reported_epoch":245,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:08.196746+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:21:08.196746+0000","last_peered":"2024-08-29T15:21:08.196746+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:21:08.196746+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:21:08.196746+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198649+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:21:08.198649+0000","last_peered":"2024-08-29T15:21:08.198649+0000","last_clean":"2024-08-29T15:21:08.198649+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:21:08.198649+0000","last_undegraded":"2024-08-29T15:21:08.198649+0000","last_fullsized":"2024-08-29T15:21:08.198649+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":142,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.196739+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:21:08.196739+0000","last_peered":"2024-08-29T15:21:08.196739+0000","last_clean":"2024-08-29T15:21:08.196739+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:21:08.196739+0000","last_undegraded":"2024-08-29T15:21:08.196739+0000","last_fullsized":"2024-08-29T15:21:08.196739+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":382,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223476+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:21:19.223476+0000","last_peered":"2024-08-29T15:21:19.223476+0000","last_clean":"2024-08-29T15:21:19.223476+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:21:19.223476+0000","last_undegraded":"2024-08-29T15:21:19.223476+0000","last_fullsized":"2024-08-29T15:21:19.223476+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":383,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223925+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:21:19.223925+0000","last_peered":"2024-08-29T15:21:19.223925+0000","last_clean":"2024-08-29T15:21:19.223925+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:21:19.223925+0000","last_undegraded":"2024-08-29T15:21:19.223925+0000","last_fullsized":"2024-08-29T15:21:19.223925+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":245,"state":"active+clean","last_fresh":"2024-08-29T15:21:08.198313+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:21:08.198313+0000","last_peered":"2024-08-29T15:21:08.198313+0000","last_clean":"2024-08-29T15:21:08.198313+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:21:08.198313+0000","last_undegraded":"2024-08-29T15:21:08.198313+0000","last_fullsized":"2024-08-29T15:21:08.198313+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":390,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.302523+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:21:19.302523+0000","last_peered":"2024-08-29T15:21:19.302523+0000","last_clean":"2024-08-29T15:21:19.302523+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:21:19.302523+0000","last_undegraded":"2024-08-29T15:21:19.302523+0000","last_fullsized":"2024-08-29T15:21:19.302523+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6047596568,"num_objects":92280,"num_object_clones":0,"num_object_copies":276840,"num_objects_missing_on_primary":6732,"num_objects_missing":6732,"num_objects_degraded":30073,"num_objects_misplaced":17226,"num_objects_unfound":0,"num_objects_dirty":92280,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158570,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14616,"num_bytes_recovered":837877760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18035847168,"data_stored":18035834952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121447,"ondisk_log_size":121447,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234320,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3275708,"kb_used_data":2969776,"kb_used_omap":1,"kb_used_meta":305918,"kb_avail":90469444,"statfs":{"total":95995035648,"available":92640710656,"internally_reserved":0,"allocated":3041050624,"data_stored":3039672635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":313260491},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495953,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":450228,"kb_used_data":288684,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":93294924,"statfs":{"total":95995035648,"available":95534002176,"internally_reserved":0,"allocated":295612416,"data_stored":294238227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":165411271},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,1,0,3,4,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426538,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1906152,"kb_used_data":1846944,"kb_used_omap":8,"kb_used_meta":59191,"kb_avail":91839000,"statfs":{"total":95995035648,"available":94043136000,"internally_reserved":0,"allocated":1891270656,"data_stored":1889903933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":60612490},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019230,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2136648,"kb_used_data":1844672,"kb_used_omap":1,"kb_used_meta":291966,"kb_avail":91608504,"statfs":{"total":95995035648,"available":93807108096,"internally_reserved":0,"allocated":1888944128,"data_stored":1887600836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":298973638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435415,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3958952,"kb_used_data":3688540,"kb_used_omap":4,"kb_used_meta":270395,"kb_avail":89786200,"statfs":{"total":95995035648,"available":91941068800,"internally_reserved":0,"allocated":3777064960,"data_stored":3775702177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":276884745},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903267,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3370668,"kb_used_data":3252832,"kb_used_omap":10,"kb_used_meta":117813,"kb_avail":90374484,"statfs":{"total":95995035648,"available":92543471616,"internally_reserved":0,"allocated":3330899968,"data_stored":3329537121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":120641040},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762468,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2524712,"kb_used_data":2280604,"kb_used_omap":7,"kb_used_meta":244088,"kb_avail":91220440,"statfs":{"total":95995035648,"available":93409730560,"internally_reserved":0,"allocated":2335338496,"data_stored":2333974473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":249946466},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925995,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810156,"kb_used_data":1495268,"kb_used_omap":7,"kb_used_meta":314872,"kb_avail":91934996,"statfs":{"total":95995035648,"available":94141435904,"internally_reserved":0,"allocated":1531154432,"data_stored":1529783188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":322429071},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1521946624,"data_stored":1521942552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2325942272,"data_stored":2325938200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3324313600,"data_stored":3324313600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3769303040,"data_stored":3769303040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1884225536,"data_stored":1884225536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1886326784,"data_stored":1886322712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":290914304,"data_stored":290914304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3032875008,"data_stored":3032875008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:21:27.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:21:27.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:27.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:27.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:27.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:27.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:27.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:27.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:27.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:27.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:27.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:27.964 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:27.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:28.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:28.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:28.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:28.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:28.224 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:28.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:28.314 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:28 smithi027 ceph-mon[98970]: pgmap v758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/275967 objects degraded (10.908%); 17173/275967 objects misplaced (6.223%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:28 smithi027 ceph-mon[93921]: pgmap v758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/275967 objects degraded (10.908%); 17173/275967 objects misplaced (6.223%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:28.326 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:28.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:28.452 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:28.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:28 smithi171 ceph-mon[90031]: pgmap v758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/275967 objects degraded (10.908%); 17173/275967 objects misplaced (6.223%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:28.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:28.604 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:28.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:28.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:28.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:28.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:28.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:28.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:28.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:28.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:28.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:28.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:28.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:29.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:29.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:29.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:29.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:29.230 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:29.244 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:21:29] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-08-29T15:21:29.251 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:21:29.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:29.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:29.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:29.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:29.500 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2179133118' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:29.500 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:29.500 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/172481779' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:21:29.500 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:21:29.500 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2179133118' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:29.501 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:29.501 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/172481779' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:21:29.501 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:29 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:21:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2179133118' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/172481779' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:21:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:29 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:21:29.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:29.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:29.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:29.735 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:29.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:29.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.861 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:29.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:29.962 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:29.984 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:29.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:29.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:30.086 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:30.108 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:30.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:30.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:30.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:30.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:30.357 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:30.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[93921]: pgmap v759: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/275220 objects degraded (10.937%); 17136/275220 objects misplaced (6.226%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:30.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:30.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:21:30.428 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[93921]: osdmap e247: 8 total, 8 up, 8 in 2024-08-29T15:21:30.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[98970]: pgmap v759: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/275220 objects degraded (10.937%); 17136/275220 objects misplaced (6.226%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:30.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:30.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:21:30.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:30 smithi027 ceph-mon[98970]: osdmap e247: 8 total, 8 up, 8 in 2024-08-29T15:21:30.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:30.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:30.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:30 smithi171 ceph-mon[90031]: pgmap v759: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/275220 objects degraded (10.937%); 17136/275220 objects misplaced (6.226%); 467 KiB/s, 7 objects/s recovering 2024-08-29T15:21:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:30 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:21:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:30 smithi171 ceph-mon[90031]: osdmap e247: 8 total, 8 up, 8 in 2024-08-29T15:21:30.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:30.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:30.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:30.650 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:21:30.650 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:21:30.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:30.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:30.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:30.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:30.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:30.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:30.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:30.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:31.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:31.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:31.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:31.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:31.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:31.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:31.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:31.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:31.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:31.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:21:31.458 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:31.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:31.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:31.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:31.649 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:21:31.649 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:21:31.649 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:21:31.649 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:21:31.649 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:21:31.649 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:21:31.650 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:21:31.651 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:21:31.651 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:21:31.706 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:31.728 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:31.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:31.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:31.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:31.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:31.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:31.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:31.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:31.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:32.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:32.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:32.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:21:32.203 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:32.225 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:32.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:32.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:32.334 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:32.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:32.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:32.401 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:32.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:32.475 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:32.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:32 smithi171 ceph-mon[90031]: pgmap v761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/274596 objects degraded (10.962%); 17097/274596 objects misplaced (6.226%); 512 KiB/s, 7 objects/s recovering 2024-08-29T15:21:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/274596 objects degraded (10.962%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:32.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:32 smithi027 ceph-mon[98970]: pgmap v761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/274596 objects degraded (10.962%); 17097/274596 objects misplaced (6.226%); 512 KiB/s, 7 objects/s recovering 2024-08-29T15:21:32.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/274596 objects degraded (10.962%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:32.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:32 smithi027 ceph-mon[93921]: pgmap v761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/274596 objects degraded (10.962%); 17097/274596 objects misplaced (6.226%); 512 KiB/s, 7 objects/s recovering 2024-08-29T15:21:32.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/274596 objects degraded (10.962%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:32.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.601 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:32.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:32.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:32.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:32.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:32.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:32.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:32.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:32.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:32.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:33.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:33.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:33.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:33.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:33.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:33.359 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:33.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:33.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:33.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:33.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:33.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:33.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:33.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:33.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:33.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:33.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:33.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:33.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:33.914 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:33.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:33.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:33.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:33.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:34.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:34.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:34.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:34.208 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:34.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.231 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:34.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:34.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:21:34.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:34.354 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:34.455 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:34.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:34 smithi171 ceph-mon[90031]: pgmap v762: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/273465 objects degraded (11.008%); 17022/273465 objects misplaced (6.225%); 512 KiB/s, 7 objects/s recovering 2024-08-29T15:21:34.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:34 smithi027 ceph-mon[93921]: pgmap v762: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/273465 objects degraded (11.008%); 17022/273465 objects misplaced (6.225%); 512 KiB/s, 7 objects/s recovering 2024-08-29T15:21:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:34 smithi027 ceph-mon[98970]: pgmap v762: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/273465 objects degraded (11.008%); 17022/273465 objects misplaced (6.225%); 512 KiB/s, 7 objects/s recovering 2024-08-29T15:21:34.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:34.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:34.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:34.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:34.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:34.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:34.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:34.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:34.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:34.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:35.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:35.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:35.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:35.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:35.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.223 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:35.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:35.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:35.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:35.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:35.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:35.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:35.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:35.698 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:35.720 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:35.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:35.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:35.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:35.949 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:21:35.949 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/273138 objects degraded (11.021%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1077,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":247,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":91046,"data_bytes":5969680381,"bytes_used":19652546560,"bytes_avail":748307738624,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":273138,"degraded_ratio":0.11020802671177207,"misplaced_objects":16983,"misplaced_total":273138,"misplaced_ratio":0.06217736089449289,"recovering_objects_per_sec":7,"recovering_bytes_per_sec":504484,"recovering_keys_per_sec":0,"num_objects_recovered":77,"num_bytes_recovered":5046272,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":126},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:21:35.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:35.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:36.076 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:36.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:36.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:36.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:36.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:36.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:36.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:36.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:36.456 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:36.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:36 smithi171 ceph-mon[90031]: pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 30102/273138 objects degraded (11.021%); 16983/273138 objects misplaced (6.218%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2733672405' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:21:36.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:36 smithi027 ceph-mon[93921]: pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 30102/273138 objects degraded (11.021%); 16983/273138 objects misplaced (6.218%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2733672405' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:21:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:36 smithi027 ceph-mon[98970]: pgmap v763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 126 op/s; 30102/273138 objects degraded (11.021%); 16983/273138 objects misplaced (6.218%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2733672405' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:21:36.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:36.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:36.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:36.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:36.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:36.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:36.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:36.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:36.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:36.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:36.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:36.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:36.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:36.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:37.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:37.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:37.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:37.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:37.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:37.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:37.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:37.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:37.431 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:37.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:37.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/273138 objects degraded (11.021%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:37.546 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/273138 objects degraded (11.021%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:37.547 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/273138 objects degraded (11.021%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:37.553 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:37.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:37.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:37.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:37.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:37.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:37.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:37.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:37.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:37.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:37.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:37.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:38.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:38.077 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:38.133 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:21:38.133 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:21:38.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:38.202 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:38.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:38 smithi027 ceph-mon[98970]: pgmap v764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/272409 objects degraded (11.050%); 16951/272409 objects misplaced (6.223%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:38 smithi027 ceph-mon[93921]: pgmap v764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/272409 objects degraded (11.050%); 16951/272409 objects misplaced (6.223%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:38.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:38.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:38.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:38.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:38.483 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:21:38.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:21:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:38 smithi171 ceph-mon[90031]: pgmap v764: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/272409 objects degraded (11.050%); 16951/272409 objects misplaced (6.223%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:38.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:38.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:38.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:38.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:38.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:38.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:38.828 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:38.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:38.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:38.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:38.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:38.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:39.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:21:39.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:39.089 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:39.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:39.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:39.214 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:39.270 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:21:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:21:39] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-08-29T15:21:39.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:39.337 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:39.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:39.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:39.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:39.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:39.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:39.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:39.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:39.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:39.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:39.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:39.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:39.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:39.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:39.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:39.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:40.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:40.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:40.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:40.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:40.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:40.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:40.263 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:21:40.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:40.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:40.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:40.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:40.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:40.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:40.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:40 smithi027 ceph-mon[98970]: pgmap v765: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/271659 objects degraded (11.081%); 16902/271659 objects misplaced (6.222%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:40.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3793456736' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:40 smithi027 ceph-mon[93921]: pgmap v765: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/271659 objects degraded (11.081%); 16902/271659 objects misplaced (6.222%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3793456736' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:40.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:40.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:40.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:40.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:40.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:40 smithi171 ceph-mon[90031]: pgmap v765: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/271659 objects degraded (11.081%); 16902/271659 objects misplaced (6.222%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3793456736' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:21:40.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:40.831 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:40.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:40.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:40.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:41.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:41.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:41.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:41.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:41.206 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:41.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:41.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:41.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:41.454 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:41.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:41.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:41 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:21:41.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3793456736' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:21:41.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:41 smithi027 ceph-mon[93921]: osdmap e248: 8 total, 8 up, 8 in 2024-08-29T15:21:41.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:41 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:21:41.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3793456736' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:21:41.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:41 smithi027 ceph-mon[98970]: osdmap e248: 8 total, 8 up, 8 in 2024-08-29T15:21:41.491 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:41.501 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:41 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:21:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3793456736' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:21:41.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:41 smithi171 ceph-mon[90031]: osdmap e248: 8 total, 8 up, 8 in 2024-08-29T15:21:41.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:41.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:41.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:41.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:41.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:41.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:41.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:41.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:41.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:41.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:41.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:41.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:42.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:42.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:42.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:42.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:42.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:42.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:42.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:42.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:42.363 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:42 smithi027 ceph-mon[98970]: pgmap v767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/271035 objects degraded (11.106%); 16866/271035 objects misplaced (6.223%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:21:42.363 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/271035 objects degraded (11.106%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:42.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:42 smithi027 ceph-mon[93921]: pgmap v767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/271035 objects degraded (11.106%); 16866/271035 objects misplaced (6.223%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:21:42.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/271035 objects degraded (11.106%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:42.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:42.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:42.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:42.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:42.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:42.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:42.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:42.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:42.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:42 smithi171 ceph-mon[90031]: pgmap v767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 118 op/s; 30102/271035 objects degraded (11.106%); 16866/271035 objects misplaced (6.223%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:21:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/271035 objects degraded (11.106%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:42.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:42.824 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:21:42.825 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:21:42.827 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:42.845 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:21:42.846 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:21:42.863 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:21:42.864 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:21:42.880 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:21:42.880 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:21:42.897 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:21:42.898 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:21:42.915 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:21:42.916 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:21:42.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:42.933 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:21:42.933 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:21:42.949 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:21:42.949 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:21:42.951 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:42.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:42.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:43.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:43.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:43.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:43.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:43.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:43.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:43.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:43.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:43.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:43.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:43.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:21:43.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:43.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2514725378' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:43.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2514725378' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:43.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:43.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:43.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2514725378' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:43.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:43.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:43.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:43.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:43.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:43.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:44.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:44.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:44.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:44.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:44.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:44.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:44.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:44.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:44.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:44.533 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:21:44.533 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:44 smithi027 ceph-mon[98970]: pgmap v768: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/269571 objects degraded (11.167%); 16775/269571 objects misplaced (6.223%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:21:44.533 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:44.533 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:44.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:44 smithi027 ceph-mon[93921]: pgmap v768: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/269571 objects degraded (11.167%); 16775/269571 objects misplaced (6.223%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:21:44.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:44.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:44.536 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:21:44.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:44.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:44.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:44.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:44.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:44 smithi171 ceph-mon[90031]: pgmap v768: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/269571 objects degraded (11.167%); 16775/269571 objects misplaced (6.223%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:21:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:21:44.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:21:44.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:44.816 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:44.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:44.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:44.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:44.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:45.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:45.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:45.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:45.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:45.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:45.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:45.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:45.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:45.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:45.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:45.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:45.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:45.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:45.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:45.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:45.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:45.845 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:45.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:45.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:45.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:45.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:46.068 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":768,"stamp":"2024-08-29T15:21:42.560342+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5891758077,"num_objects":89857,"num_object_clones":0,"num_object_copies":269571,"num_objects_missing_on_primary":6626,"num_objects_missing":6626,"num_objects_degraded":30102,"num_objects_misplaced":16775,"num_objects_unfound":0,"num_objects_dirty":89857,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":162144,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14842,"num_bytes_recovered":851512525,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123480,"ondisk_log_size":123480,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18943684,"kb_used_data":17198520,"kb_used_omap":43,"kb_used_meta":1745044,"kb_avail":731017532,"statfs":{"total":767960285184,"available":748561952768,"internally_reserved":0,"allocated":17611284480,"data_stored":17600269614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1786925548},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-85065728,"num_objects":-1298,"num_object_clones":0,"num_object_copies":-3894,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":0,"num_objects_misplaced":-247,"num_objects_unfound":0,"num_objects_dirty":-1298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1298,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":4849664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002773"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":380,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266669+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:21:40.266669+0000","last_peered":"2024-08-29T15:21:40.266669+0000","last_clean":"2024-08-29T15:21:40.266669+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:21:40.266669+0000","last_undegraded":"2024-08-29T15:21:40.266669+0000","last_fullsized":"2024-08-29T15:21:40.266669+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":23,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:19.222699+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:21:19.222699+0000","last_peered":"2024-08-29T15:21:19.222699+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:21:19.222699+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:21:19.222699+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":114,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.246296+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:21:29.246296+0000","last_peered":"2024-08-29T15:21:29.246296+0000","last_clean":"2024-08-29T15:21:29.246296+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:21:29.246296+0000","last_undegraded":"2024-08-29T15:21:29.246296+0000","last_fullsized":"2024-08-29T15:21:29.246296+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":344,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.252843+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:21:29.252843+0000","last_peered":"2024-08-29T15:21:29.252843+0000","last_clean":"2024-08-29T15:21:29.252843+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:21:29.252843+0000","last_undegraded":"2024-08-29T15:21:29.252843+0000","last_fullsized":"2024-08-29T15:21:29.252843+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":343,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.252882+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:21:29.252882+0000","last_peered":"2024-08-29T15:21:29.252882+0000","last_clean":"2024-08-29T15:21:29.252882+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:21:29.252882+0000","last_undegraded":"2024-08-29T15:21:29.252882+0000","last_fullsized":"2024-08-29T15:21:29.252882+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222840+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:21:19.222840+0000","last_peered":"2024-08-29T15:21:19.222840+0000","last_clean":"2024-08-29T15:21:19.222840+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:21:19.222840+0000","last_undegraded":"2024-08-29T15:21:19.222840+0000","last_fullsized":"2024-08-29T15:21:19.222840+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":157,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.269976+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:21:40.269976+0000","last_peered":"2024-08-29T15:21:40.269976+0000","last_clean":"2024-08-29T15:21:40.269976+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:21:40.269976+0000","last_undegraded":"2024-08-29T15:21:40.269976+0000","last_fullsized":"2024-08-29T15:21:40.269976+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":115,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.245316+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:21:29.245316+0000","last_peered":"2024-08-29T15:21:29.245316+0000","last_clean":"2024-08-29T15:21:29.245316+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:21:29.245316+0000","last_undegraded":"2024-08-29T15:21:29.245316+0000","last_fullsized":"2024-08-29T15:21:29.245316+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"248'10206","reported_seq":18231,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:41.082407+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:21:41.082407+0000","last_peered":"2024-08-29T15:21:41.082407+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:21:41.082407+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"174'200","ondisk_log_start":"174'200","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373948416,"num_objects":5706,"num_object_clones":0,"num_object_copies":17118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10206,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2781}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":22,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:19.223318+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:21:19.223318+0000","last_peered":"2024-08-29T15:21:19.223318+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:21:19.223318+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:21:19.223318+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":132,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.245702+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:21:29.245702+0000","last_peered":"2024-08-29T15:21:29.245702+0000","last_clean":"2024-08-29T15:21:29.245702+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:21:29.245702+0000","last_undegraded":"2024-08-29T15:21:29.245702+0000","last_fullsized":"2024-08-29T15:21:29.245702+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"248'9976","reported_seq":10149,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:41.726077+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:21:41.726077+0000","last_peered":"2024-08-29T15:21:41.726077+0000","last_clean":"2024-08-29T15:21:41.726077+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:21:41.726077+0000","last_undegraded":"2024-08-29T15:21:41.726077+0000","last_fullsized":"2024-08-29T15:21:41.726077+0000","mapping_epoch":168,"log_start":"178'3228","ondisk_log_start":"178'3228","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6748,"log_dups_size":0,"ondisk_log_size":6748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365297664,"num_objects":5574,"num_object_clones":0,"num_object_copies":16722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9976,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":22,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:19.226542+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:21:19.226542+0000","last_peered":"2024-08-29T15:21:19.226542+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:21:19.226542+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:21:19.226542+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222478+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:21:19.222478+0000","last_peered":"2024-08-29T15:21:19.222478+0000","last_clean":"2024-08-29T15:21:19.222478+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:21:19.222478+0000","last_undegraded":"2024-08-29T15:21:19.222478+0000","last_fullsized":"2024-08-29T15:21:19.222478+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"248'10116","reported_seq":10259,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:41.446686+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:21:41.446686+0000","last_peered":"2024-08-29T15:21:41.446686+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:21:41.446686+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'3374","ondisk_log_start":"178'3374","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366739456,"num_objects":5596,"num_object_clones":0,"num_object_copies":16788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10116,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3119}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":160,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253853+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:21:29.253853+0000","last_peered":"2024-08-29T15:21:29.253853+0000","last_clean":"2024-08-29T15:21:29.253853+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:21:29.253853+0000","last_undegraded":"2024-08-29T15:21:29.253853+0000","last_fullsized":"2024-08-29T15:21:29.253853+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:23.670685+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:21:23.670685+0000","last_peered":"2024-08-29T15:21:23.670685+0000","last_clean":"2024-08-29T15:21:23.670685+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:21:23.670685+0000","last_undegraded":"2024-08-29T15:21:23.670685+0000","last_fullsized":"2024-08-29T15:21:23.670685+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"247'10156","reported_seq":15547,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:38.911495+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:21:38.911495+0000","last_peered":"2024-08-29T15:21:38.911495+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:21:38.911495+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'1962","ondisk_log_start":"176'1962","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368443392,"num_objects":5622,"num_object_clones":0,"num_object_copies":16866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10156,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5622}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.246514+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:21:29.246514+0000","last_peered":"2024-08-29T15:21:29.246514+0000","last_clean":"2024-08-29T15:21:29.246514+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:21:29.246514+0000","last_undegraded":"2024-08-29T15:21:29.246514+0000","last_fullsized":"2024-08-29T15:21:29.246514+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":358,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253084+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:21:29.253084+0000","last_peered":"2024-08-29T15:21:29.253084+0000","last_clean":"2024-08-29T15:21:29.253084+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:21:29.253084+0000","last_undegraded":"2024-08-29T15:21:29.253084+0000","last_fullsized":"2024-08-29T15:21:29.253084+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"247'10120","reported_seq":10252,"reported_epoch":247,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:39.270342+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:21:39.270342+0000","last_peered":"2024-08-29T15:21:39.270342+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:21:39.270342+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:21:39.270342+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7787,"log_dups_size":0,"ondisk_log_size":7787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373161984,"num_objects":5694,"num_object_clones":0,"num_object_copies":17082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10120,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3217},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":354,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266459+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:21:40.266459+0000","last_peered":"2024-08-29T15:21:40.266459+0000","last_clean":"2024-08-29T15:21:40.266459+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:21:40.266459+0000","last_undegraded":"2024-08-29T15:21:40.266459+0000","last_fullsized":"2024-08-29T15:21:40.266459+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222874+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:21:19.222874+0000","last_peered":"2024-08-29T15:21:19.222874+0000","last_clean":"2024-08-29T15:21:19.222874+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:21:19.222874+0000","last_undegraded":"2024-08-29T15:21:19.222874+0000","last_fullsized":"2024-08-29T15:21:19.222874+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"248'10154","reported_seq":10344,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:41.791396+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:21:41.791396+0000","last_peered":"2024-08-29T15:21:41.791396+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:21:41.791396+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:21:41.791396+0000","mapping_epoch":234,"log_start":"178'3468","ondisk_log_start":"178'3468","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372768768,"num_objects":5688,"num_object_clones":0,"num_object_copies":17064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10154,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3223},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:23.670666+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:21:23.670666+0000","last_peered":"2024-08-29T15:21:23.670666+0000","last_clean":"2024-08-29T15:21:23.670666+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:21:23.670666+0000","last_undegraded":"2024-08-29T15:21:23.670666+0000","last_fullsized":"2024-08-29T15:21:23.670666+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223172+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:21:19.223172+0000","last_peered":"2024-08-29T15:21:19.223172+0000","last_clean":"2024-08-29T15:21:19.223172+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:21:19.223172+0000","last_undegraded":"2024-08-29T15:21:19.223172+0000","last_fullsized":"2024-08-29T15:21:19.223172+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"248'10027","reported_seq":10401,"reported_epoch":248,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:21:41.810658+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:21:41.810658+0000","last_peered":"2024-08-29T15:21:41.810658+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:21:41.810658+0000","last_undegraded":"2024-08-29T15:21:41.810658+0000","last_fullsized":"2024-08-29T15:21:41.810658+0000","mapping_epoch":202,"log_start":"178'3267","ondisk_log_start":"178'3267","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6760,"log_dups_size":0,"ondisk_log_size":6760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369033216,"num_objects":5631,"num_object_clones":0,"num_object_copies":16893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5631,"num_objects_unfound":0,"num_objects_dirty":5631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10027,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5631}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222676+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:21:19.222676+0000","last_peered":"2024-08-29T15:21:19.222676+0000","last_clean":"2024-08-29T15:21:19.222676+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:21:19.222676+0000","last_undegraded":"2024-08-29T15:21:19.222676+0000","last_fullsized":"2024-08-29T15:21:19.222676+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":230,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.226621+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:21:19.226621+0000","last_peered":"2024-08-29T15:21:19.226621+0000","last_clean":"2024-08-29T15:21:19.226621+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:21:19.226621+0000","last_undegraded":"2024-08-29T15:21:19.226621+0000","last_fullsized":"2024-08-29T15:21:19.226621+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"248'10104","reported_seq":10231,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:41.054870+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:21:41.054870+0000","last_peered":"2024-08-29T15:21:41.054870+0000","last_clean":"2024-08-29T15:21:41.054870+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:21:41.054870+0000","last_undegraded":"2024-08-29T15:21:41.054870+0000","last_fullsized":"2024-08-29T15:21:41.054870+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8826,"log_dups_size":0,"ondisk_log_size":8826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370606104,"num_objects":5656,"num_object_clones":0,"num_object_copies":16968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5656,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10104,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":357,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.269363+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:21:40.269363+0000","last_peered":"2024-08-29T15:21:40.269363+0000","last_clean":"2024-08-29T15:21:40.269363+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:21:40.269363+0000","last_undegraded":"2024-08-29T15:21:40.269363+0000","last_fullsized":"2024-08-29T15:21:40.269363+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":350,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.270049+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:21:40.270049+0000","last_peered":"2024-08-29T15:21:40.270049+0000","last_clean":"2024-08-29T15:21:40.270049+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:21:40.270049+0000","last_undegraded":"2024-08-29T15:21:40.270049+0000","last_fullsized":"2024-08-29T15:21:40.270049+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"248'9769","reported_seq":9944,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.985144+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:21:40.985144+0000","last_peered":"2024-08-29T15:21:40.985144+0000","last_clean":"2024-08-29T15:21:40.985144+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:21:40.985144+0000","last_undegraded":"2024-08-29T15:21:40.985144+0000","last_fullsized":"2024-08-29T15:21:40.985144+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9769,"log_dups_size":0,"ondisk_log_size":9769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356450304,"num_objects":5439,"num_object_clones":0,"num_object_copies":16317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9769,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":446,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.254268+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:21:29.254268+0000","last_peered":"2024-08-29T15:21:29.254268+0000","last_clean":"2024-08-29T15:21:29.254268+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:21:29.254268+0000","last_undegraded":"2024-08-29T15:21:29.254268+0000","last_fullsized":"2024-08-29T15:21:29.254268+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":716,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.246015+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:21:29.246015+0000","last_peered":"2024-08-29T15:21:29.246015+0000","last_clean":"2024-08-29T15:21:29.246015+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:21:29.246015+0000","last_undegraded":"2024-08-29T15:21:29.246015+0000","last_fullsized":"2024-08-29T15:21:29.246015+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":224,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.247284+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:21:29.247284+0000","last_peered":"2024-08-29T15:21:29.247284+0000","last_clean":"2024-08-29T15:21:29.247284+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:21:29.247284+0000","last_undegraded":"2024-08-29T15:21:29.247284+0000","last_fullsized":"2024-08-29T15:21:29.247284+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.224561+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:21:19.224561+0000","last_peered":"2024-08-29T15:21:19.224561+0000","last_clean":"2024-08-29T15:21:19.224561+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:21:19.224561+0000","last_undegraded":"2024-08-29T15:21:19.224561+0000","last_fullsized":"2024-08-29T15:21:19.224561+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"248'10128","reported_seq":10500,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.270584+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:21:40.270584+0000","last_peered":"2024-08-29T15:21:40.270584+0000","last_clean":"2024-08-29T15:21:40.270584+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:21:40.270584+0000","last_undegraded":"2024-08-29T15:21:40.270584+0000","last_fullsized":"2024-08-29T15:21:40.270584+0000","mapping_epoch":201,"log_start":"178'3053","ondisk_log_start":"178'3053","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363986944,"num_objects":5554,"num_object_clones":0,"num_object_copies":16662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10128,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":822,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.269173+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:21:40.269173+0000","last_peered":"2024-08-29T15:21:40.269173+0000","last_clean":"2024-08-29T15:21:40.269173+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:21:40.269173+0000","last_undegraded":"2024-08-29T15:21:40.269173+0000","last_fullsized":"2024-08-29T15:21:40.269173+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":358,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253423+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:21:29.253423+0000","last_peered":"2024-08-29T15:21:29.253423+0000","last_clean":"2024-08-29T15:21:29.253423+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:21:29.253423+0000","last_undegraded":"2024-08-29T15:21:29.253423+0000","last_fullsized":"2024-08-29T15:21:29.253423+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":366,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253657+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:21:29.253657+0000","last_peered":"2024-08-29T15:21:29.253657+0000","last_clean":"2024-08-29T15:21:29.253657+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:21:29.253657+0000","last_undegraded":"2024-08-29T15:21:29.253657+0000","last_fullsized":"2024-08-29T15:21:29.253657+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"248'9988","reported_seq":15783,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:41.687265+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:21:41.687265+0000","last_peered":"2024-08-29T15:21:41.687265+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:21:41.687265+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'3230","ondisk_log_start":"178'3230","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373686272,"num_objects":5702,"num_object_clones":0,"num_object_copies":17106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9988,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1268},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":631,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266417+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:21:40.266417+0000","last_peered":"2024-08-29T15:21:40.266417+0000","last_clean":"2024-08-29T15:21:40.266417+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:21:40.266417+0000","last_undegraded":"2024-08-29T15:21:40.266417+0000","last_fullsized":"2024-08-29T15:21:40.266417+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":76,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.223991+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:21:19.223991+0000","last_peered":"2024-08-29T15:21:19.223991+0000","last_clean":"2024-08-29T15:21:19.223991+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:21:19.223991+0000","last_undegraded":"2024-08-29T15:21:19.223991+0000","last_fullsized":"2024-08-29T15:21:19.223991+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":395,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.269848+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:21:40.269848+0000","last_peered":"2024-08-29T15:21:40.269848+0000","last_clean":"2024-08-29T15:21:40.269848+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:21:40.269848+0000","last_undegraded":"2024-08-29T15:21:40.269848+0000","last_fullsized":"2024-08-29T15:21:40.269848+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"248'10107","reported_seq":10290,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:41.853629+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:21:41.853629+0000","last_peered":"2024-08-29T15:21:41.853629+0000","last_clean":"2024-08-29T15:21:41.853629+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:21:41.853629+0000","last_undegraded":"2024-08-29T15:21:41.853629+0000","last_fullsized":"2024-08-29T15:21:41.853629+0000","mapping_epoch":234,"log_start":"178'3397","ondisk_log_start":"178'3397","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367329280,"num_objects":5605,"num_object_clones":0,"num_object_copies":16815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10107,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":184,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.246636+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:21:29.246636+0000","last_peered":"2024-08-29T15:21:29.246636+0000","last_clean":"2024-08-29T15:21:29.246636+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:21:29.246636+0000","last_undegraded":"2024-08-29T15:21:29.246636+0000","last_fullsized":"2024-08-29T15:21:29.246636+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222585+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:21:19.222585+0000","last_peered":"2024-08-29T15:21:19.222585+0000","last_clean":"2024-08-29T15:21:19.222585+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:21:19.222585+0000","last_undegraded":"2024-08-29T15:21:19.222585+0000","last_fullsized":"2024-08-29T15:21:19.222585+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.246532+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:21:29.246532+0000","last_peered":"2024-08-29T15:21:29.246532+0000","last_clean":"2024-08-29T15:21:29.246532+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:21:29.246532+0000","last_undegraded":"2024-08-29T15:21:29.246532+0000","last_fullsized":"2024-08-29T15:21:29.246532+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"247'10105","reported_seq":10285,"reported_epoch":247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:38.973890+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:21:38.973890+0000","last_peered":"2024-08-29T15:21:38.973890+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:21:38.973890+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"176'1942","ondisk_log_start":"176'1942","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8163,"log_dups_size":0,"ondisk_log_size":8163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374931456,"num_objects":5721,"num_object_clones":0,"num_object_copies":17163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10105,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3298}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.246225+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:21:29.246225+0000","last_peered":"2024-08-29T15:21:29.246225+0000","last_clean":"2024-08-29T15:21:29.246225+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:21:29.246225+0000","last_undegraded":"2024-08-29T15:21:29.246225+0000","last_fullsized":"2024-08-29T15:21:29.246225+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":787,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.270732+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:21:40.270732+0000","last_peered":"2024-08-29T15:21:40.270732+0000","last_clean":"2024-08-29T15:21:40.270732+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:21:40.270732+0000","last_undegraded":"2024-08-29T15:21:40.270732+0000","last_fullsized":"2024-08-29T15:21:40.270732+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":364,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.269611+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:21:40.269611+0000","last_peered":"2024-08-29T15:21:40.269611+0000","last_clean":"2024-08-29T15:21:40.269611+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:21:40.269611+0000","last_undegraded":"2024-08-29T15:21:40.269611+0000","last_fullsized":"2024-08-29T15:21:40.269611+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"248'10057","reported_seq":10795,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:40.375594+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:21:40.375594+0000","last_peered":"2024-08-29T15:21:40.375594+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:21:40.375594+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"178'3080","ondisk_log_start":"178'3080","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367722496,"num_objects":5611,"num_object_clones":0,"num_object_copies":16833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5575,"num_objects_unfound":0,"num_objects_dirty":5611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10057,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3460}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":369,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.270642+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:21:40.270642+0000","last_peered":"2024-08-29T15:21:40.270642+0000","last_clean":"2024-08-29T15:21:40.270642+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:21:40.270642+0000","last_undegraded":"2024-08-29T15:21:40.270642+0000","last_fullsized":"2024-08-29T15:21:40.270642+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":673,"reported_epoch":247,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:29.265684+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:21:29.265684+0000","last_peered":"2024-08-29T15:21:29.265684+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:21:29.265684+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:21:29.265684+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":358,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253003+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:21:29.253003+0000","last_peered":"2024-08-29T15:21:29.253003+0000","last_clean":"2024-08-29T15:21:29.253003+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:21:29.253003+0000","last_undegraded":"2024-08-29T15:21:29.253003+0000","last_fullsized":"2024-08-29T15:21:29.253003+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"247'9978","reported_seq":17108,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:39.274798+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:21:39.274798+0000","last_peered":"2024-08-29T15:21:39.274798+0000","last_clean":"2024-08-29T15:21:39.274798+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:21:39.274798+0000","last_undegraded":"2024-08-29T15:21:39.274798+0000","last_fullsized":"2024-08-29T15:21:39.274798+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7579,"log_dups_size":0,"ondisk_log_size":7579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357171200,"num_objects":5450,"num_object_clones":0,"num_object_copies":16350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9978,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":354,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.267663+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:21:40.267663+0000","last_peered":"2024-08-29T15:21:40.267663+0000","last_clean":"2024-08-29T15:21:40.267663+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:21:40.267663+0000","last_undegraded":"2024-08-29T15:21:40.267663+0000","last_fullsized":"2024-08-29T15:21:40.267663+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":977,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:40.269412+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:21:40.269412+0000","last_peered":"2024-08-29T15:21:40.269412+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:21:40.269412+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":352,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266682+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:21:40.266682+0000","last_peered":"2024-08-29T15:21:40.266682+0000","last_clean":"2024-08-29T15:21:40.266682+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:21:40.266682+0000","last_undegraded":"2024-08-29T15:21:40.266682+0000","last_fullsized":"2024-08-29T15:21:40.266682+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"247'10030","reported_seq":12384,"reported_epoch":247,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:39.101144+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:21:39.101144+0000","last_peered":"2024-08-29T15:21:39.101144+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:21:39.101144+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"177'2512","ondisk_log_start":"177'2512","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365690880,"num_objects":5580,"num_object_clones":0,"num_object_copies":16740,"num_objects_missing_on_primary":6618,"num_objects_missing":6618,"num_objects_degraded":2313,"num_objects_misplaced":5569,"num_objects_unfound":0,"num_objects_dirty":5580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10030,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1106,"num_bytes_recovered":72351744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":11},{"shards":"1,2,7","objects":3198},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":348,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.252762+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:21:29.252762+0000","last_peered":"2024-08-29T15:21:29.252762+0000","last_clean":"2024-08-29T15:21:29.252762+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:21:29.252762+0000","last_undegraded":"2024-08-29T15:21:29.252762+0000","last_fullsized":"2024-08-29T15:21:29.252762+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":97,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.224629+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:21:19.224629+0000","last_peered":"2024-08-29T15:21:19.224629+0000","last_clean":"2024-08-29T15:21:19.224629+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:21:19.224629+0000","last_undegraded":"2024-08-29T15:21:19.224629+0000","last_fullsized":"2024-08-29T15:21:19.224629+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222763+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:21:19.222763+0000","last_peered":"2024-08-29T15:21:19.222763+0000","last_clean":"2024-08-29T15:21:19.222763+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:21:19.222763+0000","last_undegraded":"2024-08-29T15:21:19.222763+0000","last_fullsized":"2024-08-29T15:21:19.222763+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222664+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:21:19.222664+0000","last_peered":"2024-08-29T15:21:19.222664+0000","last_clean":"2024-08-29T15:21:19.222664+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:21:19.222664+0000","last_undegraded":"2024-08-29T15:21:19.222664+0000","last_fullsized":"2024-08-29T15:21:19.222664+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":22,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:19.222947+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:21:19.222947+0000","last_peered":"2024-08-29T15:21:19.222947+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:21:19.222947+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:21:19.222947+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":358,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.269466+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:21:40.269466+0000","last_peered":"2024-08-29T15:21:40.269466+0000","last_clean":"2024-08-29T15:21:40.269466+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:21:40.269466+0000","last_undegraded":"2024-08-29T15:21:40.269466+0000","last_fullsized":"2024-08-29T15:21:40.269466+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":109,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:40.267794+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:21:40.267794+0000","last_peered":"2024-08-29T15:21:40.267794+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:21:40.267794+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:21:40.267794+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":114,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.245923+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:21:29.245923+0000","last_peered":"2024-08-29T15:21:29.245923+0000","last_clean":"2024-08-29T15:21:29.245923+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:21:29.245923+0000","last_undegraded":"2024-08-29T15:21:29.245923+0000","last_fullsized":"2024-08-29T15:21:29.245923+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":318,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266099+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:21:40.266099+0000","last_peered":"2024-08-29T15:21:40.266099+0000","last_clean":"2024-08-29T15:21:40.266099+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:21:40.266099+0000","last_undegraded":"2024-08-29T15:21:40.266099+0000","last_fullsized":"2024-08-29T15:21:40.266099+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":369,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253321+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:21:29.253321+0000","last_peered":"2024-08-29T15:21:29.253321+0000","last_clean":"2024-08-29T15:21:29.253321+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:21:29.253321+0000","last_undegraded":"2024-08-29T15:21:29.253321+0000","last_fullsized":"2024-08-29T15:21:29.253321+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":370,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266574+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:21:40.266574+0000","last_peered":"2024-08-29T15:21:40.266574+0000","last_clean":"2024-08-29T15:21:40.266574+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:21:40.266574+0000","last_undegraded":"2024-08-29T15:21:40.266574+0000","last_fullsized":"2024-08-29T15:21:40.266574+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":85,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.224694+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:21:19.224694+0000","last_peered":"2024-08-29T15:21:19.224694+0000","last_clean":"2024-08-29T15:21:19.224694+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:21:19.224694+0000","last_undegraded":"2024-08-29T15:21:19.224694+0000","last_fullsized":"2024-08-29T15:21:19.224694+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.254162+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:21:29.254162+0000","last_peered":"2024-08-29T15:21:29.254162+0000","last_clean":"2024-08-29T15:21:29.254162+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:21:29.254162+0000","last_undegraded":"2024-08-29T15:21:29.254162+0000","last_fullsized":"2024-08-29T15:21:29.254162+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":345,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253563+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:21:29.253563+0000","last_peered":"2024-08-29T15:21:29.253563+0000","last_clean":"2024-08-29T15:21:29.253563+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:21:29.253563+0000","last_undegraded":"2024-08-29T15:21:29.253563+0000","last_fullsized":"2024-08-29T15:21:29.253563+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":350,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.272987+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:21:40.272987+0000","last_peered":"2024-08-29T15:21:40.272987+0000","last_clean":"2024-08-29T15:21:40.272987+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:21:40.272987+0000","last_undegraded":"2024-08-29T15:21:40.272987+0000","last_fullsized":"2024-08-29T15:21:40.272987+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":355,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.267518+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:21:40.267518+0000","last_peered":"2024-08-29T15:21:40.267518+0000","last_clean":"2024-08-29T15:21:40.267518+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:21:40.267518+0000","last_undegraded":"2024-08-29T15:21:40.267518+0000","last_fullsized":"2024-08-29T15:21:40.267518+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222571+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:21:19.222571+0000","last_peered":"2024-08-29T15:21:19.222571+0000","last_clean":"2024-08-29T15:21:19.222571+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:21:19.222571+0000","last_undegraded":"2024-08-29T15:21:19.222571+0000","last_fullsized":"2024-08-29T15:21:19.222571+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":375,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.270840+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:21:40.270840+0000","last_peered":"2024-08-29T15:21:40.270840+0000","last_clean":"2024-08-29T15:21:40.270840+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:21:40.270840+0000","last_undegraded":"2024-08-29T15:21:40.270840+0000","last_fullsized":"2024-08-29T15:21:40.270840+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":336,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.252902+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:21:29.252902+0000","last_peered":"2024-08-29T15:21:29.252902+0000","last_clean":"2024-08-29T15:21:29.252902+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:21:29.252902+0000","last_undegraded":"2024-08-29T15:21:29.252902+0000","last_fullsized":"2024-08-29T15:21:29.252902+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222568+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:21:19.222568+0000","last_peered":"2024-08-29T15:21:19.222568+0000","last_clean":"2024-08-29T15:21:19.222568+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:21:19.222568+0000","last_undegraded":"2024-08-29T15:21:19.222568+0000","last_fullsized":"2024-08-29T15:21:19.222568+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":138,"reported_epoch":247,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:29.246282+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:21:29.246282+0000","last_peered":"2024-08-29T15:21:29.246282+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:21:29.246282+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:21:29.246282+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222641+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:21:19.222641+0000","last_peered":"2024-08-29T15:21:19.222641+0000","last_clean":"2024-08-29T15:21:19.222641+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:21:19.222641+0000","last_undegraded":"2024-08-29T15:21:19.222641+0000","last_fullsized":"2024-08-29T15:21:19.222641+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":144,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.246376+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:21:29.246376+0000","last_peered":"2024-08-29T15:21:29.246376+0000","last_clean":"2024-08-29T15:21:29.246376+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:21:29.246376+0000","last_undegraded":"2024-08-29T15:21:29.246376+0000","last_fullsized":"2024-08-29T15:21:29.246376+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":386,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266520+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:21:40.266520+0000","last_peered":"2024-08-29T15:21:40.266520+0000","last_clean":"2024-08-29T15:21:40.266520+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:21:40.266520+0000","last_undegraded":"2024-08-29T15:21:40.266520+0000","last_fullsized":"2024-08-29T15:21:40.266520+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":387,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.267162+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:21:40.267162+0000","last_peered":"2024-08-29T15:21:40.267162+0000","last_clean":"2024-08-29T15:21:40.267162+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:21:40.267162+0000","last_undegraded":"2024-08-29T15:21:40.267162+0000","last_fullsized":"2024-08-29T15:21:40.267162+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":24,"reported_epoch":246,"state":"active+clean","last_fresh":"2024-08-29T15:21:19.222387+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:21:19.222387+0000","last_peered":"2024-08-29T15:21:19.222387+0000","last_clean":"2024-08-29T15:21:19.222387+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:21:19.222387+0000","last_undegraded":"2024-08-29T15:21:19.222387+0000","last_fullsized":"2024-08-29T15:21:19.222387+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":392,"reported_epoch":247,"state":"active+clean","last_fresh":"2024-08-29T15:21:29.253369+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:21:29.253369+0000","last_peered":"2024-08-29T15:21:29.253369+0000","last_clean":"2024-08-29T15:21:29.253369+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:21:29.253369+0000","last_undegraded":"2024-08-29T15:21:29.253369+0000","last_fullsized":"2024-08-29T15:21:29.253369+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5886967832,"num_objects":89829,"num_object_clones":0,"num_object_copies":269487,"num_objects_missing_on_primary":6618,"num_objects_missing":6618,"num_objects_degraded":30073,"num_objects_misplaced":16775,"num_objects_unfound":0,"num_objects_dirty":89829,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161021,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14733,"num_bytes_recovered":845545472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17557827584,"data_stored":17557815368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122298,"ondisk_log_size":122298,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234324,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3198408,"kb_used_data":2888384,"kb_used_omap":1,"kb_used_meta":310014,"kb_avail":90546744,"statfs":{"total":95995035648,"available":92719865856,"internally_reserved":0,"allocated":2957705216,"data_stored":2956319579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":317454795},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495956,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":433604,"kb_used_data":272060,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":93311548,"statfs":{"total":95995035648,"available":95551025152,"internally_reserved":0,"allocated":278589440,"data_stored":277207603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":165411271},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426542,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1858956,"kb_used_data":1795648,"kb_used_omap":8,"kb_used_meta":63287,"kb_avail":91886196,"statfs":{"total":95995035648,"available":94091464704,"internally_reserved":0,"allocated":1838743552,"data_stored":1837361533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":64806794},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019234,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2070568,"kb_used_data":1790880,"kb_used_omap":1,"kb_used_meta":279678,"kb_avail":91674584,"statfs":{"total":95995035648,"available":93874774016,"internally_reserved":0,"allocated":1833861120,"data_stored":1832502532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":286390726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435419,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3864776,"kb_used_data":3586044,"kb_used_omap":4,"kb_used_meta":278715,"kb_avail":89880376,"statfs":{"total":95995035648,"available":92037505024,"internally_reserved":0,"allocated":3672109056,"data_stored":3670730977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":285404425},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903271,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3275084,"kb_used_data":3169728,"kb_used_omap":10,"kb_used_meta":105333,"kb_avail":90470068,"statfs":{"total":95995035648,"available":92641349632,"internally_reserved":0,"allocated":3245801472,"data_stored":3244423329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":107861520},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762472,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2477876,"kb_used_data":2229612,"kb_used_omap":7,"kb_used_meta":248248,"kb_avail":91267276,"statfs":{"total":95995035648,"available":93457690624,"internally_reserved":0,"allocated":2283122688,"data_stored":2281751017,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":254206306},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925998,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1764412,"kb_used_data":1466164,"kb_used_omap":7,"kb_used_meta":298232,"kb_avail":91980740,"statfs":{"total":95995035648,"available":94188277760,"internally_reserved":0,"allocated":1501351936,"data_stored":1499973044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":305389711},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1490227200,"data_stored":1490223128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2275086336,"data_stored":2275082264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3240886272,"data_stored":3240886272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3664379904,"data_stored":3664379904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1830223872,"data_stored":1830223872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1834291200,"data_stored":1834287128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":274530304,"data_stored":274530304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2948202496,"data_stored":2948202496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:21:46.072 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:46.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:46.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:46.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:46.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:46 smithi027 ceph-mon[98970]: from='client.45449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:46 smithi027 ceph-mon[98970]: pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/269241 objects degraded (11.180%); 16733/269241 objects misplaced (6.215%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:46 smithi027 ceph-mon[93921]: from='client.45449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:46 smithi027 ceph-mon[93921]: pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/269241 objects degraded (11.180%); 16733/269241 objects misplaced (6.215%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:46.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:46.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:46.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:46.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:46.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:46.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:46.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:46.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:46.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:46.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:46 smithi171 ceph-mon[90031]: from='client.45449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:46.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:46 smithi171 ceph-mon[90031]: pgmap v769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/269241 objects degraded (11.180%); 16733/269241 objects misplaced (6.215%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:46.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:46.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:46.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:21:46.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:46.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:46.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:46.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:46.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:47.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:47.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:47.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:47.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:47.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:47.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:47.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/269241 objects degraded (11.180%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:47.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/269241 objects degraded (11.180%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:47.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:47.341 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:47.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:47.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:47.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:47.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:47.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:47.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:47.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:47.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/269241 objects degraded (11.180%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:47.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:47.853 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:47.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:47.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:47.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:47.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:48.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:48.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:48.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:48.207 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:48.229 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:48.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:48.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:48.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:48.441 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:48 smithi027 ceph-mon[93921]: pgmap v770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 30102/268563 objects degraded (11.209%); 16699/268563 objects misplaced (6.218%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:48.441 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:48 smithi027 ceph-mon[98970]: pgmap v770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 30102/268563 objects degraded (11.209%); 16699/268563 objects misplaced (6.218%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:48.458 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:48.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:48.560 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:21:48.560 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:21:48.560 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:21:48.561 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:21:48.561 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:21:48.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:48.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:48.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:48.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:48.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:48 smithi171 ceph-mon[90031]: pgmap v770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 30102/268563 objects degraded (11.209%); 16699/268563 objects misplaced (6.218%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:48.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:48.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:48.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:48.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:48.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:49.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:21:49.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:49.150 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:49.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:49.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:49.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:49.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:49.298 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:21:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:21:49] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-08-29T15:21:49.303 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:21:49.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:49.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:49.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:49.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:49.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:49.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:49.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:49.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:49.552 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3495983267' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:49.553 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:49 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3495983267' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:49.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:21:49.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:49.645 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:49.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:49.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3495983267' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:21:49.772 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:49.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:49.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:49.874 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:49.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:49.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:49.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:50.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:50.024 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:50.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:50.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:50.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:50.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:50.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:50.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:50.377 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:50.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:50.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:50.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:50.521 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:50.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:50.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[93921]: pgmap v771: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 30102/267987 objects degraded (11.233%); 16672/267987 objects misplaced (6.221%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:50.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:50.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3495983267' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:21:50.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[93921]: osdmap e249: 8 total, 8 up, 8 in 2024-08-29T15:21:50.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[98970]: pgmap v771: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 30102/267987 objects degraded (11.233%); 16672/267987 objects misplaced (6.221%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:50.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:50.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3495983267' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:21:50.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:50 smithi027 ceph-mon[98970]: osdmap e249: 8 total, 8 up, 8 in 2024-08-29T15:21:50.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:50.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:50.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:50.667 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:50.667 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:50.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:50 smithi171 ceph-mon[90031]: pgmap v771: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 30102/267987 objects degraded (11.233%); 16672/267987 objects misplaced (6.221%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:21:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:50 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:21:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3495983267' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:21:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:50 smithi171 ceph-mon[90031]: osdmap e249: 8 total, 8 up, 8 in 2024-08-29T15:21:50.772 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:50.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:50.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:50.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:50.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:50.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:50.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:51.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:51.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:51.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:51.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:51.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:51.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:51.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:51.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:51.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:51.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:51.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:51.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:51.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:51.787 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:51.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:51.889 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:51.911 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:51.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:51.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:52.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:52.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:52.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:52.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:52.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:52.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:52.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:52.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:21:52.390 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:52.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:52.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:52.515 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:52.537 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:52 smithi027 ceph-mon[93921]: pgmap v773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/267330 objects degraded (11.260%); 16636/267330 objects misplaced (6.223%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/267330 objects degraded (11.260%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:52.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:52 smithi027 ceph-mon[98970]: pgmap v773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/267330 objects degraded (11.260%); 16636/267330 objects misplaced (6.223%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:52.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/267330 objects degraded (11.260%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:52.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:52.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:52.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:52 smithi171 ceph-mon[90031]: pgmap v773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/267330 objects degraded (11.260%); 16636/267330 objects misplaced (6.223%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/267330 objects degraded (11.260%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:52.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:52.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:52.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:52.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:53.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:53.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:53.139 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:53.166 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:53.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:53.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:53.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:53.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:53.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:53.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:53.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:53.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:53.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:53.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:53.794 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:53.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:53.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:53.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:53.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:54.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:54.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:54.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:54.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:54.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:54.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:54.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:54.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:54.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:54.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:54.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:54 smithi027 ceph-mon[98970]: pgmap v774: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/265800 objects degraded (11.325%); 16546/265800 objects misplaced (6.225%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:54.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:54 smithi027 ceph-mon[93921]: pgmap v774: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/265800 objects degraded (11.325%); 16546/265800 objects misplaced (6.225%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:54.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:54.541 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:54.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:54.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:54.766 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:54 smithi171 ceph-mon[90031]: pgmap v774: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/265800 objects degraded (11.325%); 16546/265800 objects misplaced (6.225%); 486 KiB/s, 7 objects/s recovering 2024-08-29T15:21:54.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:54.806 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:21:54.807 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:21:54.808 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:54.827 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:21:54.827 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:21:54.849 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:21:54.849 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:21:54.865 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:21:54.865 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:21:54.886 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:21:54.886 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:21:54.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:54.904 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:21:54.905 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:21:54.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:54.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:54.923 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:21:54.923 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:21:54.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:21:54.944 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:21:55.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.040 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:55.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:55.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:55.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:55.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:55.258 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:21:55.259 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:21:55.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:55.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3065656668' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:55.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3065656668' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:55.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:55.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:55.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:55.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:55.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:55.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3065656668' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:21:55.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:55.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:55.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:55.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:55.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:55.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:56.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:56.044 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:56.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:56.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:56.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:56.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:56.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:56.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:56.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:56.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:56.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:56.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:56 smithi027 ceph-mon[93921]: pgmap v775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/265425 objects degraded (11.341%); 16507/265425 objects misplaced (6.219%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:56 smithi027 ceph-mon[93921]: from='client.35550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:56 smithi027 ceph-mon[98970]: pgmap v775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/265425 objects degraded (11.341%); 16507/265425 objects misplaced (6.219%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:56 smithi027 ceph-mon[98970]: from='client.35550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:56.616 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:21:56.617 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:21:56.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:56.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:56 smithi171 ceph-mon[90031]: pgmap v775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/265425 objects degraded (11.341%); 16507/265425 objects misplaced (6.219%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:56 smithi171 ceph-mon[90031]: from='client.35550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:21:56.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:56.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.796 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:56.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:56.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:56.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:56.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:57.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:57.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:57.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:57.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:57.167 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:57.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:57.291 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:57.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/265425 objects degraded (11.341%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/265425 objects degraded (11.341%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:57.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:21:57.411 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":775,"stamp":"2024-08-29T15:21:54.563878+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5801187325,"num_objects":88475,"num_object_clones":0,"num_object_copies":265425,"num_objects_missing_on_primary":6509,"num_objects_missing":6509,"num_objects_degraded":30102,"num_objects_misplaced":16507,"num_objects_unfound":0,"num_objects_dirty":88475,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":163526,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14957,"num_bytes_recovered":859049165,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123862,"ondisk_log_size":123862,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18671196,"kb_used_data":16938440,"kb_used_omap":43,"kb_used_meta":1732628,"kb_avail":731290020,"statfs":{"total":767960285184,"available":748840980480,"internally_reserved":0,"allocated":17344962560,"data_stored":17333848270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1774211564},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,2,0,3,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-83361792,"num_objects":-1272,"num_object_clones":0,"num_object_copies":-3816,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":0,"num_objects_misplaced":-226,"num_objects_unfound":0,"num_objects_dirty":-1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1272,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":5046272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002985"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":382,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.305028+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:21:49.305028+0000","last_peered":"2024-08-29T15:21:49.305028+0000","last_clean":"2024-08-29T15:21:49.305028+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:21:49.305028+0000","last_undegraded":"2024-08-29T15:21:49.305028+0000","last_fullsized":"2024-08-29T15:21:49.305028+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":25,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:40.266257+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:21:40.266257+0000","last_peered":"2024-08-29T15:21:40.266257+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:21:40.266257+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:21:40.266257+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":115,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.264915+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:21:40.264915+0000","last_peered":"2024-08-29T15:21:40.264915+0000","last_clean":"2024-08-29T15:21:40.264915+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:21:40.264915+0000","last_undegraded":"2024-08-29T15:21:40.264915+0000","last_fullsized":"2024-08-29T15:21:40.264915+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":348,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316163+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:21:49.316163+0000","last_peered":"2024-08-29T15:21:49.316163+0000","last_clean":"2024-08-29T15:21:49.316163+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:21:49.316163+0000","last_undegraded":"2024-08-29T15:21:49.316163+0000","last_fullsized":"2024-08-29T15:21:49.316163+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":348,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316464+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:21:49.316464+0000","last_peered":"2024-08-29T15:21:49.316464+0000","last_clean":"2024-08-29T15:21:49.316464+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:21:49.316464+0000","last_undegraded":"2024-08-29T15:21:49.316464+0000","last_fullsized":"2024-08-29T15:21:49.316464+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266154+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:21:40.266154+0000","last_peered":"2024-08-29T15:21:40.266154+0000","last_clean":"2024-08-29T15:21:40.266154+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:21:40.266154+0000","last_undegraded":"2024-08-29T15:21:40.266154+0000","last_fullsized":"2024-08-29T15:21:40.266154+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":159,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.311397+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:21:49.311397+0000","last_peered":"2024-08-29T15:21:49.311397+0000","last_clean":"2024-08-29T15:21:49.311397+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:21:49.311397+0000","last_undegraded":"2024-08-29T15:21:49.311397+0000","last_fullsized":"2024-08-29T15:21:49.311397+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":117,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.303808+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:21:40.303808+0000","last_peered":"2024-08-29T15:21:40.303808+0000","last_clean":"2024-08-29T15:21:40.303808+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:21:40.303808+0000","last_undegraded":"2024-08-29T15:21:40.303808+0000","last_fullsized":"2024-08-29T15:21:40.303808+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"249'10294","reported_seq":18320,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:51.071508+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:21:51.071508+0000","last_peered":"2024-08-29T15:21:51.071508+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:21:51.071508+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"174'200","ondisk_log_start":"174'200","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368181248,"num_objects":5618,"num_object_clones":0,"num_object_copies":16854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10294,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2693}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":24,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:40.266700+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:21:40.266700+0000","last_peered":"2024-08-29T15:21:40.266700+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:21:40.266700+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:21:40.266700+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.264561+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:21:40.264561+0000","last_peered":"2024-08-29T15:21:40.264561+0000","last_clean":"2024-08-29T15:21:40.264561+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:21:40.264561+0000","last_undegraded":"2024-08-29T15:21:40.264561+0000","last_fullsized":"2024-08-29T15:21:40.264561+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"249'10042","reported_seq":10217,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:51.845949+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:21:51.845949+0000","last_peered":"2024-08-29T15:21:51.845949+0000","last_clean":"2024-08-29T15:21:51.845949+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:21:51.845949+0000","last_undegraded":"2024-08-29T15:21:51.845949+0000","last_fullsized":"2024-08-29T15:21:51.845949+0000","mapping_epoch":168,"log_start":"178'3328","ondisk_log_start":"178'3328","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6714,"log_dups_size":0,"ondisk_log_size":6714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360972288,"num_objects":5508,"num_object_clones":0,"num_object_copies":16524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10042,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":24,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:40.272005+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:21:40.272005+0000","last_peered":"2024-08-29T15:21:40.272005+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:21:40.272005+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:21:40.272005+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266071+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:21:40.266071+0000","last_peered":"2024-08-29T15:21:40.266071+0000","last_clean":"2024-08-29T15:21:40.266071+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:21:40.266071+0000","last_undegraded":"2024-08-29T15:21:40.266071+0000","last_fullsized":"2024-08-29T15:21:40.266071+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"249'10194","reported_seq":10339,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:51.055448+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:21:51.055448+0000","last_peered":"2024-08-29T15:21:51.055448+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:21:51.055448+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'3474","ondisk_log_start":"178'3474","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361627648,"num_objects":5518,"num_object_clones":0,"num_object_copies":16554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10194,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3041}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":164,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316419+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:21:49.316419+0000","last_peered":"2024-08-29T15:21:49.316419+0000","last_clean":"2024-08-29T15:21:49.316419+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:21:49.316419+0000","last_undegraded":"2024-08-29T15:21:49.316419+0000","last_fullsized":"2024-08-29T15:21:49.316419+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.304239+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:21:40.304239+0000","last_peered":"2024-08-29T15:21:40.304239+0000","last_clean":"2024-08-29T15:21:40.304239+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:21:40.304239+0000","last_undegraded":"2024-08-29T15:21:40.304239+0000","last_fullsized":"2024-08-29T15:21:40.304239+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"249'10292","reported_seq":15688,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:53.714256+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:21:53.714256+0000","last_peered":"2024-08-29T15:21:53.714256+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:21:53.714256+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'2162","ondisk_log_start":"176'2162","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359530496,"num_objects":5486,"num_object_clones":0,"num_object_copies":16458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10292,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5486}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":133,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.265185+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:21:40.265185+0000","last_peered":"2024-08-29T15:21:40.265185+0000","last_clean":"2024-08-29T15:21:40.265185+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:21:40.265185+0000","last_undegraded":"2024-08-29T15:21:40.265185+0000","last_fullsized":"2024-08-29T15:21:40.265185+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":362,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316339+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:21:49.316339+0000","last_peered":"2024-08-29T15:21:49.316339+0000","last_clean":"2024-08-29T15:21:49.316339+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:21:49.316339+0000","last_undegraded":"2024-08-29T15:21:49.316339+0000","last_fullsized":"2024-08-29T15:21:49.316339+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"249'10193","reported_seq":10327,"reported_epoch":249,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:49.710798+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:21:49.710798+0000","last_peered":"2024-08-29T15:21:49.710798+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:21:49.710798+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:21:49.710798+0000","mapping_epoch":234,"log_start":"177'2333","ondisk_log_start":"177'2333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7860,"log_dups_size":0,"ondisk_log_size":7860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368377856,"num_objects":5621,"num_object_clones":0,"num_object_copies":16863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10193,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":3144},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":356,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.304907+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:21:49.304907+0000","last_peered":"2024-08-29T15:21:49.304907+0000","last_clean":"2024-08-29T15:21:49.304907+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:21:49.304907+0000","last_undegraded":"2024-08-29T15:21:49.304907+0000","last_fullsized":"2024-08-29T15:21:49.304907+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266424+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:21:40.266424+0000","last_peered":"2024-08-29T15:21:40.266424+0000","last_clean":"2024-08-29T15:21:40.266424+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:21:40.266424+0000","last_undegraded":"2024-08-29T15:21:40.266424+0000","last_fullsized":"2024-08-29T15:21:40.266424+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"249'10225","reported_seq":10417,"reported_epoch":249,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:51.785493+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:21:51.785493+0000","last_peered":"2024-08-29T15:21:51.785493+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:21:51.785493+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:21:51.785493+0000","mapping_epoch":234,"log_start":"178'3468","ondisk_log_start":"178'3468","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368115712,"num_objects":5617,"num_object_clones":0,"num_object_copies":16851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10225,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3152},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":90,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.304155+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:21:40.304155+0000","last_peered":"2024-08-29T15:21:40.304155+0000","last_clean":"2024-08-29T15:21:40.304155+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:21:40.304155+0000","last_undegraded":"2024-08-29T15:21:40.304155+0000","last_fullsized":"2024-08-29T15:21:40.304155+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266577+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:21:40.266577+0000","last_peered":"2024-08-29T15:21:40.266577+0000","last_clean":"2024-08-29T15:21:40.266577+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:21:40.266577+0000","last_undegraded":"2024-08-29T15:21:40.266577+0000","last_fullsized":"2024-08-29T15:21:40.266577+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"249'10103","reported_seq":10479,"reported_epoch":249,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:21:51.803022+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:21:51.803022+0000","last_peered":"2024-08-29T15:21:51.803022+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:21:51.803022+0000","last_undegraded":"2024-08-29T15:21:51.803022+0000","last_fullsized":"2024-08-29T15:21:51.803022+0000","mapping_epoch":202,"log_start":"178'3367","ondisk_log_start":"178'3367","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364052480,"num_objects":5555,"num_object_clones":0,"num_object_copies":16665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5555,"num_objects_unfound":0,"num_objects_dirty":5555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10103,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5555}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266046+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:21:40.266046+0000","last_peered":"2024-08-29T15:21:40.266046+0000","last_clean":"2024-08-29T15:21:40.266046+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:21:40.266046+0000","last_undegraded":"2024-08-29T15:21:40.266046+0000","last_fullsized":"2024-08-29T15:21:40.266046+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":232,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.272067+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:21:40.272067+0000","last_peered":"2024-08-29T15:21:40.272067+0000","last_clean":"2024-08-29T15:21:40.272067+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:21:40.272067+0000","last_undegraded":"2024-08-29T15:21:40.272067+0000","last_fullsized":"2024-08-29T15:21:40.272067+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"249'10190","reported_seq":10318,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:51.101509+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:21:51.101509+0000","last_peered":"2024-08-29T15:21:51.101509+0000","last_clean":"2024-08-29T15:21:51.101509+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:21:51.101509+0000","last_undegraded":"2024-08-29T15:21:51.101509+0000","last_fullsized":"2024-08-29T15:21:51.101509+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8912,"log_dups_size":0,"ondisk_log_size":8912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364970008,"num_objects":5570,"num_object_clones":0,"num_object_copies":16710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5570,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10190,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":359,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.308434+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:21:49.308434+0000","last_peered":"2024-08-29T15:21:49.308434+0000","last_clean":"2024-08-29T15:21:49.308434+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:21:49.308434+0000","last_undegraded":"2024-08-29T15:21:49.308434+0000","last_fullsized":"2024-08-29T15:21:49.308434+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":352,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.311483+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:21:49.311483+0000","last_peered":"2024-08-29T15:21:49.311483+0000","last_clean":"2024-08-29T15:21:49.311483+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:21:49.311483+0000","last_undegraded":"2024-08-29T15:21:49.311483+0000","last_fullsized":"2024-08-29T15:21:49.311483+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"249'9858","reported_seq":10034,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:51.059004+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:21:51.059004+0000","last_peered":"2024-08-29T15:21:51.059004+0000","last_clean":"2024-08-29T15:21:51.059004+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:21:51.059004+0000","last_undegraded":"2024-08-29T15:21:51.059004+0000","last_fullsized":"2024-08-29T15:21:51.059004+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9858,"log_dups_size":0,"ondisk_log_size":9858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350617600,"num_objects":5350,"num_object_clones":0,"num_object_copies":16050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9858,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":450,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316625+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:21:49.316625+0000","last_peered":"2024-08-29T15:21:49.316625+0000","last_clean":"2024-08-29T15:21:49.316625+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:21:49.316625+0000","last_undegraded":"2024-08-29T15:21:49.316625+0000","last_fullsized":"2024-08-29T15:21:49.316625+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":718,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307081+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:21:49.307081+0000","last_peered":"2024-08-29T15:21:49.307081+0000","last_clean":"2024-08-29T15:21:49.307081+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:21:49.307081+0000","last_undegraded":"2024-08-29T15:21:49.307081+0000","last_fullsized":"2024-08-29T15:21:49.307081+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":226,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:46.156006+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:21:46.156006+0000","last_peered":"2024-08-29T15:21:46.156006+0000","last_clean":"2024-08-29T15:21:46.156006+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:21:46.156006+0000","last_undegraded":"2024-08-29T15:21:46.156006+0000","last_fullsized":"2024-08-29T15:21:46.156006+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":88,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.304205+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:21:40.304205+0000","last_peered":"2024-08-29T15:21:40.304205+0000","last_clean":"2024-08-29T15:21:40.304205+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:21:40.304205+0000","last_undegraded":"2024-08-29T15:21:40.304205+0000","last_fullsized":"2024-08-29T15:21:40.304205+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"249'10202","reported_seq":10576,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.872165+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:21:49.872165+0000","last_peered":"2024-08-29T15:21:49.872165+0000","last_clean":"2024-08-29T15:21:49.872165+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:21:49.872165+0000","last_undegraded":"2024-08-29T15:21:49.872165+0000","last_fullsized":"2024-08-29T15:21:49.872165+0000","mapping_epoch":201,"log_start":"178'3153","ondisk_log_start":"178'3153","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7049,"log_dups_size":0,"ondisk_log_size":7049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359137280,"num_objects":5480,"num_object_clones":0,"num_object_copies":16440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10202,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":824,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.308281+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:21:49.308281+0000","last_peered":"2024-08-29T15:21:49.308281+0000","last_clean":"2024-08-29T15:21:49.308281+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:21:49.308281+0000","last_undegraded":"2024-08-29T15:21:49.308281+0000","last_fullsized":"2024-08-29T15:21:49.308281+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":363,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316550+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:21:49.316550+0000","last_peered":"2024-08-29T15:21:49.316550+0000","last_clean":"2024-08-29T15:21:49.316550+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:21:49.316550+0000","last_undegraded":"2024-08-29T15:21:49.316550+0000","last_fullsized":"2024-08-29T15:21:49.316550+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":370,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.317048+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:21:49.317048+0000","last_peered":"2024-08-29T15:21:49.317048+0000","last_clean":"2024-08-29T15:21:49.317048+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:21:49.317048+0000","last_undegraded":"2024-08-29T15:21:49.317048+0000","last_fullsized":"2024-08-29T15:21:49.317048+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"249'10083","reported_seq":15880,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:51.777024+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:21:51.777024+0000","last_peered":"2024-08-29T15:21:51.777024+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:21:51.777024+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'3330","ondisk_log_start":"178'3330","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367460352,"num_objects":5607,"num_object_clones":0,"num_object_copies":16821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10083,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1173},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":633,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.304652+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:21:49.304652+0000","last_peered":"2024-08-29T15:21:49.304652+0000","last_clean":"2024-08-29T15:21:49.304652+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:21:49.304652+0000","last_undegraded":"2024-08-29T15:21:49.304652+0000","last_fullsized":"2024-08-29T15:21:49.304652+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.303636+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:21:40.303636+0000","last_peered":"2024-08-29T15:21:40.303636+0000","last_clean":"2024-08-29T15:21:40.303636+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:21:40.303636+0000","last_undegraded":"2024-08-29T15:21:40.303636+0000","last_fullsized":"2024-08-29T15:21:40.303636+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":397,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.311246+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:21:49.311246+0000","last_peered":"2024-08-29T15:21:49.311246+0000","last_clean":"2024-08-29T15:21:49.311246+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:21:49.311246+0000","last_undegraded":"2024-08-29T15:21:49.311246+0000","last_fullsized":"2024-08-29T15:21:49.311246+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"249'10176","reported_seq":10362,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:51.842458+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:21:51.842458+0000","last_peered":"2024-08-29T15:21:51.842458+0000","last_clean":"2024-08-29T15:21:51.842458+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:21:51.842458+0000","last_undegraded":"2024-08-29T15:21:51.842458+0000","last_fullsized":"2024-08-29T15:21:51.842458+0000","mapping_epoch":234,"log_start":"178'3497","ondisk_log_start":"178'3497","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6679,"log_dups_size":0,"ondisk_log_size":6679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362807296,"num_objects":5536,"num_object_clones":0,"num_object_copies":16608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10176,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":186,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:46.155980+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:21:46.155980+0000","last_peered":"2024-08-29T15:21:46.155980+0000","last_clean":"2024-08-29T15:21:46.155980+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:21:46.155980+0000","last_undegraded":"2024-08-29T15:21:46.155980+0000","last_fullsized":"2024-08-29T15:21:46.155980+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266199+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:21:40.266199+0000","last_peered":"2024-08-29T15:21:40.266199+0000","last_clean":"2024-08-29T15:21:40.266199+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:21:40.266199+0000","last_undegraded":"2024-08-29T15:21:40.266199+0000","last_fullsized":"2024-08-29T15:21:40.266199+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307085+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:21:49.307085+0000","last_peered":"2024-08-29T15:21:49.307085+0000","last_clean":"2024-08-29T15:21:49.307085+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:21:49.307085+0000","last_undegraded":"2024-08-29T15:21:49.307085+0000","last_fullsized":"2024-08-29T15:21:49.307085+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"249'10229","reported_seq":10413,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:53.849935+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:21:53.849935+0000","last_peered":"2024-08-29T15:21:53.849935+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:21:53.849935+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"176'2042","ondisk_log_start":"176'2042","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366804992,"num_objects":5597,"num_object_clones":0,"num_object_copies":16791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10229,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3174}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":118,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307571+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:21:49.307571+0000","last_peered":"2024-08-29T15:21:49.307571+0000","last_clean":"2024-08-29T15:21:49.307571+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:21:49.307571+0000","last_undegraded":"2024-08-29T15:21:49.307571+0000","last_fullsized":"2024-08-29T15:21:49.307571+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":789,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.311135+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:21:49.311135+0000","last_peered":"2024-08-29T15:21:49.311135+0000","last_clean":"2024-08-29T15:21:49.311135+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:21:49.311135+0000","last_undegraded":"2024-08-29T15:21:49.311135+0000","last_fullsized":"2024-08-29T15:21:49.311135+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":366,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.308041+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:21:49.308041+0000","last_peered":"2024-08-29T15:21:49.308041+0000","last_clean":"2024-08-29T15:21:49.308041+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:21:49.308041+0000","last_undegraded":"2024-08-29T15:21:49.308041+0000","last_fullsized":"2024-08-29T15:21:49.308041+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"249'10134","reported_seq":10874,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:50.353726+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:21:50.353726+0000","last_peered":"2024-08-29T15:21:50.353726+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:21:50.353726+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"178'3080","ondisk_log_start":"178'3080","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362676224,"num_objects":5534,"num_object_clones":0,"num_object_copies":16602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5500,"num_objects_unfound":0,"num_objects_dirty":5534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10134,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3383}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":371,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.311031+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:21:49.311031+0000","last_peered":"2024-08-29T15:21:49.311031+0000","last_clean":"2024-08-29T15:21:49.311031+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:21:49.311031+0000","last_undegraded":"2024-08-29T15:21:49.311031+0000","last_fullsized":"2024-08-29T15:21:49.311031+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":675,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:40.341358+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:21:40.341358+0000","last_peered":"2024-08-29T15:21:40.341358+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:21:40.341358+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:21:40.341358+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":363,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316315+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:21:49.316315+0000","last_peered":"2024-08-29T15:21:49.316315+0000","last_clean":"2024-08-29T15:21:49.316315+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:21:49.316315+0000","last_undegraded":"2024-08-29T15:21:49.316315+0000","last_fullsized":"2024-08-29T15:21:49.316315+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"248'10043","reported_seq":17175,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.257952+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:21:49.257952+0000","last_peered":"2024-08-29T15:21:49.257952+0000","last_clean":"2024-08-29T15:21:49.257952+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:21:49.257952+0000","last_undegraded":"2024-08-29T15:21:49.257952+0000","last_fullsized":"2024-08-29T15:21:49.257952+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7644,"log_dups_size":0,"ondisk_log_size":7644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352911360,"num_objects":5385,"num_object_clones":0,"num_object_copies":16155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10043,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":356,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.305619+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:21:49.305619+0000","last_peered":"2024-08-29T15:21:49.305619+0000","last_clean":"2024-08-29T15:21:49.305619+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:21:49.305619+0000","last_undegraded":"2024-08-29T15:21:49.305619+0000","last_fullsized":"2024-08-29T15:21:49.305619+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":979,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:49.309248+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:21:49.309248+0000","last_peered":"2024-08-29T15:21:49.309248+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:21:49.309248+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":354,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.304975+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:21:49.304975+0000","last_peered":"2024-08-29T15:21:49.304975+0000","last_clean":"2024-08-29T15:21:49.304975+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:21:49.304975+0000","last_undegraded":"2024-08-29T15:21:49.304975+0000","last_fullsized":"2024-08-29T15:21:49.304975+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"249'10145","reported_seq":12733,"reported_epoch":249,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:21:54.317114+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:21:54.317114+0000","last_peered":"2024-08-29T15:21:54.317114+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:21:54.317114+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"177'2612","ondisk_log_start":"177'2612","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7533,"log_dups_size":0,"ondisk_log_size":7533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358154240,"num_objects":5465,"num_object_clones":0,"num_object_copies":16395,"num_objects_missing_on_primary":6501,"num_objects_missing":6501,"num_objects_degraded":2313,"num_objects_misplaced":5452,"num_objects_unfound":0,"num_objects_dirty":5465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10145,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1221,"num_bytes_recovered":79888384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":3081},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":352,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316363+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:21:49.316363+0000","last_peered":"2024-08-29T15:21:49.316363+0000","last_clean":"2024-08-29T15:21:49.316363+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:21:49.316363+0000","last_undegraded":"2024-08-29T15:21:49.316363+0000","last_fullsized":"2024-08-29T15:21:49.316363+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":99,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.304166+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:21:40.304166+0000","last_peered":"2024-08-29T15:21:40.304166+0000","last_clean":"2024-08-29T15:21:40.304166+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:21:40.304166+0000","last_undegraded":"2024-08-29T15:21:40.304166+0000","last_fullsized":"2024-08-29T15:21:40.304166+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266123+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:21:40.266123+0000","last_peered":"2024-08-29T15:21:40.266123+0000","last_clean":"2024-08-29T15:21:40.266123+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:21:40.266123+0000","last_undegraded":"2024-08-29T15:21:40.266123+0000","last_fullsized":"2024-08-29T15:21:40.266123+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266269+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:21:40.266269+0000","last_peered":"2024-08-29T15:21:40.266269+0000","last_clean":"2024-08-29T15:21:40.266269+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:21:40.266269+0000","last_undegraded":"2024-08-29T15:21:40.266269+0000","last_fullsized":"2024-08-29T15:21:40.266269+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":24,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:40.266509+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:21:40.266509+0000","last_peered":"2024-08-29T15:21:40.266509+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:21:40.266509+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:21:40.266509+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":361,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307943+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:21:49.307943+0000","last_peered":"2024-08-29T15:21:49.307943+0000","last_clean":"2024-08-29T15:21:49.307943+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:21:49.307943+0000","last_undegraded":"2024-08-29T15:21:49.307943+0000","last_fullsized":"2024-08-29T15:21:49.307943+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":111,"reported_epoch":249,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:49.307031+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:21:49.307031+0000","last_peered":"2024-08-29T15:21:49.307031+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:21:49.307031+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:21:49.307031+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":115,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.264751+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:21:40.264751+0000","last_peered":"2024-08-29T15:21:40.264751+0000","last_clean":"2024-08-29T15:21:40.264751+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:21:40.264751+0000","last_undegraded":"2024-08-29T15:21:40.264751+0000","last_fullsized":"2024-08-29T15:21:40.264751+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":320,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.304372+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:21:49.304372+0000","last_peered":"2024-08-29T15:21:49.304372+0000","last_clean":"2024-08-29T15:21:49.304372+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:21:49.304372+0000","last_undegraded":"2024-08-29T15:21:49.304372+0000","last_fullsized":"2024-08-29T15:21:49.304372+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":374,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316197+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:21:49.316197+0000","last_peered":"2024-08-29T15:21:49.316197+0000","last_clean":"2024-08-29T15:21:49.316197+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:21:49.316197+0000","last_undegraded":"2024-08-29T15:21:49.316197+0000","last_fullsized":"2024-08-29T15:21:49.316197+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":372,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.304891+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:21:49.304891+0000","last_peered":"2024-08-29T15:21:49.304891+0000","last_clean":"2024-08-29T15:21:49.304891+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:21:49.304891+0000","last_undegraded":"2024-08-29T15:21:49.304891+0000","last_fullsized":"2024-08-29T15:21:49.304891+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.304134+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:21:40.304134+0000","last_peered":"2024-08-29T15:21:40.304134+0000","last_clean":"2024-08-29T15:21:40.304134+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:21:40.304134+0000","last_undegraded":"2024-08-29T15:21:40.304134+0000","last_fullsized":"2024-08-29T15:21:40.304134+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":229,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.315938+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:21:49.315938+0000","last_peered":"2024-08-29T15:21:49.315938+0000","last_clean":"2024-08-29T15:21:49.315938+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:21:49.315938+0000","last_undegraded":"2024-08-29T15:21:49.315938+0000","last_fullsized":"2024-08-29T15:21:49.315938+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":349,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316959+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:21:49.316959+0000","last_peered":"2024-08-29T15:21:49.316959+0000","last_clean":"2024-08-29T15:21:49.316959+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:21:49.316959+0000","last_undegraded":"2024-08-29T15:21:49.316959+0000","last_fullsized":"2024-08-29T15:21:49.316959+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":352,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.310667+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:21:49.310667+0000","last_peered":"2024-08-29T15:21:49.310667+0000","last_clean":"2024-08-29T15:21:49.310667+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:21:49.310667+0000","last_undegraded":"2024-08-29T15:21:49.310667+0000","last_fullsized":"2024-08-29T15:21:49.310667+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":357,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.306903+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:21:49.306903+0000","last_peered":"2024-08-29T15:21:49.306903+0000","last_clean":"2024-08-29T15:21:49.306903+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:21:49.306903+0000","last_undegraded":"2024-08-29T15:21:49.306903+0000","last_fullsized":"2024-08-29T15:21:49.306903+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266131+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:21:40.266131+0000","last_peered":"2024-08-29T15:21:40.266131+0000","last_clean":"2024-08-29T15:21:40.266131+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:21:40.266131+0000","last_undegraded":"2024-08-29T15:21:40.266131+0000","last_fullsized":"2024-08-29T15:21:40.266131+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":377,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.311156+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:21:49.311156+0000","last_peered":"2024-08-29T15:21:49.311156+0000","last_clean":"2024-08-29T15:21:49.311156+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:21:49.311156+0000","last_undegraded":"2024-08-29T15:21:49.311156+0000","last_fullsized":"2024-08-29T15:21:49.311156+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":340,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316227+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:21:49.316227+0000","last_peered":"2024-08-29T15:21:49.316227+0000","last_clean":"2024-08-29T15:21:49.316227+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:21:49.316227+0000","last_undegraded":"2024-08-29T15:21:49.316227+0000","last_fullsized":"2024-08-29T15:21:49.316227+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266114+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:21:40.266114+0000","last_peered":"2024-08-29T15:21:40.266114+0000","last_clean":"2024-08-29T15:21:40.266114+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:21:40.266114+0000","last_undegraded":"2024-08-29T15:21:40.266114+0000","last_fullsized":"2024-08-29T15:21:40.266114+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":139,"reported_epoch":248,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:40.265058+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:21:40.265058+0000","last_peered":"2024-08-29T15:21:40.265058+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:21:40.265058+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:21:40.265058+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.266214+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:21:40.266214+0000","last_peered":"2024-08-29T15:21:40.266214+0000","last_clean":"2024-08-29T15:21:40.266214+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:21:40.266214+0000","last_undegraded":"2024-08-29T15:21:40.266214+0000","last_fullsized":"2024-08-29T15:21:40.266214+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":145,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.265042+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:21:40.265042+0000","last_peered":"2024-08-29T15:21:40.265042+0000","last_clean":"2024-08-29T15:21:40.265042+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:21:40.265042+0000","last_undegraded":"2024-08-29T15:21:40.265042+0000","last_fullsized":"2024-08-29T15:21:40.265042+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":388,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.304784+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:21:49.304784+0000","last_peered":"2024-08-29T15:21:49.304784+0000","last_clean":"2024-08-29T15:21:49.304784+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:21:49.304784+0000","last_undegraded":"2024-08-29T15:21:49.304784+0000","last_fullsized":"2024-08-29T15:21:49.304784+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":389,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.305380+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:21:49.305380+0000","last_peered":"2024-08-29T15:21:49.305380+0000","last_clean":"2024-08-29T15:21:49.305380+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:21:49.305380+0000","last_undegraded":"2024-08-29T15:21:49.305380+0000","last_fullsized":"2024-08-29T15:21:49.305380+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":26,"reported_epoch":248,"state":"active+clean","last_fresh":"2024-08-29T15:21:40.265931+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:21:40.265931+0000","last_peered":"2024-08-29T15:21:40.265931+0000","last_clean":"2024-08-29T15:21:40.265931+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:21:40.265931+0000","last_undegraded":"2024-08-29T15:21:40.265931+0000","last_fullsized":"2024-08-29T15:21:40.265931+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":397,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316142+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:21:49.316142+0000","last_peered":"2024-08-29T15:21:49.316142+0000","last_clean":"2024-08-29T15:21:49.316142+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:21:49.316142+0000","last_undegraded":"2024-08-29T15:21:49.316142+0000","last_fullsized":"2024-08-29T15:21:49.316142+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5796397080,"num_objects":88447,"num_object_clones":0,"num_object_copies":265341,"num_objects_missing_on_primary":6501,"num_objects_missing":6501,"num_objects_degraded":30073,"num_objects_misplaced":16507,"num_objects_unfound":0,"num_objects_dirty":88447,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162403,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14848,"num_bytes_recovered":853082112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17294503936,"data_stored":17294491720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122680,"ondisk_log_size":122680,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234326,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3145900,"kb_used_data":2848160,"kb_used_omap":1,"kb_used_meta":297726,"kb_avail":90599252,"statfs":{"total":95995035648,"available":92773634048,"internally_reserved":0,"allocated":2916515840,"data_stored":2915114907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":304871883},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495959,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":420196,"kb_used_data":254556,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":93324956,"statfs":{"total":95995035648,"available":95564754944,"internally_reserved":0,"allocated":260665344,"data_stored":259268211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":169605575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,2,0,3,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426544,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1837528,"kb_used_data":1770128,"kb_used_omap":8,"kb_used_meta":67383,"kb_avail":91907624,"statfs":{"total":95995035648,"available":94113406976,"internally_reserved":0,"allocated":1812611072,"data_stored":1811221405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":69001098},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019236,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2046924,"kb_used_data":1767232,"kb_used_omap":1,"kb_used_meta":279678,"kb_avail":91698228,"statfs":{"total":95995035648,"available":93898985472,"internally_reserved":0,"allocated":1809645568,"data_stored":1808271684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":286390726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435421,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3794200,"kb_used_data":3532108,"kb_used_omap":4,"kb_used_meta":262075,"kb_avail":89950952,"statfs":{"total":95995035648,"available":92109774848,"internally_reserved":0,"allocated":3616878592,"data_stored":3615492865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":268365065},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903273,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3238044,"kb_used_data":3128528,"kb_used_omap":10,"kb_used_meta":109493,"kb_avail":90507108,"statfs":{"total":95995035648,"available":92679278592,"internally_reserved":0,"allocated":3203612672,"data_stored":3202226881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":112121360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762474,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2454296,"kb_used_data":2206028,"kb_used_omap":7,"kb_used_meta":248248,"kb_avail":91290856,"statfs":{"total":95995035648,"available":93481836544,"internally_reserved":0,"allocated":2258972672,"data_stored":2257585705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":254206306},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926001,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1734108,"kb_used_data":1431700,"kb_used_omap":7,"kb_used_meta":302392,"kb_avail":92011044,"statfs":{"total":95995035648,"available":94219309056,"internally_reserved":0,"allocated":1466060800,"data_stored":1464666612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":309649551},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1456345088,"data_stored":1456341016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2250313728,"data_stored":2250309656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3198812160,"data_stored":3198812160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3613261824,"data_stored":3613261824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1805582336,"data_stored":1805582336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1807159296,"data_stored":1807155224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":256573440,"data_stored":256573440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2906456064,"data_stored":2906456064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:21:57.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:21:57.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:57.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:21:57.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:57.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:57.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:57.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:57.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/265425 objects degraded (11.341%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:21:57.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:57.796 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:57.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:57.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:57.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:57.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:57.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:57.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:21:58.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:58.047 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:58.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:58.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:58.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:58.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:58.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:58.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:58.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:58.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:58.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:58 smithi027 ceph-mon[98970]: pgmap v776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/264552 objects degraded (11.378%); 16465/264552 objects misplaced (6.224%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:58 smithi027 ceph-mon[93921]: pgmap v776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/264552 objects degraded (11.378%); 16465/264552 objects misplaced (6.224%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:58.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:58.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:58.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:58 smithi171 ceph-mon[90031]: pgmap v776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/264552 objects degraded (11.378%); 16465/264552 objects misplaced (6.224%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:21:58.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:21:58.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:58.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:21:58.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:21:58.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:58.902 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:21:58.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:58.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:59.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:59.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:59.148 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:21:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:21:59] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-08-29T15:21:59.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:21:59.175 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:59.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:21:59.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:21:59.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:21:59.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:21:59.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:59.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:21:59.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:21:59.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:59.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:21:59.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:21:59.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:21:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:21:59.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:59.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:59.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:21:59.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:21:59.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:21:59.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:21:59.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:59 smithi171 ceph-mon[90031]: pgmap v777: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/263919 objects degraded (11.406%); 16443/263919 objects misplaced (6.230%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:21:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:00.028 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:00.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:59 smithi027 ceph-mon[98970]: pgmap v777: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/263919 objects degraded (11.406%); 16443/263919 objects misplaced (6.230%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:21:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:59 smithi027 ceph-mon[93921]: pgmap v777: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/263919 objects degraded (11.406%); 16443/263919 objects misplaced (6.230%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:21:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:00.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:00.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:00.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:00.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:00.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:00.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:00.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:00.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:00.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:00.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:00.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:00.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:00.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:00.670 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:00.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:00.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:00.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:00.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:00.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:00.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:01.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:01.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:01.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:01.150 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:01.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:01.179 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:01.180 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:01.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:01.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:01.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:01.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:01.495 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:01.495 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/263241 objects degraded (11.435%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1103,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":249,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":87747,"data_bytes":5753477117,"bytes_used":18981265408,"bytes_avail":748979019776,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":263241,"degraded_ratio":0.11435148780015272,"misplaced_objects":16404,"misplaced_total":263241,"misplaced_ratio":0.062315520758544449,"recovering_objects_per_sec":10,"recovering_bytes_per_sec":675170,"recovering_keys_per_sec":0,"num_objects_recovered":116,"num_bytes_recovered":7602176,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":131},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:22:01.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:01.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:01.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:01.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:01.702 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[93921]: pgmap v778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/263241 objects degraded (11.435%); 16404/263241 objects misplaced (6.232%); 659 KiB/s, 10 objects/s recovering 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4146636939' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1985289277' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[98970]: pgmap v778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/263241 objects degraded (11.435%); 16404/263241 objects misplaced (6.232%); 659 KiB/s, 10 objects/s recovering 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4146636939' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:01.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1985289277' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:01.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:01.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:01.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:01.923 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:01.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:01.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:01 smithi171 ceph-mon[90031]: pgmap v778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/263241 objects degraded (11.435%); 16404/263241 objects misplaced (6.232%); 659 KiB/s, 10 objects/s recovering 2024-08-29T15:22:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4146636939' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:01 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1985289277' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:02.025 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:02.047 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:02.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:02.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:02.171 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:02.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:02.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:02.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:02.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:02.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:02.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:02.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:02.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:02.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:02.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:02.799 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:02.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[98970]: osdmap e250: 8 total, 8 up, 8 in 2024-08-29T15:22:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/263241 objects degraded (11.435%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:02.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:02.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:02.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[93921]: osdmap e250: 8 total, 8 up, 8 in 2024-08-29T15:22:02.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/263241 objects degraded (11.435%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:02.863 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:22:02.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:22:02.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:02.923 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:02.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:02.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:02 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:02 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:02 smithi171 ceph-mon[90031]: osdmap e250: 8 total, 8 up, 8 in 2024-08-29T15:22:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/263241 objects degraded (11.435%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:03.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:03.047 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:03.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:03.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:03.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:03.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:03.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:03.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:03.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:03.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:03.543 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:03.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:03.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:03.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:03.792 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:03.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:03 smithi027 ceph-mon[93921]: pgmap v780: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/261831 objects degraded (11.497%); 16317/261831 objects misplaced (6.232%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:22:03.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:03 smithi027 ceph-mon[98970]: pgmap v780: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/261831 objects degraded (11.497%); 16317/261831 objects misplaced (6.232%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:22:03.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:03.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:03.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:03 smithi171 ceph-mon[90031]: pgmap v780: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/261831 objects degraded (11.497%); 16317/261831 objects misplaced (6.232%); 499 KiB/s, 7 objects/s recovering 2024-08-29T15:22:04.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:04.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:04.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:04.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:04.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:04.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:04.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:04.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:04.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:04.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:04.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:04.543 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:04.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:04.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:04.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:04.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:04.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:04.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:04.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:04.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:04.954 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:05.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:05.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:05.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:05.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:05.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:05.289 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:05.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:05.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:05.414 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:05.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:05.494 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:22:05.494 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:22:05.513 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:22:05.513 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:22:05.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:05.536 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:22:05.536 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:22:05.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:05.554 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:22:05.554 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:22:05.570 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:22:05.570 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:22:05.587 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:22:05.588 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:22:05.608 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:22:05.609 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:22:05.629 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:22:05.629 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:22:05.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:05.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:05.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:05.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:05.796 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:05.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:05.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:05.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:05.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:05.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:22:05.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:05 smithi027 ceph-mon[93921]: pgmap v781: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/261588 objects degraded (11.507%); 16317/261588 objects misplaced (6.238%); 250 KiB/s, 3 objects/s recovering 2024-08-29T15:22:05.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1440234774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:05.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:05 smithi027 ceph-mon[98970]: pgmap v781: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/261588 objects degraded (11.507%); 16317/261588 objects misplaced (6.238%); 250 KiB/s, 3 objects/s recovering 2024-08-29T15:22:05.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1440234774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:05 smithi171 ceph-mon[90031]: pgmap v781: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/261588 objects degraded (11.507%); 16317/261588 objects misplaced (6.238%); 250 KiB/s, 3 objects/s recovering 2024-08-29T15:22:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1440234774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:06.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:06.044 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:06.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:06.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:06.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:06.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:06.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:06.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:06.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:06.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:06.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:06.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:06.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:06.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:06.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:06.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:06.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:06.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:06.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:06.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:06.895 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:06.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:06.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:06.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:07.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:07.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:07.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:07.205 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:07.306 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:07.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:07.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:07.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:07.460 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:07.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:07.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:07.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:07.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:07.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:07.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:07.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:07 smithi027 ceph-mon[93921]: pgmap v782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/260772 objects degraded (11.543%); 16239/260772 objects misplaced (6.227%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:07.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/260772 objects degraded (11.543%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:07.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:07 smithi027 ceph-mon[98970]: pgmap v782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/260772 objects degraded (11.543%); 16239/260772 objects misplaced (6.227%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:07.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/260772 objects degraded (11.543%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:07.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:07.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:07.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:07.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:07.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:07 smithi171 ceph-mon[90031]: pgmap v782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/260772 objects degraded (11.543%); 16239/260772 objects misplaced (6.227%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/260772 objects degraded (11.543%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:08.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:08.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:08.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:08.201 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:08.207 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:22:08.207 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:22:08.207 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:22:08.207 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:22:08.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:08.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:08.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:08.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:08.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:08.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:08.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:08.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:08.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:08.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:08.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:08.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:08.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:08.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:08.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:09.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:09.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:09.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:09.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:09.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:09.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:22:09.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:22:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:22:09] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-08-29T15:22:09.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:09.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:09.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:22:09.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:09.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:09.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:09.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:09.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:09.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:09.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:09.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:09.812 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:09.833 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:09.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:09.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:09.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:09.967 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:09 smithi027 ceph-mon[98970]: pgmap v783: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/260100 objects degraded (11.573%); 16195/260100 objects misplaced (6.226%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:09.967 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:09 smithi027 ceph-mon[93921]: pgmap v783: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/260100 objects degraded (11.573%); 16195/260100 objects misplaced (6.226%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:09 smithi171 ceph-mon[90031]: pgmap v783: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/260100 objects degraded (11.573%); 16195/260100 objects misplaced (6.226%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:10.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:10.081 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:10.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:10.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:10.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:10.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:10.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:10.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:10.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:10.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:10.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:10.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:10.593 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:22:10.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:10.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:10.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:10.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:10.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:10.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:10.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:11.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:11.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:11.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:11.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:11.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:11.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:11.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:11.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:11.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:11.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:11.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:11.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:11.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:11.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:11.845 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:11.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:11.970 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:11.970 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:11.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:11.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:11.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:11 smithi171 ceph-mon[90031]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/259485 objects degraded (11.601%); 16155/259485 objects misplaced (6.226%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:11 smithi171 ceph-mon[90031]: from='client.35574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:11 smithi027 ceph-mon[98970]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/259485 objects degraded (11.601%); 16155/259485 objects misplaced (6.226%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:12.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:11 smithi027 ceph-mon[98970]: from='client.35574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:12.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:11 smithi027 ceph-mon[93921]: pgmap v784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/259485 objects degraded (11.601%); 16155/259485 objects misplaced (6.226%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:12.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:11 smithi027 ceph-mon[93921]: from='client.35574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:12.072 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:12.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:12.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:12.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:12.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:12.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:12.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:12.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:12.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:12.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:12.495 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":784,"stamp":"2024-08-29T15:22:10.568320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5671426045,"num_objects":86495,"num_object_clones":0,"num_object_copies":259485,"num_objects_missing_on_primary":6388,"num_objects_missing":6388,"num_objects_degraded":30102,"num_objects_misplaced":16155,"num_objects_unfound":0,"num_objects_dirty":86495,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":165506,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15077,"num_bytes_recovered":866913485,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124142,"ondisk_log_size":124142,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18307784,"kb_used_data":16564936,"kb_used_omap":43,"kb_used_meta":1742676,"kb_avail":731653432,"statfs":{"total":767960285184,"available":749213114368,"internally_reserved":0,"allocated":16962494464,"data_stored":16951318990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1784500716},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,0,1,0,1,2,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-82051072,"num_objects":-1252,"num_object_clones":0,"num_object_copies":-3756,"num_objects_missing_on_primary":-82,"num_objects_missing":-82,"num_objects_degraded":0,"num_objects_misplaced":-249,"num_objects_unfound":0,"num_objects_dirty":-1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1252,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":81,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002720"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":384,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698771+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:22:01.698771+0000","last_peered":"2024-08-29T15:22:01.698771+0000","last_clean":"2024-08-29T15:22:01.698771+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:22:01.698771+0000","last_undegraded":"2024-08-29T15:22:01.698771+0000","last_fullsized":"2024-08-29T15:22:01.698771+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":27,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:01.698985+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:22:01.698985+0000","last_peered":"2024-08-29T15:22:01.698985+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:22:01.698985+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:22:01.698985+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":116,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307253+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:21:49.307253+0000","last_peered":"2024-08-29T15:21:49.307253+0000","last_clean":"2024-08-29T15:21:49.307253+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:21:49.307253+0000","last_undegraded":"2024-08-29T15:21:49.307253+0000","last_fullsized":"2024-08-29T15:21:49.307253+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":350,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763305+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:22:01.763305+0000","last_peered":"2024-08-29T15:22:01.763305+0000","last_clean":"2024-08-29T15:22:01.763305+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:22:01.763305+0000","last_undegraded":"2024-08-29T15:22:01.763305+0000","last_fullsized":"2024-08-29T15:22:01.763305+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":350,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763612+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:22:01.763612+0000","last_peered":"2024-08-29T15:22:01.763612+0000","last_clean":"2024-08-29T15:22:01.763612+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:22:01.763612+0000","last_undegraded":"2024-08-29T15:22:01.763612+0000","last_fullsized":"2024-08-29T15:22:01.763612+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698953+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:22:01.698953+0000","last_peered":"2024-08-29T15:22:01.698953+0000","last_clean":"2024-08-29T15:22:01.698953+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:22:01.698953+0000","last_undegraded":"2024-08-29T15:22:01.698953+0000","last_fullsized":"2024-08-29T15:22:01.698953+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":161,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700497+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:22:01.700497+0000","last_peered":"2024-08-29T15:22:01.700497+0000","last_clean":"2024-08-29T15:22:01.700497+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:22:01.700497+0000","last_undegraded":"2024-08-29T15:22:01.700497+0000","last_fullsized":"2024-08-29T15:22:01.700497+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":118,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316264+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:21:49.316264+0000","last_peered":"2024-08-29T15:21:49.316264+0000","last_clean":"2024-08-29T15:21:49.316264+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:21:49.316264+0000","last_undegraded":"2024-08-29T15:21:49.316264+0000","last_fullsized":"2024-08-29T15:21:49.316264+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"250'10415","reported_seq":18443,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:06.051740+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:22:06.051740+0000","last_peered":"2024-08-29T15:22:06.051740+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:22:06.051740+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'400","ondisk_log_start":"175'400","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10015,"log_dups_size":0,"ondisk_log_size":10015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360251392,"num_objects":5497,"num_object_clones":0,"num_object_copies":16491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10415,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2572}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":26,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:01.699734+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:22:01.699734+0000","last_peered":"2024-08-29T15:22:01.699734+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:22:01.699734+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:22:01.699734+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":135,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.159481+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:22:01.159481+0000","last_peered":"2024-08-29T15:22:01.159481+0000","last_clean":"2024-08-29T15:22:01.159481+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:22:01.159481+0000","last_undegraded":"2024-08-29T15:22:01.159481+0000","last_fullsized":"2024-08-29T15:22:01.159481+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"250'10162","reported_seq":10340,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:06.758397+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:22:06.758397+0000","last_peered":"2024-08-29T15:22:06.758397+0000","last_clean":"2024-08-29T15:22:06.758397+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:22:06.758397+0000","last_undegraded":"2024-08-29T15:22:06.758397+0000","last_fullsized":"2024-08-29T15:22:06.758397+0000","mapping_epoch":168,"log_start":"178'3428","ondisk_log_start":"178'3428","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353107968,"num_objects":5388,"num_object_clones":0,"num_object_copies":16164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10162,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":26,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:01.699770+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:22:01.699770+0000","last_peered":"2024-08-29T15:22:01.699770+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:22:01.699770+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:22:01.699770+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":28,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:59.784720+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:21:59.784720+0000","last_peered":"2024-08-29T15:21:59.784720+0000","last_clean":"2024-08-29T15:21:59.784720+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:21:59.784720+0000","last_undegraded":"2024-08-29T15:21:59.784720+0000","last_fullsized":"2024-08-29T15:21:59.784720+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"250'10317","reported_seq":10464,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:06.768157+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:22:06.768157+0000","last_peered":"2024-08-29T15:22:06.768157+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:22:06.768157+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'3574","ondisk_log_start":"178'3574","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353566720,"num_objects":5395,"num_object_clones":0,"num_object_copies":16185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10317,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2918}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":166,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.766078+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:22:01.766078+0000","last_peered":"2024-08-29T15:22:01.766078+0000","last_clean":"2024-08-29T15:22:01.766078+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:22:01.766078+0000","last_undegraded":"2024-08-29T15:22:01.766078+0000","last_fullsized":"2024-08-29T15:22:01.766078+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":73,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.315850+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:21:49.315850+0000","last_peered":"2024-08-29T15:21:49.315850+0000","last_clean":"2024-08-29T15:21:49.315850+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:21:49.315850+0000","last_undegraded":"2024-08-29T15:21:49.315850+0000","last_fullsized":"2024-08-29T15:21:49.315850+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"250'10419","reported_seq":15817,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:08.523648+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:22:08.523648+0000","last_peered":"2024-08-29T15:22:08.523648+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:22:08.523648+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"176'2262","ondisk_log_start":"176'2262","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8157,"log_dups_size":0,"ondisk_log_size":8157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351207424,"num_objects":5359,"num_object_clones":0,"num_object_copies":16077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10419,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5359}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":135,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700563+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:22:01.700563+0000","last_peered":"2024-08-29T15:22:01.700563+0000","last_clean":"2024-08-29T15:22:01.700563+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:22:01.700563+0000","last_undegraded":"2024-08-29T15:22:01.700563+0000","last_fullsized":"2024-08-29T15:22:01.700563+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":364,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.764405+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:22:01.764405+0000","last_peered":"2024-08-29T15:22:01.764405+0000","last_clean":"2024-08-29T15:22:01.764405+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:22:01.764405+0000","last_undegraded":"2024-08-29T15:22:01.764405+0000","last_fullsized":"2024-08-29T15:22:01.764405+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"250'10346","reported_seq":10482,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:09.348464+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:22:09.348464+0000","last_peered":"2024-08-29T15:22:09.348464+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:22:09.348464+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:22:09.348464+0000","mapping_epoch":234,"log_start":"177'2433","ondisk_log_start":"177'2433","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7913,"log_dups_size":0,"ondisk_log_size":7913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358350848,"num_objects":5468,"num_object_clones":0,"num_object_copies":16404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10346,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2991},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":358,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698417+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:22:01.698417+0000","last_peered":"2024-08-29T15:22:01.698417+0000","last_clean":"2024-08-29T15:22:01.698417+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:22:01.698417+0000","last_undegraded":"2024-08-29T15:22:01.698417+0000","last_fullsized":"2024-08-29T15:22:01.698417+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":28,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.699226+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:22:01.699226+0000","last_peered":"2024-08-29T15:22:01.699226+0000","last_clean":"2024-08-29T15:22:01.699226+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:22:01.699226+0000","last_undegraded":"2024-08-29T15:22:01.699226+0000","last_fullsized":"2024-08-29T15:22:01.699226+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"250'10339","reported_seq":10533,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:06.815873+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:22:06.815873+0000","last_peered":"2024-08-29T15:22:06.815873+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:22:06.815873+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:22:06.815873+0000","mapping_epoch":234,"log_start":"179'3668","ondisk_log_start":"179'3668","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6671,"log_dups_size":0,"ondisk_log_size":6671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360644608,"num_objects":5503,"num_object_clones":0,"num_object_copies":16509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10339,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":3038},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.315729+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:21:49.315729+0000","last_peered":"2024-08-29T15:21:49.315729+0000","last_clean":"2024-08-29T15:21:49.315729+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:21:49.315729+0000","last_undegraded":"2024-08-29T15:21:49.315729+0000","last_fullsized":"2024-08-29T15:21:49.315729+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":28,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.699169+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:22:01.699169+0000","last_peered":"2024-08-29T15:22:01.699169+0000","last_clean":"2024-08-29T15:22:01.699169+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:22:01.699169+0000","last_undegraded":"2024-08-29T15:22:01.699169+0000","last_fullsized":"2024-08-29T15:22:01.699169+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"250'10216","reported_seq":10595,"reported_epoch":250,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:22:06.867000+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:22:06.867000+0000","last_peered":"2024-08-29T15:22:06.867000+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:22:06.867000+0000","last_undegraded":"2024-08-29T15:22:06.867000+0000","last_fullsized":"2024-08-29T15:22:06.867000+0000","mapping_epoch":202,"log_start":"178'3467","ondisk_log_start":"178'3467","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356646912,"num_objects":5442,"num_object_clones":0,"num_object_copies":16326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5442,"num_objects_unfound":0,"num_objects_dirty":5442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10216,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5442}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":28,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:59.784732+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:21:59.784732+0000","last_peered":"2024-08-29T15:21:59.784732+0000","last_clean":"2024-08-29T15:21:59.784732+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:21:59.784732+0000","last_undegraded":"2024-08-29T15:21:59.784732+0000","last_fullsized":"2024-08-29T15:21:59.784732+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.311762+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:21:49.311762+0000","last_peered":"2024-08-29T15:21:49.311762+0000","last_clean":"2024-08-29T15:21:49.311762+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:21:49.311762+0000","last_undegraded":"2024-08-29T15:21:49.311762+0000","last_fullsized":"2024-08-29T15:21:49.311762+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"250'10305","reported_seq":10434,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:06.097391+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:22:06.097391+0000","last_peered":"2024-08-29T15:22:06.097391+0000","last_clean":"2024-08-29T15:22:06.097391+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:22:06.097391+0000","last_undegraded":"2024-08-29T15:22:06.097391+0000","last_fullsized":"2024-08-29T15:22:06.097391+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9027,"log_dups_size":0,"ondisk_log_size":9027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357433368,"num_objects":5455,"num_object_clones":0,"num_object_copies":16365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5455,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10305,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":362,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.699299+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:22:01.699299+0000","last_peered":"2024-08-29T15:22:01.699299+0000","last_clean":"2024-08-29T15:22:01.699299+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:22:01.699299+0000","last_undegraded":"2024-08-29T15:22:01.699299+0000","last_fullsized":"2024-08-29T15:22:01.699299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":354,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700571+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:22:01.700571+0000","last_peered":"2024-08-29T15:22:01.700571+0000","last_clean":"2024-08-29T15:22:01.700571+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:22:01.700571+0000","last_undegraded":"2024-08-29T15:22:01.700571+0000","last_fullsized":"2024-08-29T15:22:01.700571+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"250'9958","reported_seq":10136,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:05.883989+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:22:05.883989+0000","last_peered":"2024-08-29T15:22:05.883989+0000","last_clean":"2024-08-29T15:22:05.883989+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:22:05.883989+0000","last_undegraded":"2024-08-29T15:22:05.883989+0000","last_fullsized":"2024-08-29T15:22:05.883989+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9958,"log_dups_size":0,"ondisk_log_size":9958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344064000,"num_objects":5250,"num_object_clones":0,"num_object_copies":15750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9958,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":452,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.764147+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:22:01.764147+0000","last_peered":"2024-08-29T15:22:01.764147+0000","last_clean":"2024-08-29T15:22:01.764147+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:22:01.764147+0000","last_undegraded":"2024-08-29T15:22:01.764147+0000","last_fullsized":"2024-08-29T15:22:01.764147+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":719,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:51.158430+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:21:51.158430+0000","last_peered":"2024-08-29T15:21:51.158430+0000","last_clean":"2024-08-29T15:21:51.158430+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:21:51.158430+0000","last_undegraded":"2024-08-29T15:21:51.158430+0000","last_fullsized":"2024-08-29T15:21:51.158430+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":227,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307602+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:21:49.307602+0000","last_peered":"2024-08-29T15:21:49.307602+0000","last_clean":"2024-08-29T15:21:49.307602+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:21:49.307602+0000","last_undegraded":"2024-08-29T15:21:49.307602+0000","last_fullsized":"2024-08-29T15:21:49.307602+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":89,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316505+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:21:49.316505+0000","last_peered":"2024-08-29T15:21:49.316505+0000","last_clean":"2024-08-29T15:21:49.316505+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:21:49.316505+0000","last_undegraded":"2024-08-29T15:21:49.316505+0000","last_fullsized":"2024-08-29T15:21:49.316505+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"250'10322","reported_seq":10698,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:05.685115+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:22:05.685115+0000","last_peered":"2024-08-29T15:22:05.685115+0000","last_clean":"2024-08-29T15:22:05.685115+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:22:05.685115+0000","last_undegraded":"2024-08-29T15:22:05.685115+0000","last_fullsized":"2024-08-29T15:22:05.685115+0000","mapping_epoch":201,"log_start":"178'3253","ondisk_log_start":"178'3253","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351272960,"num_objects":5360,"num_object_clones":0,"num_object_copies":16080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10322,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":826,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.699787+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:22:01.699787+0000","last_peered":"2024-08-29T15:22:01.699787+0000","last_clean":"2024-08-29T15:22:01.699787+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:22:01.699787+0000","last_undegraded":"2024-08-29T15:22:01.699787+0000","last_fullsized":"2024-08-29T15:22:01.699787+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":365,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763423+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:22:01.763423+0000","last_peered":"2024-08-29T15:22:01.763423+0000","last_clean":"2024-08-29T15:22:01.763423+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:22:01.763423+0000","last_undegraded":"2024-08-29T15:22:01.763423+0000","last_fullsized":"2024-08-29T15:22:01.763423+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":372,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763834+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:22:01.763834+0000","last_peered":"2024-08-29T15:22:01.763834+0000","last_clean":"2024-08-29T15:22:01.763834+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:22:01.763834+0000","last_undegraded":"2024-08-29T15:22:01.763834+0000","last_fullsized":"2024-08-29T15:22:01.763834+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"250'10210","reported_seq":16009,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:06.619140+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:22:06.619140+0000","last_peered":"2024-08-29T15:22:06.619140+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:22:06.619140+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'3530","ondisk_log_start":"178'3530","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359137280,"num_objects":5480,"num_object_clones":0,"num_object_copies":16440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10210,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":1046},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":635,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698355+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:22:01.698355+0000","last_peered":"2024-08-29T15:22:01.698355+0000","last_clean":"2024-08-29T15:22:01.698355+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:22:01.698355+0000","last_undegraded":"2024-08-29T15:22:01.698355+0000","last_fullsized":"2024-08-29T15:22:01.698355+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":79,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.315981+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:21:49.315981+0000","last_peered":"2024-08-29T15:21:49.315981+0000","last_clean":"2024-08-29T15:21:49.315981+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:21:49.315981+0000","last_undegraded":"2024-08-29T15:21:49.315981+0000","last_fullsized":"2024-08-29T15:21:49.315981+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":399,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700373+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:22:01.700373+0000","last_peered":"2024-08-29T15:22:01.700373+0000","last_clean":"2024-08-29T15:22:01.700373+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:22:01.700373+0000","last_undegraded":"2024-08-29T15:22:01.700373+0000","last_fullsized":"2024-08-29T15:22:01.700373+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"250'10282","reported_seq":10470,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:06.754906+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:22:06.754906+0000","last_peered":"2024-08-29T15:22:06.754906+0000","last_clean":"2024-08-29T15:22:06.754906+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:22:06.754906+0000","last_undegraded":"2024-08-29T15:22:06.754906+0000","last_fullsized":"2024-08-29T15:22:06.754906+0000","mapping_epoch":234,"log_start":"178'3597","ondisk_log_start":"178'3597","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355860480,"num_objects":5430,"num_object_clones":0,"num_object_copies":16290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10282,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":187,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307517+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:21:49.307517+0000","last_peered":"2024-08-29T15:21:49.307517+0000","last_clean":"2024-08-29T15:21:49.307517+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:21:49.307517+0000","last_undegraded":"2024-08-29T15:21:49.307517+0000","last_fullsized":"2024-08-29T15:21:49.307517+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":28,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:54.782186+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:21:54.782186+0000","last_peered":"2024-08-29T15:21:54.782186+0000","last_clean":"2024-08-29T15:21:54.782186+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:21:54.782186+0000","last_undegraded":"2024-08-29T15:21:54.782186+0000","last_fullsized":"2024-08-29T15:21:54.782186+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":94,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:51.158449+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:21:51.158449+0000","last_peered":"2024-08-29T15:21:51.158449+0000","last_clean":"2024-08-29T15:21:51.158449+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:21:51.158449+0000","last_undegraded":"2024-08-29T15:21:51.158449+0000","last_fullsized":"2024-08-29T15:21:51.158449+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"250'10364","reported_seq":10550,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:08.520224+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:22:08.520224+0000","last_peered":"2024-08-29T15:22:08.520224+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:22:08.520224+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"177'2242","ondisk_log_start":"177'2242","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357957632,"num_objects":5462,"num_object_clones":0,"num_object_copies":16386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10364,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3039}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":119,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:51.158485+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:21:51.158485+0000","last_peered":"2024-08-29T15:21:51.158485+0000","last_clean":"2024-08-29T15:21:51.158485+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:21:51.158485+0000","last_undegraded":"2024-08-29T15:21:51.158485+0000","last_fullsized":"2024-08-29T15:21:51.158485+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":791,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700472+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:22:01.700472+0000","last_peered":"2024-08-29T15:22:01.700472+0000","last_clean":"2024-08-29T15:22:01.700472+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:22:01.700472+0000","last_undegraded":"2024-08-29T15:22:01.700472+0000","last_fullsized":"2024-08-29T15:22:01.700472+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":368,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700467+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:22:01.700467+0000","last_peered":"2024-08-29T15:22:01.700467+0000","last_clean":"2024-08-29T15:22:01.700467+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:22:01.700467+0000","last_undegraded":"2024-08-29T15:22:01.700467+0000","last_fullsized":"2024-08-29T15:22:01.700467+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"250'10252","reported_seq":10994,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:05.691883+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:22:05.691883+0000","last_peered":"2024-08-29T15:22:05.691883+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:22:05.691883+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"178'3180","ondisk_log_start":"178'3180","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354942976,"num_objects":5416,"num_object_clones":0,"num_object_copies":16248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5382,"num_objects_unfound":0,"num_objects_dirty":5416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10252,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3265}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":373,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700374+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:22:01.700374+0000","last_peered":"2024-08-29T15:22:01.700374+0000","last_clean":"2024-08-29T15:22:01.700374+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:22:01.700374+0000","last_undegraded":"2024-08-29T15:22:01.700374+0000","last_fullsized":"2024-08-29T15:22:01.700374+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":676,"reported_epoch":249,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:49.316029+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:21:49.316029+0000","last_peered":"2024-08-29T15:21:49.316029+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:21:49.316029+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:21:49.316029+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":365,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.764304+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:22:01.764304+0000","last_peered":"2024-08-29T15:22:01.764304+0000","last_clean":"2024-08-29T15:22:01.764304+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:22:01.764304+0000","last_undegraded":"2024-08-29T15:22:01.764304+0000","last_fullsized":"2024-08-29T15:22:01.764304+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"250'10211","reported_seq":17345,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:09.463315+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:22:09.463315+0000","last_peered":"2024-08-29T15:22:09.463315+0000","last_clean":"2024-08-29T15:22:09.463315+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:22:09.463315+0000","last_undegraded":"2024-08-29T15:22:09.463315+0000","last_fullsized":"2024-08-29T15:22:09.463315+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7812,"log_dups_size":0,"ondisk_log_size":7812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341901312,"num_objects":5217,"num_object_clones":0,"num_object_copies":15651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10211,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":359,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.701556+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:22:01.701556+0000","last_peered":"2024-08-29T15:22:01.701556+0000","last_clean":"2024-08-29T15:22:01.701556+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:22:01.701556+0000","last_undegraded":"2024-08-29T15:22:01.701556+0000","last_fullsized":"2024-08-29T15:22:01.701556+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":981,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:01.699678+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:22:01.699678+0000","last_peered":"2024-08-29T15:22:01.699678+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:22:01.699678+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":356,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698080+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:22:01.698080+0000","last_peered":"2024-08-29T15:22:01.698080+0000","last_clean":"2024-08-29T15:22:01.698080+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:22:01.698080+0000","last_undegraded":"2024-08-29T15:22:01.698080+0000","last_fullsized":"2024-08-29T15:22:01.698080+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"250'10265","reported_seq":13096,"reported_epoch":250,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:09.709097+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:22:09.709097+0000","last_peered":"2024-08-29T15:22:09.709097+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:22:09.709097+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"177'2712","ondisk_log_start":"177'2712","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350289920,"num_objects":5345,"num_object_clones":0,"num_object_copies":16035,"num_objects_missing_on_primary":6380,"num_objects_missing":6380,"num_objects_degraded":2313,"num_objects_misplaced":5331,"num_objects_unfound":0,"num_objects_dirty":5345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10265,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1341,"num_bytes_recovered":87752704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":2960},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":354,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763516+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:22:01.763516+0000","last_peered":"2024-08-29T15:22:01.763516+0000","last_clean":"2024-08-29T15:22:01.763516+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:22:01.763516+0000","last_undegraded":"2024-08-29T15:22:01.763516+0000","last_fullsized":"2024-08-29T15:22:01.763516+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":100,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316581+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:21:49.316581+0000","last_peered":"2024-08-29T15:21:49.316581+0000","last_clean":"2024-08-29T15:21:49.316581+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:21:49.316581+0000","last_undegraded":"2024-08-29T15:21:49.316581+0000","last_fullsized":"2024-08-29T15:21:49.316581+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":28,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:54.782102+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:21:54.782102+0000","last_peered":"2024-08-29T15:21:54.782102+0000","last_clean":"2024-08-29T15:21:54.782102+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:21:54.782102+0000","last_undegraded":"2024-08-29T15:21:54.782102+0000","last_fullsized":"2024-08-29T15:21:54.782102+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":28,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.699013+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:22:01.699013+0000","last_peered":"2024-08-29T15:22:01.699013+0000","last_clean":"2024-08-29T15:22:01.699013+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:22:01.699013+0000","last_undegraded":"2024-08-29T15:22:01.699013+0000","last_fullsized":"2024-08-29T15:22:01.699013+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":26,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:01.699339+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:22:01.699339+0000","last_peered":"2024-08-29T15:22:01.699339+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:22:01.699339+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:22:01.699339+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":363,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700313+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:22:01.700313+0000","last_peered":"2024-08-29T15:22:01.700313+0000","last_clean":"2024-08-29T15:22:01.700313+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:22:01.700313+0000","last_undegraded":"2024-08-29T15:22:01.700313+0000","last_fullsized":"2024-08-29T15:22:01.700313+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":113,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:01.698858+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:22:01.698858+0000","last_peered":"2024-08-29T15:22:01.698858+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:22:01.698858+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:22:01.698858+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":116,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307451+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:21:49.307451+0000","last_peered":"2024-08-29T15:21:49.307451+0000","last_clean":"2024-08-29T15:21:49.307451+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:21:49.307451+0000","last_undegraded":"2024-08-29T15:21:49.307451+0000","last_fullsized":"2024-08-29T15:21:49.307451+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":322,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698102+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:22:01.698102+0000","last_peered":"2024-08-29T15:22:01.698102+0000","last_clean":"2024-08-29T15:22:01.698102+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:22:01.698102+0000","last_undegraded":"2024-08-29T15:22:01.698102+0000","last_fullsized":"2024-08-29T15:22:01.698102+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":376,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.764018+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:22:01.764018+0000","last_peered":"2024-08-29T15:22:01.764018+0000","last_clean":"2024-08-29T15:22:01.764018+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:22:01.764018+0000","last_undegraded":"2024-08-29T15:22:01.764018+0000","last_fullsized":"2024-08-29T15:22:01.764018+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":374,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698006+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:22:01.698006+0000","last_peered":"2024-08-29T15:22:01.698006+0000","last_clean":"2024-08-29T15:22:01.698006+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:22:01.698006+0000","last_undegraded":"2024-08-29T15:22:01.698006+0000","last_fullsized":"2024-08-29T15:22:01.698006+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.316594+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:21:49.316594+0000","last_peered":"2024-08-29T15:21:49.316594+0000","last_clean":"2024-08-29T15:21:49.316594+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:21:49.316594+0000","last_undegraded":"2024-08-29T15:21:49.316594+0000","last_fullsized":"2024-08-29T15:21:49.316594+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":231,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763190+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:22:01.763190+0000","last_peered":"2024-08-29T15:22:01.763190+0000","last_clean":"2024-08-29T15:22:01.763190+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:22:01.763190+0000","last_undegraded":"2024-08-29T15:22:01.763190+0000","last_fullsized":"2024-08-29T15:22:01.763190+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":351,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763745+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:22:01.763745+0000","last_peered":"2024-08-29T15:22:01.763745+0000","last_clean":"2024-08-29T15:22:01.763745+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:22:01.763745+0000","last_undegraded":"2024-08-29T15:22:01.763745+0000","last_fullsized":"2024-08-29T15:22:01.763745+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":354,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700160+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:22:01.700160+0000","last_peered":"2024-08-29T15:22:01.700160+0000","last_clean":"2024-08-29T15:22:01.700160+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:22:01.700160+0000","last_undegraded":"2024-08-29T15:22:01.700160+0000","last_fullsized":"2024-08-29T15:22:01.700160+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":359,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698713+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:22:01.698713+0000","last_peered":"2024-08-29T15:22:01.698713+0000","last_clean":"2024-08-29T15:22:01.698713+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:22:01.698713+0000","last_undegraded":"2024-08-29T15:22:01.698713+0000","last_fullsized":"2024-08-29T15:22:01.698713+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":27,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.309116+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:21:49.309116+0000","last_peered":"2024-08-29T15:21:49.309116+0000","last_clean":"2024-08-29T15:21:49.309116+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:21:49.309116+0000","last_undegraded":"2024-08-29T15:21:49.309116+0000","last_fullsized":"2024-08-29T15:21:49.309116+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":379,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700567+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:22:01.700567+0000","last_peered":"2024-08-29T15:22:01.700567+0000","last_clean":"2024-08-29T15:22:01.700567+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:22:01.700567+0000","last_undegraded":"2024-08-29T15:22:01.700567+0000","last_fullsized":"2024-08-29T15:22:01.700567+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":342,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.764297+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:22:01.764297+0000","last_peered":"2024-08-29T15:22:01.764297+0000","last_clean":"2024-08-29T15:22:01.764297+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:22:01.764297+0000","last_undegraded":"2024-08-29T15:22:01.764297+0000","last_fullsized":"2024-08-29T15:22:01.764297+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":27,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.309023+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:21:49.309023+0000","last_peered":"2024-08-29T15:21:49.309023+0000","last_clean":"2024-08-29T15:21:49.309023+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:21:49.309023+0000","last_undegraded":"2024-08-29T15:21:49.309023+0000","last_fullsized":"2024-08-29T15:21:49.309023+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":140,"reported_epoch":249,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:21:49.307243+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:21:49.307243+0000","last_peered":"2024-08-29T15:21:49.307243+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:21:49.307243+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:21:49.307243+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.309189+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:21:49.309189+0000","last_peered":"2024-08-29T15:21:49.309189+0000","last_clean":"2024-08-29T15:21:49.309189+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:21:49.309189+0000","last_undegraded":"2024-08-29T15:21:49.309189+0000","last_fullsized":"2024-08-29T15:21:49.309189+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":146,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.307338+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:21:49.307338+0000","last_peered":"2024-08-29T15:21:49.307338+0000","last_clean":"2024-08-29T15:21:49.307338+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:21:49.307338+0000","last_undegraded":"2024-08-29T15:21:49.307338+0000","last_fullsized":"2024-08-29T15:21:49.307338+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":390,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698287+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:22:01.698287+0000","last_peered":"2024-08-29T15:22:01.698287+0000","last_clean":"2024-08-29T15:22:01.698287+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:22:01.698287+0000","last_undegraded":"2024-08-29T15:22:01.698287+0000","last_fullsized":"2024-08-29T15:22:01.698287+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":391,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698794+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:22:01.698794+0000","last_peered":"2024-08-29T15:22:01.698794+0000","last_clean":"2024-08-29T15:22:01.698794+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:22:01.698794+0000","last_undegraded":"2024-08-29T15:22:01.698794+0000","last_fullsized":"2024-08-29T15:22:01.698794+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":27,"reported_epoch":249,"state":"active+clean","last_fresh":"2024-08-29T15:21:49.308751+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:21:49.308751+0000","last_peered":"2024-08-29T15:21:49.308751+0000","last_clean":"2024-08-29T15:21:49.308751+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:21:49.308751+0000","last_undegraded":"2024-08-29T15:21:49.308751+0000","last_fullsized":"2024-08-29T15:21:49.308751+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":399,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.763931+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:22:01.763931+0000","last_peered":"2024-08-29T15:22:01.763931+0000","last_clean":"2024-08-29T15:22:01.763931+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:22:01.763931+0000","last_undegraded":"2024-08-29T15:22:01.763931+0000","last_fullsized":"2024-08-29T15:22:01.763931+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5666635800,"num_objects":86467,"num_object_clones":0,"num_object_copies":259401,"num_objects_missing_on_primary":6380,"num_objects_missing":6380,"num_objects_degraded":30073,"num_objects_misplaced":16155,"num_objects_unfound":0,"num_objects_dirty":86467,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":164383,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14968,"num_bytes_recovered":860946432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16897880064,"data_stored":16897867848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122960,"ondisk_log_size":122960,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234330,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3070840,"kb_used_data":2768944,"kb_used_omap":1,"kb_used_meta":301822,"kb_avail":90674312,"statfs":{"total":95995035648,"available":92850495488,"internally_reserved":0,"allocated":2835398656,"data_stored":2833990075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":309066187},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495962,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":393332,"kb_used_data":239980,"kb_used_omap":1,"kb_used_meta":153342,"kb_avail":93351820,"statfs":{"total":95995035648,"available":95592263680,"internally_reserved":0,"allocated":245739520,"data_stored":244334739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":157022663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,0,1,2,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426547,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1785704,"kb_used_data":1734688,"kb_used_omap":8,"kb_used_meta":50999,"kb_avail":91959448,"statfs":{"total":95995035648,"available":94166474752,"internally_reserved":0,"allocated":1776320512,"data_stored":1774923197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":52223882},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019240,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2021976,"kb_used_data":1720144,"kb_used_omap":1,"kb_used_meta":301822,"kb_avail":91723176,"statfs":{"total":95995035648,"available":93924532224,"internally_reserved":0,"allocated":1761427456,"data_stored":1760045924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":309066182},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435424,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3732644,"kb_used_data":3462236,"kb_used_omap":4,"kb_used_meta":270395,"kb_avail":90012508,"statfs":{"total":95995035648,"available":92172808192,"internally_reserved":0,"allocated":3545329664,"data_stored":3543936289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":276884745},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903276,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3177576,"kb_used_data":3068064,"kb_used_omap":10,"kb_used_meta":109493,"kb_avail":90567576,"statfs":{"total":95995035648,"available":92741197824,"internally_reserved":0,"allocated":3141697536,"data_stored":3140304097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":112121360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762477,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2423076,"kb_used_data":2170652,"kb_used_omap":7,"kb_used_meta":252408,"kb_avail":91322076,"statfs":{"total":95995035648,"available":93513805824,"internally_reserved":0,"allocated":2222747648,"data_stored":2221353033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":258466146},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926004,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702636,"kb_used_data":1400228,"kb_used_omap":7,"kb_used_meta":302392,"kb_avail":92042516,"statfs":{"total":95995035648,"available":94251536384,"internally_reserved":0,"allocated":1433833472,"data_stored":1432431636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":309649551},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1423839232,"data_stored":1423835160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2212302848,"data_stored":2212298776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3133210624,"data_stored":3133210624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3534553088,"data_stored":3534553088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1756954624,"data_stored":1756954624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1769869312,"data_stored":1769865240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":241238016,"data_stored":241238016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2825912320,"data_stored":2825912320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:22:12.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:12.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:12.692 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:12.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:12.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:12.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:12.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/259485 objects degraded (11.601%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:12.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/259485 objects degraded (11.601%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:12.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:12.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:12.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:12.964 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:12.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:12.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/259485 objects degraded (11.601%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:13.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:13.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:13.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:13.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:13.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:13.338 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:13.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:13.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:13.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:13.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:13.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:13.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:13.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:13.726 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:22:13.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:13.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:13.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:13.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:13.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:13 smithi171 ceph-mon[90031]: pgmap v785: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/258210 objects degraded (11.658%); 16087/258210 objects misplaced (6.230%); 477 KiB/s, 7 objects/s recovering 2024-08-29T15:22:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2881326533' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:14.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[98970]: pgmap v785: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/258210 objects degraded (11.658%); 16087/258210 objects misplaced (6.230%); 477 KiB/s, 7 objects/s recovering 2024-08-29T15:22:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2881326533' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[93921]: pgmap v785: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/258210 objects degraded (11.658%); 16087/258210 objects misplaced (6.230%); 477 KiB/s, 7 objects/s recovering 2024-08-29T15:22:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2881326533' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:14.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:14.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:14.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:14.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:14.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:14.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:14.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:14.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:14.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:14.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:14.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:14.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:14.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:14.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:14.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:14.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:14.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:14.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:14 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:14.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:14 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:22:14.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:14 smithi027 ceph-mon[98970]: osdmap e251: 8 total, 8 up, 8 in 2024-08-29T15:22:14.868 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:14 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:14.868 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:14 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:22:14.869 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:14 smithi027 ceph-mon[93921]: osdmap e251: 8 total, 8 up, 8 in 2024-08-29T15:22:14.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:14.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:14.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:14.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:14 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:14 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:22:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:14 smithi171 ceph-mon[90031]: osdmap e251: 8 total, 8 up, 8 in 2024-08-29T15:22:15.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:15.097 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:15.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.223 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:15.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:15.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:15.346 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:15.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:15.447 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:15.470 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:15.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:22:15.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:15.603 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:15.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:15.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:15.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:15.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:15.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:15.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:15.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:15.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:15.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:15 smithi171 ceph-mon[90031]: pgmap v787: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/257979 objects degraded (11.668%); 16087/257979 objects misplaced (6.236%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:16.036 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:15 smithi027 ceph-mon[98970]: pgmap v787: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/257979 objects degraded (11.668%); 16087/257979 objects misplaced (6.236%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:16.037 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:15 smithi027 ceph-mon[93921]: pgmap v787: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/257979 objects degraded (11.668%); 16087/257979 objects misplaced (6.236%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:16.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:16.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:16.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:16.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:16.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:16.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:16.355 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:16.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:16.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:16.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:16.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:16.567 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:22:16.567 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:22:16.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:16.584 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:22:16.585 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:22:16.601 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:22:16.601 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:22:16.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:16.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:16.617 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:22:16.618 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:22:16.634 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:22:16.635 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:22:16.651 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:22:16.652 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:22:16.667 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:22:16.668 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:22:16.684 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:22:16.684 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:22:16.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:16.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:16.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:16.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:16.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:16.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:16.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:16.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:16.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:16.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:17.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:16 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2645346690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:17.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:16 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2645346690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:17.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:16 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2645346690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:17.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:17.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:17.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:17.235 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:17.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:17.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:17.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:17.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:17.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:17.531 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:22:17.531 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:22:17.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:17.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:17.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:22:17.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:17.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:17.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:17 smithi027 ceph-mon[98970]: pgmap v788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/257001 objects degraded (11.713%); 16005/257001 objects misplaced (6.228%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:17.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/257001 objects degraded (11.713%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:17.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:17 smithi027 ceph-mon[93921]: pgmap v788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/257001 objects degraded (11.713%); 16005/257001 objects misplaced (6.228%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:17.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/257001 objects degraded (11.713%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:17.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:17.869 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:17.971 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:17.998 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:17.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:17 smithi171 ceph-mon[90031]: pgmap v788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/257001 objects degraded (11.713%); 16005/257001 objects misplaced (6.228%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/257001 objects degraded (11.713%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:18.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:18.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:18.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:18.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:22:18.224 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:18.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:18.257 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:18.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:18.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:18.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:18.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:18.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:18.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:18.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:18.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:18.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:18.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:18.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:18.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:18.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:18.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:18.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:18.880 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:18.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:18.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:18.984 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:19.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:19.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:19.063 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:19.064 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:19.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:19.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:19.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:19.258 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:22:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:22:19] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:22:19.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:19.374 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:22:19.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:19.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:19.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:19.506 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:19.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:19.608 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:19.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.630 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:19.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:19.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:19.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:19.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:19.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:19.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:19.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:19.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:19.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:20.001 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:20.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:19 smithi171 ceph-mon[90031]: pgmap v789: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 30102/256245 objects degraded (11.747%); 15964/256245 objects misplaced (6.230%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:19 smithi027 ceph-mon[98970]: pgmap v789: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 30102/256245 objects degraded (11.747%); 15964/256245 objects misplaced (6.230%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:19 smithi027 ceph-mon[93921]: pgmap v789: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 30102/256245 objects degraded (11.747%); 15964/256245 objects misplaced (6.230%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:20.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:20.125 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:20.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:20.227 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:20.249 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:20.351 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:20.373 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:20.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:20.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:20.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:20.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:20.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:20.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:20.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:20.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:20.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:20.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:20.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:20.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:20.994 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:20.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:21.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:21.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:21.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:21.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:21.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:21.343 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:21.371 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:21.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:21.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:21.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:21.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:21.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:21.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:21.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:21.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:21.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:21.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:21.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:21.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:21.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:21.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:22.001 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:21 smithi027 ceph-mon[93921]: pgmap v790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 30102/255456 objects degraded (11.784%); 15922/255456 objects misplaced (6.233%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:22.001 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:21 smithi027 ceph-mon[98970]: pgmap v790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 30102/255456 objects degraded (11.784%); 15922/255456 objects misplaced (6.233%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:21 smithi171 ceph-mon[90031]: pgmap v790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 30102/255456 objects degraded (11.784%); 15922/255456 objects misplaced (6.233%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:22.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:22.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:22.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:22.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:22.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:22.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:22.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:22.296 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:22.297 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:22:22.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:22.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:22.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:22.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:22.476 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:22.516 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:22.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:22.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:22.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:22.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:22.748 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:22:22.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:22.771 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:22.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:22.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:22.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:22.896 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:22.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:22.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/255456 objects degraded (11.784%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:22 smithi171 ceph-mon[90031]: from='client.45509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/424552603' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:23.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:23.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/255456 objects degraded (11.784%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:23.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[93921]: from='client.45509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:23.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/424552603' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:23.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:23.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/255456 objects degraded (11.784%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:23.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[98970]: from='client.45509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:23.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/424552603' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:23.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:23.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:23.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:23.160 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":790,"stamp":"2024-08-29T15:22:20.571067+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5583411197,"num_objects":85152,"num_object_clones":0,"num_object_copies":255456,"num_objects_missing_on_primary":6305,"num_objects_missing":6305,"num_objects_degraded":30102,"num_objects_misplaced":15922,"num_objects_unfound":0,"num_objects_dirty":85152,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":166849,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15159,"num_bytes_recovered":872287437,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124585,"ondisk_log_size":124585,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18091916,"kb_used_data":16296584,"kb_used_omap":43,"kb_used_meta":1795220,"kb_avail":731869300,"statfs":{"total":767960285184,"available":749434163200,"internally_reserved":0,"allocated":16687702016,"data_stored":16676465358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1838305772},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-88014848,"num_objects":-1343,"num_object_clones":0,"num_object_copies":-4029,"num_objects_missing_on_primary":-83,"num_objects_missing":-83,"num_objects_degraded":0,"num_objects_misplaced":-233,"num_objects_unfound":0,"num_objects_dirty":-1343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1343,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":5373952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002746"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":386,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723566+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:22:13.723566+0000","last_peered":"2024-08-29T15:22:13.723566+0000","last_clean":"2024-08-29T15:22:13.723566+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:22:13.723566+0000","last_undegraded":"2024-08-29T15:22:13.723566+0000","last_fullsized":"2024-08-29T15:22:13.723566+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":28,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:09.787564+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:22:09.787564+0000","last_peered":"2024-08-29T15:22:09.787564+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:22:09.787564+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:22:09.787564+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.701121+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:22:01.701121+0000","last_peered":"2024-08-29T15:22:01.701121+0000","last_clean":"2024-08-29T15:22:01.701121+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:22:01.701121+0000","last_undegraded":"2024-08-29T15:22:01.701121+0000","last_fullsized":"2024-08-29T15:22:01.701121+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":352,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.759956+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:22:13.759956+0000","last_peered":"2024-08-29T15:22:13.759956+0000","last_clean":"2024-08-29T15:22:13.759956+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:22:13.759956+0000","last_undegraded":"2024-08-29T15:22:13.759956+0000","last_fullsized":"2024-08-29T15:22:13.759956+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":352,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760379+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:22:13.760379+0000","last_peered":"2024-08-29T15:22:13.760379+0000","last_clean":"2024-08-29T15:22:13.760379+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:22:13.760379+0000","last_undegraded":"2024-08-29T15:22:13.760379+0000","last_fullsized":"2024-08-29T15:22:13.760379+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:09.787515+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:22:09.787515+0000","last_peered":"2024-08-29T15:22:09.787515+0000","last_clean":"2024-08-29T15:22:09.787515+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:22:09.787515+0000","last_undegraded":"2024-08-29T15:22:09.787515+0000","last_fullsized":"2024-08-29T15:22:09.787515+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":163,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.728160+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:22:13.728160+0000","last_peered":"2024-08-29T15:22:13.728160+0000","last_clean":"2024-08-29T15:22:13.728160+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:22:13.728160+0000","last_undegraded":"2024-08-29T15:22:13.728160+0000","last_fullsized":"2024-08-29T15:22:13.728160+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":119,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.732136+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:22:01.732136+0000","last_peered":"2024-08-29T15:22:01.732136+0000","last_clean":"2024-08-29T15:22:01.732136+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:22:01.732136+0000","last_undegraded":"2024-08-29T15:22:01.732136+0000","last_fullsized":"2024-08-29T15:22:01.732136+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"251'10497","reported_seq":18526,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:15.963203+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:22:15.963203+0000","last_peered":"2024-08-29T15:22:15.963203+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:22:15.963203+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'400","ondisk_log_start":"175'400","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10097,"log_dups_size":0,"ondisk_log_size":10097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354877440,"num_objects":5415,"num_object_clones":0,"num_object_copies":16245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10497,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2490}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":27,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:04.787027+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:22:04.787027+0000","last_peered":"2024-08-29T15:22:04.787027+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:22:04.787027+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:22:04.787027+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700246+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:22:01.700246+0000","last_peered":"2024-08-29T15:22:01.700246+0000","last_clean":"2024-08-29T15:22:01.700246+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:22:01.700246+0000","last_undegraded":"2024-08-29T15:22:01.700246+0000","last_fullsized":"2024-08-29T15:22:01.700246+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"251'10223","reported_seq":10403,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:16.338514+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:22:16.338514+0000","last_peered":"2024-08-29T15:22:16.338514+0000","last_clean":"2024-08-29T15:22:16.338514+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:22:16.338514+0000","last_undegraded":"2024-08-29T15:22:16.338514+0000","last_fullsized":"2024-08-29T15:22:16.338514+0000","mapping_epoch":168,"log_start":"178'3528","ondisk_log_start":"178'3528","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6695,"log_dups_size":0,"ondisk_log_size":6695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349110272,"num_objects":5327,"num_object_clones":0,"num_object_copies":15981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10223,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":27,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:04.786965+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:22:04.786965+0000","last_peered":"2024-08-29T15:22:04.786965+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:22:04.786965+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:22:04.786965+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698843+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:22:01.698843+0000","last_peered":"2024-08-29T15:22:01.698843+0000","last_clean":"2024-08-29T15:22:01.698843+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:22:01.698843+0000","last_undegraded":"2024-08-29T15:22:01.698843+0000","last_fullsized":"2024-08-29T15:22:01.698843+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"251'10405","reported_seq":10554,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:16.860324+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:22:16.860324+0000","last_peered":"2024-08-29T15:22:16.860324+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:22:16.860324+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"178'3674","ondisk_log_start":"178'3674","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347799552,"num_objects":5307,"num_object_clones":0,"num_object_copies":15921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10405,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2830}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":168,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760219+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:22:13.760219+0000","last_peered":"2024-08-29T15:22:13.760219+0000","last_clean":"2024-08-29T15:22:13.760219+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:22:13.760219+0000","last_undegraded":"2024-08-29T15:22:13.760219+0000","last_fullsized":"2024-08-29T15:22:13.760219+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":74,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.731190+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:22:01.731190+0000","last_peered":"2024-08-29T15:22:01.731190+0000","last_clean":"2024-08-29T15:22:01.731190+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:22:01.731190+0000","last_undegraded":"2024-08-29T15:22:01.731190+0000","last_fullsized":"2024-08-29T15:22:01.731190+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"251'10521","reported_seq":15921,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:18.938957+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:22:18.938957+0000","last_peered":"2024-08-29T15:22:18.938957+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:22:18.938957+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"177'2362","ondisk_log_start":"177'2362","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8159,"log_dups_size":0,"ondisk_log_size":8159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344522752,"num_objects":5257,"num_object_clones":0,"num_object_copies":15771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10521,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5257}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:06.162127+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:22:06.162127+0000","last_peered":"2024-08-29T15:22:06.162127+0000","last_clean":"2024-08-29T15:22:06.162127+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:22:06.162127+0000","last_undegraded":"2024-08-29T15:22:06.162127+0000","last_fullsized":"2024-08-29T15:22:06.162127+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":366,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760344+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:22:13.760344+0000","last_peered":"2024-08-29T15:22:13.760344+0000","last_clean":"2024-08-29T15:22:13.760344+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:22:13.760344+0000","last_undegraded":"2024-08-29T15:22:13.760344+0000","last_fullsized":"2024-08-29T15:22:13.760344+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"251'10425","reported_seq":10562,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:19.202214+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:22:19.202214+0000","last_peered":"2024-08-29T15:22:19.202214+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:22:19.202214+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:22:19.202214+0000","mapping_epoch":234,"log_start":"177'2433","ondisk_log_start":"177'2433","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7992,"log_dups_size":0,"ondisk_log_size":7992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353173504,"num_objects":5389,"num_object_clones":0,"num_object_copies":16167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10425,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2912},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":360,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723380+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:22:13.723380+0000","last_peered":"2024-08-29T15:22:13.723380+0000","last_clean":"2024-08-29T15:22:13.723380+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:22:13.723380+0000","last_undegraded":"2024-08-29T15:22:13.723380+0000","last_fullsized":"2024-08-29T15:22:13.723380+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:04.785276+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:22:04.785276+0000","last_peered":"2024-08-29T15:22:04.785276+0000","last_clean":"2024-08-29T15:22:04.785276+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:22:04.785276+0000","last_undegraded":"2024-08-29T15:22:04.785276+0000","last_fullsized":"2024-08-29T15:22:04.785276+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"251'10421","reported_seq":10617,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:16.818941+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:22:16.818941+0000","last_peered":"2024-08-29T15:22:16.818941+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:22:16.818941+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:22:16.818941+0000","mapping_epoch":234,"log_start":"179'3668","ondisk_log_start":"179'3668","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355270656,"num_objects":5421,"num_object_clones":0,"num_object_copies":16263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10421,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2956},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":92,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.731078+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:22:01.731078+0000","last_peered":"2024-08-29T15:22:01.731078+0000","last_clean":"2024-08-29T15:22:01.731078+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:22:01.731078+0000","last_undegraded":"2024-08-29T15:22:01.731078+0000","last_fullsized":"2024-08-29T15:22:01.731078+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:04.785300+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:22:04.785300+0000","last_peered":"2024-08-29T15:22:04.785300+0000","last_clean":"2024-08-29T15:22:04.785300+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:22:04.785300+0000","last_undegraded":"2024-08-29T15:22:04.785300+0000","last_fullsized":"2024-08-29T15:22:04.785300+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"251'10282","reported_seq":10663,"reported_epoch":251,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:22:16.832009+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:22:16.832009+0000","last_peered":"2024-08-29T15:22:16.832009+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:22:16.832009+0000","last_undegraded":"2024-08-29T15:22:16.832009+0000","last_fullsized":"2024-08-29T15:22:16.832009+0000","mapping_epoch":202,"log_start":"179'3567","ondisk_log_start":"179'3567","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352321536,"num_objects":5376,"num_object_clones":0,"num_object_copies":16128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5376,"num_objects_unfound":0,"num_objects_dirty":5376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10282,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5376}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698866+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:22:01.698866+0000","last_peered":"2024-08-29T15:22:01.698866+0000","last_clean":"2024-08-29T15:22:01.698866+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:22:01.698866+0000","last_undegraded":"2024-08-29T15:22:01.698866+0000","last_fullsized":"2024-08-29T15:22:01.698866+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":234,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.699964+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:22:01.699964+0000","last_peered":"2024-08-29T15:22:01.699964+0000","last_clean":"2024-08-29T15:22:01.699964+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:22:01.699964+0000","last_undegraded":"2024-08-29T15:22:01.699964+0000","last_fullsized":"2024-08-29T15:22:01.699964+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"251'10400","reported_seq":10531,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:15.906435+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:22:15.906435+0000","last_peered":"2024-08-29T15:22:15.906435+0000","last_clean":"2024-08-29T15:22:15.906435+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:22:15.906435+0000","last_undegraded":"2024-08-29T15:22:15.906435+0000","last_fullsized":"2024-08-29T15:22:15.906435+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9122,"log_dups_size":0,"ondisk_log_size":9122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351207448,"num_objects":5360,"num_object_clones":0,"num_object_copies":16080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5360,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10400,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":364,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723661+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:22:13.723661+0000","last_peered":"2024-08-29T15:22:13.723661+0000","last_clean":"2024-08-29T15:22:13.723661+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:22:13.723661+0000","last_undegraded":"2024-08-29T15:22:13.723661+0000","last_fullsized":"2024-08-29T15:22:13.723661+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":356,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.728286+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:22:13.728286+0000","last_peered":"2024-08-29T15:22:13.728286+0000","last_clean":"2024-08-29T15:22:13.728286+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:22:13.728286+0000","last_undegraded":"2024-08-29T15:22:13.728286+0000","last_fullsized":"2024-08-29T15:22:13.728286+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"251'10029","reported_seq":10208,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:16.003524+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:22:16.003524+0000","last_peered":"2024-08-29T15:22:16.003524+0000","last_clean":"2024-08-29T15:22:16.003524+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:22:16.003524+0000","last_undegraded":"2024-08-29T15:22:16.003524+0000","last_fullsized":"2024-08-29T15:22:16.003524+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10029,"log_dups_size":0,"ondisk_log_size":10029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339410944,"num_objects":5179,"num_object_clones":0,"num_object_copies":15537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10029,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":455,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760029+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:22:13.760029+0000","last_peered":"2024-08-29T15:22:13.760029+0000","last_clean":"2024-08-29T15:22:13.760029+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:22:13.760029+0000","last_undegraded":"2024-08-29T15:22:13.760029+0000","last_fullsized":"2024-08-29T15:22:13.760029+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":720,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700270+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:22:01.700270+0000","last_peered":"2024-08-29T15:22:01.700270+0000","last_clean":"2024-08-29T15:22:01.700270+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:22:01.700270+0000","last_undegraded":"2024-08-29T15:22:01.700270+0000","last_fullsized":"2024-08-29T15:22:01.700270+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":228,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.701135+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:22:01.701135+0000","last_peered":"2024-08-29T15:22:01.701135+0000","last_clean":"2024-08-29T15:22:01.701135+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:22:01.701135+0000","last_undegraded":"2024-08-29T15:22:01.701135+0000","last_fullsized":"2024-08-29T15:22:01.701135+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":90,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.731465+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:22:01.731465+0000","last_peered":"2024-08-29T15:22:01.731465+0000","last_clean":"2024-08-29T15:22:01.731465+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:22:01.731465+0000","last_undegraded":"2024-08-29T15:22:01.731465+0000","last_fullsized":"2024-08-29T15:22:01.731465+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"251'10414","reported_seq":10792,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:15.548319+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:22:15.548319+0000","last_peered":"2024-08-29T15:22:15.548319+0000","last_clean":"2024-08-29T15:22:15.548319+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:22:15.548319+0000","last_undegraded":"2024-08-29T15:22:15.548319+0000","last_fullsized":"2024-08-29T15:22:15.548319+0000","mapping_epoch":201,"log_start":"178'3353","ondisk_log_start":"178'3353","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7061,"log_dups_size":0,"ondisk_log_size":7061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10414,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":829,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.729296+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:22:13.729296+0000","last_peered":"2024-08-29T15:22:13.729296+0000","last_clean":"2024-08-29T15:22:13.729296+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:22:13.729296+0000","last_undegraded":"2024-08-29T15:22:13.729296+0000","last_fullsized":"2024-08-29T15:22:13.729296+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":367,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760111+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:22:13.760111+0000","last_peered":"2024-08-29T15:22:13.760111+0000","last_clean":"2024-08-29T15:22:13.760111+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:22:13.760111+0000","last_undegraded":"2024-08-29T15:22:13.760111+0000","last_fullsized":"2024-08-29T15:22:13.760111+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":374,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760511+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:22:13.760511+0000","last_peered":"2024-08-29T15:22:13.760511+0000","last_clean":"2024-08-29T15:22:13.760511+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:22:13.760511+0000","last_undegraded":"2024-08-29T15:22:13.760511+0000","last_fullsized":"2024-08-29T15:22:13.760511+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"251'10294","reported_seq":16095,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:16.789712+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:22:16.789712+0000","last_peered":"2024-08-29T15:22:16.789712+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:22:16.789712+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"178'3530","ondisk_log_start":"178'3530","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353632256,"num_objects":5396,"num_object_clones":0,"num_object_copies":16188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10294,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":962},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":637,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723344+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:22:13.723344+0000","last_peered":"2024-08-29T15:22:13.723344+0000","last_clean":"2024-08-29T15:22:13.723344+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:22:13.723344+0000","last_undegraded":"2024-08-29T15:22:13.723344+0000","last_fullsized":"2024-08-29T15:22:13.723344+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":81,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.730805+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:22:13.730805+0000","last_peered":"2024-08-29T15:22:13.730805+0000","last_clean":"2024-08-29T15:22:13.730805+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:22:13.730805+0000","last_undegraded":"2024-08-29T15:22:13.730805+0000","last_fullsized":"2024-08-29T15:22:13.730805+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":401,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.728006+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:22:13.728006+0000","last_peered":"2024-08-29T15:22:13.728006+0000","last_clean":"2024-08-29T15:22:13.728006+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:22:13.728006+0000","last_undegraded":"2024-08-29T15:22:13.728006+0000","last_fullsized":"2024-08-29T15:22:13.728006+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"251'10370","reported_seq":10560,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:16.309528+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:22:16.309528+0000","last_peered":"2024-08-29T15:22:16.309528+0000","last_clean":"2024-08-29T15:22:16.309528+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:22:16.309528+0000","last_undegraded":"2024-08-29T15:22:16.309528+0000","last_fullsized":"2024-08-29T15:22:16.309528+0000","mapping_epoch":234,"log_start":"179'3697","ondisk_log_start":"179'3697","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350093312,"num_objects":5342,"num_object_clones":0,"num_object_copies":16026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10370,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":188,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.701057+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:22:01.701057+0000","last_peered":"2024-08-29T15:22:01.701057+0000","last_clean":"2024-08-29T15:22:01.701057+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:22:01.701057+0000","last_undegraded":"2024-08-29T15:22:01.701057+0000","last_fullsized":"2024-08-29T15:22:01.701057+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698915+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:22:01.698915+0000","last_peered":"2024-08-29T15:22:01.698915+0000","last_clean":"2024-08-29T15:22:01.698915+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:22:01.698915+0000","last_undegraded":"2024-08-29T15:22:01.698915+0000","last_fullsized":"2024-08-29T15:22:01.698915+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":95,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700310+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:22:01.700310+0000","last_peered":"2024-08-29T15:22:01.700310+0000","last_clean":"2024-08-29T15:22:01.700310+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:22:01.700310+0000","last_undegraded":"2024-08-29T15:22:01.700310+0000","last_fullsized":"2024-08-29T15:22:01.700310+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"251'10464","reported_seq":10652,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:18.436014+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:22:18.436014+0000","last_peered":"2024-08-29T15:22:18.436014+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:22:18.436014+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"177'2342","ondisk_log_start":"177'2342","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351404032,"num_objects":5362,"num_object_clones":0,"num_object_copies":16086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10464,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2939}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700454+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:22:01.700454+0000","last_peered":"2024-08-29T15:22:01.700454+0000","last_clean":"2024-08-29T15:22:01.700454+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:22:01.700454+0000","last_undegraded":"2024-08-29T15:22:01.700454+0000","last_fullsized":"2024-08-29T15:22:01.700454+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":793,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.728920+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:22:13.728920+0000","last_peered":"2024-08-29T15:22:13.728920+0000","last_clean":"2024-08-29T15:22:13.728920+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:22:13.728920+0000","last_undegraded":"2024-08-29T15:22:13.728920+0000","last_fullsized":"2024-08-29T15:22:13.728920+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":370,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.728951+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:22:13.728951+0000","last_peered":"2024-08-29T15:22:13.728951+0000","last_clean":"2024-08-29T15:22:13.728951+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:22:13.728951+0000","last_undegraded":"2024-08-29T15:22:13.728951+0000","last_fullsized":"2024-08-29T15:22:13.728951+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"251'10336","reported_seq":11080,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:15.613611+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:22:15.613611+0000","last_peered":"2024-08-29T15:22:15.613611+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:22:15.613611+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"178'3280","ondisk_log_start":"178'3280","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7056,"log_dups_size":0,"ondisk_log_size":7056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349437952,"num_objects":5332,"num_object_clones":0,"num_object_copies":15996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5298,"num_objects_unfound":0,"num_objects_dirty":5332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10336,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3181}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":375,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.728839+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:22:13.728839+0000","last_peered":"2024-08-29T15:22:13.728839+0000","last_clean":"2024-08-29T15:22:13.728839+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:22:13.728839+0000","last_undegraded":"2024-08-29T15:22:13.728839+0000","last_fullsized":"2024-08-29T15:22:13.728839+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":677,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:01.730676+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:22:01.730676+0000","last_peered":"2024-08-29T15:22:01.730676+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:22:01.730676+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:22:01.730676+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":367,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760252+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:22:13.760252+0000","last_peered":"2024-08-29T15:22:13.760252+0000","last_clean":"2024-08-29T15:22:13.760252+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:22:13.760252+0000","last_undegraded":"2024-08-29T15:22:13.760252+0000","last_fullsized":"2024-08-29T15:22:13.760252+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"251'10298","reported_seq":17433,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:19.379313+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:22:19.379313+0000","last_peered":"2024-08-29T15:22:19.379313+0000","last_clean":"2024-08-29T15:22:19.379313+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:22:19.379313+0000","last_undegraded":"2024-08-29T15:22:19.379313+0000","last_fullsized":"2024-08-29T15:22:19.379313+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7899,"log_dups_size":0,"ondisk_log_size":7899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336199680,"num_objects":5130,"num_object_clones":0,"num_object_copies":15390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10298,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":361,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.726784+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:22:13.726784+0000","last_peered":"2024-08-29T15:22:13.726784+0000","last_clean":"2024-08-29T15:22:13.726784+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:22:13.726784+0000","last_undegraded":"2024-08-29T15:22:13.726784+0000","last_fullsized":"2024-08-29T15:22:13.726784+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":983,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:13.727887+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:22:13.727887+0000","last_peered":"2024-08-29T15:22:13.727887+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:22:13.727887+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":358,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723237+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:22:13.723237+0000","last_peered":"2024-08-29T15:22:13.723237+0000","last_clean":"2024-08-29T15:22:13.723237+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:22:13.723237+0000","last_undegraded":"2024-08-29T15:22:13.723237+0000","last_fullsized":"2024-08-29T15:22:13.723237+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"251'10347","reported_seq":13343,"reported_epoch":251,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:20.051172+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:22:20.051172+0000","last_peered":"2024-08-29T15:22:20.051172+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:22:20.051172+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'2812","ondisk_log_start":"178'2812","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344915968,"num_objects":5263,"num_object_clones":0,"num_object_copies":15789,"num_objects_missing_on_primary":6297,"num_objects_missing":6297,"num_objects_degraded":2313,"num_objects_misplaced":5248,"num_objects_unfound":0,"num_objects_dirty":5263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10347,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1423,"num_bytes_recovered":93126656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":15},{"shards":"1,2,7","objects":2877},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":356,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760335+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:22:13.760335+0000","last_peered":"2024-08-29T15:22:13.760335+0000","last_clean":"2024-08-29T15:22:13.760335+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:22:13.760335+0000","last_undegraded":"2024-08-29T15:22:13.760335+0000","last_fullsized":"2024-08-29T15:22:13.760335+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":101,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.731856+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:22:01.731856+0000","last_peered":"2024-08-29T15:22:01.731856+0000","last_clean":"2024-08-29T15:22:01.731856+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:22:01.731856+0000","last_undegraded":"2024-08-29T15:22:01.731856+0000","last_fullsized":"2024-08-29T15:22:01.731856+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.698914+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:22:01.698914+0000","last_peered":"2024-08-29T15:22:01.698914+0000","last_clean":"2024-08-29T15:22:01.698914+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:22:01.698914+0000","last_undegraded":"2024-08-29T15:22:01.698914+0000","last_fullsized":"2024-08-29T15:22:01.698914+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":29,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:09.787617+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:22:09.787617+0000","last_peered":"2024-08-29T15:22:09.787617+0000","last_clean":"2024-08-29T15:22:09.787617+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:22:09.787617+0000","last_undegraded":"2024-08-29T15:22:09.787617+0000","last_fullsized":"2024-08-29T15:22:09.787617+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":27,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:09.787635+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:22:09.787635+0000","last_peered":"2024-08-29T15:22:09.787635+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:22:09.787635+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:22:09.787635+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":365,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.728776+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:22:13.728776+0000","last_peered":"2024-08-29T15:22:13.728776+0000","last_clean":"2024-08-29T15:22:13.728776+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:22:13.728776+0000","last_undegraded":"2024-08-29T15:22:13.728776+0000","last_fullsized":"2024-08-29T15:22:13.728776+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":115,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:13.725841+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:22:13.725841+0000","last_peered":"2024-08-29T15:22:13.725841+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:22:13.725841+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:22:13.725841+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":117,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700660+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:22:01.700660+0000","last_peered":"2024-08-29T15:22:01.700660+0000","last_clean":"2024-08-29T15:22:01.700660+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:22:01.700660+0000","last_undegraded":"2024-08-29T15:22:01.700660+0000","last_fullsized":"2024-08-29T15:22:01.700660+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":324,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723005+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:22:13.723005+0000","last_peered":"2024-08-29T15:22:13.723005+0000","last_clean":"2024-08-29T15:22:13.723005+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:22:13.723005+0000","last_undegraded":"2024-08-29T15:22:13.723005+0000","last_fullsized":"2024-08-29T15:22:13.723005+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":378,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760552+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:22:13.760552+0000","last_peered":"2024-08-29T15:22:13.760552+0000","last_clean":"2024-08-29T15:22:13.760552+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:22:13.760552+0000","last_undegraded":"2024-08-29T15:22:13.760552+0000","last_fullsized":"2024-08-29T15:22:13.760552+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":376,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723128+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:22:13.723128+0000","last_peered":"2024-08-29T15:22:13.723128+0000","last_clean":"2024-08-29T15:22:13.723128+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:22:13.723128+0000","last_undegraded":"2024-08-29T15:22:13.723128+0000","last_fullsized":"2024-08-29T15:22:13.723128+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.731918+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:22:01.731918+0000","last_peered":"2024-08-29T15:22:01.731918+0000","last_clean":"2024-08-29T15:22:01.731918+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:22:01.731918+0000","last_undegraded":"2024-08-29T15:22:01.731918+0000","last_fullsized":"2024-08-29T15:22:01.731918+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":233,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.759770+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:22:13.759770+0000","last_peered":"2024-08-29T15:22:13.759770+0000","last_clean":"2024-08-29T15:22:13.759770+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:22:13.759770+0000","last_undegraded":"2024-08-29T15:22:13.759770+0000","last_fullsized":"2024-08-29T15:22:13.759770+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":353,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760458+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:22:13.760458+0000","last_peered":"2024-08-29T15:22:13.760458+0000","last_clean":"2024-08-29T15:22:13.760458+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:22:13.760458+0000","last_undegraded":"2024-08-29T15:22:13.760458+0000","last_fullsized":"2024-08-29T15:22:13.760458+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":356,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.727599+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:22:13.727599+0000","last_peered":"2024-08-29T15:22:13.727599+0000","last_clean":"2024-08-29T15:22:13.727599+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:22:13.727599+0000","last_undegraded":"2024-08-29T15:22:13.727599+0000","last_fullsized":"2024-08-29T15:22:13.727599+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":361,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723479+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:22:13.723479+0000","last_peered":"2024-08-29T15:22:13.723479+0000","last_clean":"2024-08-29T15:22:13.723479+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:22:13.723479+0000","last_undegraded":"2024-08-29T15:22:13.723479+0000","last_fullsized":"2024-08-29T15:22:13.723479+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":29,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.726004+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:22:13.726004+0000","last_peered":"2024-08-29T15:22:13.726004+0000","last_clean":"2024-08-29T15:22:13.726004+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:22:13.726004+0000","last_undegraded":"2024-08-29T15:22:13.726004+0000","last_fullsized":"2024-08-29T15:22:13.726004+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":381,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.729000+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:22:13.729000+0000","last_peered":"2024-08-29T15:22:13.729000+0000","last_clean":"2024-08-29T15:22:13.729000+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:22:13.729000+0000","last_undegraded":"2024-08-29T15:22:13.729000+0000","last_fullsized":"2024-08-29T15:22:13.729000+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":345,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760198+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:22:13.760198+0000","last_peered":"2024-08-29T15:22:13.760198+0000","last_clean":"2024-08-29T15:22:13.760198+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:22:13.760198+0000","last_undegraded":"2024-08-29T15:22:13.760198+0000","last_fullsized":"2024-08-29T15:22:13.760198+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.725789+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:22:13.725789+0000","last_peered":"2024-08-29T15:22:13.725789+0000","last_clean":"2024-08-29T15:22:13.725789+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:22:13.725789+0000","last_undegraded":"2024-08-29T15:22:13.725789+0000","last_fullsized":"2024-08-29T15:22:13.725789+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":141,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:01.700437+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:22:01.700437+0000","last_peered":"2024-08-29T15:22:01.700437+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:22:01.700437+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:22:01.700437+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":29,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.725658+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:22:13.725658+0000","last_peered":"2024-08-29T15:22:13.725658+0000","last_clean":"2024-08-29T15:22:13.725658+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:22:13.725658+0000","last_undegraded":"2024-08-29T15:22:13.725658+0000","last_fullsized":"2024-08-29T15:22:13.725658+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":147,"reported_epoch":250,"state":"active+clean","last_fresh":"2024-08-29T15:22:01.700536+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:22:01.700536+0000","last_peered":"2024-08-29T15:22:01.700536+0000","last_clean":"2024-08-29T15:22:01.700536+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:22:01.700536+0000","last_undegraded":"2024-08-29T15:22:01.700536+0000","last_fullsized":"2024-08-29T15:22:01.700536+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":392,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723406+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:22:13.723406+0000","last_peered":"2024-08-29T15:22:13.723406+0000","last_clean":"2024-08-29T15:22:13.723406+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:22:13.723406+0000","last_undegraded":"2024-08-29T15:22:13.723406+0000","last_fullsized":"2024-08-29T15:22:13.723406+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":393,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.725981+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:22:13.725981+0000","last_peered":"2024-08-29T15:22:13.725981+0000","last_clean":"2024-08-29T15:22:13.725981+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:22:13.725981+0000","last_undegraded":"2024-08-29T15:22:13.725981+0000","last_fullsized":"2024-08-29T15:22:13.725981+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":29,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723443+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:22:13.723443+0000","last_peered":"2024-08-29T15:22:13.723443+0000","last_clean":"2024-08-29T15:22:13.723443+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:22:13.723443+0000","last_undegraded":"2024-08-29T15:22:13.723443+0000","last_fullsized":"2024-08-29T15:22:13.723443+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":401,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.760429+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:22:13.760429+0000","last_peered":"2024-08-29T15:22:13.760429+0000","last_clean":"2024-08-29T15:22:13.760429+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:22:13.760429+0000","last_undegraded":"2024-08-29T15:22:13.760429+0000","last_fullsized":"2024-08-29T15:22:13.760429+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5578620952,"num_objects":85124,"num_object_clones":0,"num_object_copies":255372,"num_objects_missing_on_primary":6297,"num_objects_missing":6297,"num_objects_degraded":30073,"num_objects_misplaced":15922,"num_objects_unfound":0,"num_objects_dirty":85124,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165726,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15050,"num_bytes_recovered":866320384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16636719104,"data_stored":16636706888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123403,"ondisk_log_size":123403,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234332,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3029064,"kb_used_data":2723136,"kb_used_omap":1,"kb_used_meta":305918,"kb_avail":90716088,"statfs":{"total":95995035648,"available":92893274112,"internally_reserved":0,"allocated":2788491264,"data_stored":2787075035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":313260491},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495964,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381188,"kb_used_data":227836,"kb_used_omap":1,"kb_used_meta":153342,"kb_avail":93363964,"statfs":{"total":95995035648,"available":95604699136,"internally_reserved":0,"allocated":233304064,"data_stored":231891635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":157022663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426549,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1762424,"kb_used_data":1707312,"kb_used_omap":8,"kb_used_meta":55095,"kb_avail":91982728,"statfs":{"total":95995035648,"available":94190313472,"internally_reserved":0,"allocated":1748287488,"data_stored":1746882525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":56418186},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019242,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2014312,"kb_used_data":1690336,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":91730840,"statfs":{"total":95995035648,"available":93932380160,"internally_reserved":0,"allocated":1730904064,"data_stored":1729514884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435426,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3693428,"kb_used_data":3404972,"kb_used_omap":4,"kb_used_meta":288443,"kb_avail":90051724,"statfs":{"total":95995035648,"available":92212965376,"internally_reserved":0,"allocated":3486691328,"data_stored":3485290305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":295365897},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903278,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3138168,"kb_used_data":3024496,"kb_used_omap":10,"kb_used_meta":113653,"kb_avail":90606984,"statfs":{"total":95995035648,"available":92781551616,"internally_reserved":0,"allocated":3097083904,"data_stored":3095682817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":116381200},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762479,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2395704,"kb_used_data":2143276,"kb_used_omap":7,"kb_used_meta":252408,"kb_avail":91349448,"statfs":{"total":95995035648,"available":93541834752,"internally_reserved":0,"allocated":2194714624,"data_stored":2193312361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":258466146},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926006,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1677628,"kb_used_data":1375220,"kb_used_omap":7,"kb_used_meta":302392,"kb_avail":92067524,"statfs":{"total":95995035648,"available":94277144576,"internally_reserved":0,"allocated":1408225280,"data_stored":1406815796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":309649551},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1398345728,"data_stored":1398341656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2185039872,"data_stored":2185035800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3091202048,"data_stored":3091202048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3482648576,"data_stored":3482648576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1727594496,"data_stored":1727594496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1742737408,"data_stored":1742733336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":229179392,"data_stored":229179392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2779971584,"data_stored":2779971584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:22:23.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:22:23.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:23.275 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:23.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:23.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:23.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:23.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:23.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:23.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:23.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:23.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:23.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:23.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:23.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:23.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:23.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:23.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:23.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:24.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:23 smithi171 ceph-mon[90031]: pgmap v791: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 30102/254061 objects degraded (11.848%); 15830/254061 objects misplaced (6.231%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:23 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:22:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:23 smithi171 ceph-mon[90031]: osdmap e252: 8 total, 8 up, 8 in 2024-08-29T15:22:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:22:24.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:24.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:24.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:24.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[98970]: pgmap v791: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 30102/254061 objects degraded (11.848%); 15830/254061 objects misplaced (6.231%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:22:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[98970]: osdmap e252: 8 total, 8 up, 8 in 2024-08-29T15:22:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:22:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:24.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[93921]: pgmap v791: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 30102/254061 objects degraded (11.848%); 15830/254061 objects misplaced (6.231%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:24.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:22:24.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:24.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[93921]: osdmap e252: 8 total, 8 up, 8 in 2024-08-29T15:22:24.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:22:24.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:24.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:24.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:24.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:24.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:24.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:24.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:24.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:24.402 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:24.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:24.406 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:24.419 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:24.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:24.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:24.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:24.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:24.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:24.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:24.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:24.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:24.878 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:24.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:24.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:24.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:25.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:25.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:25.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:25.127 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:25.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:25.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:25.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:25.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:25.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:25.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:25.503 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:25.526 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:25.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:25.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:25.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:25.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:25.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:25.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:25.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:25.890 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:22:25.890 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:22:25.902 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:25.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:25.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:26.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:25 smithi171 ceph-mon[90031]: pgmap v793: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 30102/253833 objects degraded (11.859%); 15830/253833 objects misplaced (6.236%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:26.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:26.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:26.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:25 smithi027 ceph-mon[93921]: pgmap v793: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 30102/253833 objects degraded (11.859%); 15830/253833 objects misplaced (6.236%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:26.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:25 smithi027 ceph-mon[98970]: pgmap v793: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 30102/253833 objects degraded (11.859%); 15830/253833 objects misplaced (6.236%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:26.130 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:26.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:26.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:26.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:26.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:26.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:26.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:22:26.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:26.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:26.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:26.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:26.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:26.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:26.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:26.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:26.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:26.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:26.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:26.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:27.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:27.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:27.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:27.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:27.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:27.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:27.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:27.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:27.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:27.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:27.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:27.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:27.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:27.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:27.659 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:27.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:27.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:27.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:27.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:27.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:27.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:27.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:27.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:27.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:22:27.951 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:27 smithi171 ceph-mon[90031]: pgmap v794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/252864 objects degraded (11.904%); 15748/252864 objects misplaced (6.228%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:22:27.951 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/252864 objects degraded (11.904%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:28.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:28.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:28.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:28.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:27 smithi027 ceph-mon[98970]: pgmap v794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/252864 objects degraded (11.904%); 15748/252864 objects misplaced (6.228%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:22:28.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/252864 objects degraded (11.904%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:28.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:27 smithi027 ceph-mon[93921]: pgmap v794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/252864 objects degraded (11.904%); 15748/252864 objects misplaced (6.228%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:22:28.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/252864 objects degraded (11.904%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:28.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:28.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:28.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:28.167 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:28.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:28.291 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:28.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:28.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:28.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:28.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:28.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:28.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:28.646 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:28.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:28.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:28.793 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:28.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:28.917 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:28.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:29.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:29.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:29.097 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:22:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:22:29] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-08-29T15:22:29.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:29.098 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.098 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.098 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:29.098 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:29.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:29.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:29.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:29.209 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:29.209 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/252171 objects degraded (11.937%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1131,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":252,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":84057,"data_bytes":5511649277,"bytes_used":18354630656,"bytes_avail":749605654528,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":252171,"degraded_ratio":0.11937137894523954,"misplaced_objects":15712,"misplaced_total":252171,"misplaced_ratio":0.062306926648980257,"recovering_objects_per_sec":7,"recovering_bytes_per_sec":517594,"recovering_keys_per_sec":0,"num_objects_recovered":79,"num_bytes_recovered":5177344,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":135},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:22:29.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:29.296 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:29.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:29.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:29.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:29.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:29.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:29.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:29.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.672 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:29.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:29.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:29.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:29.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:29.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:29.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:29.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:29.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:30.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:30.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:30.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:30.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:30.162 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:29 smithi171 ceph-mon[90031]: pgmap v795: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/252171 objects degraded (11.937%); 15712/252171 objects misplaced (6.231%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:22:30.162 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3905247520' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:30.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:30.191 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:29 smithi027 ceph-mon[98970]: pgmap v795: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/252171 objects degraded (11.937%); 15712/252171 objects misplaced (6.231%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:22:30.191 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3905247520' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:30.192 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:29 smithi027 ceph-mon[93921]: pgmap v795: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 135 op/s; 30102/252171 objects degraded (11.937%); 15712/252171 objects misplaced (6.231%); 505 KiB/s, 7 objects/s recovering 2024-08-29T15:22:30.192 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3905247520' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:30.197 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:22:30.198 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:22:30.216 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:22:30.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:22:30.221 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:30.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:30.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:30.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:30.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:30.524 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:30.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:30.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:30.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:30.678 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:30.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:30.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:30.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:30.890 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:22:30.890 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:22:30.890 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:22:30.891 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:22:30.891 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:22:30.891 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:22:30.891 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:22:30.891 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:22:30.891 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:22:30.892 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:22:30.902 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:22:30.902 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:22:30.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:30.924 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:22:30.924 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:22:30.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:30.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:30.948 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:22:30.948 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:22:31.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:22:31.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3152006534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3152006534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:31.089 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:31.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:31.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:31.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:31.261 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:31.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3152006534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:31.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:31.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:31.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:31.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:31.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:31.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:31.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:31.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:31.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:31.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:31.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:31.839 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:31.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:31.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:31.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:31.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:31.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:31 smithi171 ceph-mon[90031]: pgmap v796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/251616 objects degraded (11.963%); 15676/251616 objects misplaced (6.230%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:31 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/251616 objects degraded (11.963%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:32.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:32.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:31 smithi027 ceph-mon[98970]: pgmap v796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/251616 objects degraded (11.963%); 15676/251616 objects misplaced (6.230%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:32.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:31 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/251616 objects degraded (11.963%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:32.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:31 smithi027 ceph-mon[93921]: pgmap v796: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/251616 objects degraded (11.963%); 15676/251616 objects misplaced (6.230%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:32.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:31 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/251616 objects degraded (11.963%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:32.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:32.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:32.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:32.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:32.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:32.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:32.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:32.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:32.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:32.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:32.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:32.787 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:32.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:32.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:32.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:32.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:32.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:33.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:33.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:33.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:33.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:33.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:33.267 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:22:33.267 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:22:33.268 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:22:33.268 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:22:33.268 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:22:33.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:33.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:33.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:33.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:33.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:33.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:33.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:33.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:33.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:22:33.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:33.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:33.877 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:22:33.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:22:33.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:33.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:33.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:33 smithi027 ceph-mon[98970]: pgmap v797: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/250167 objects degraded (12.033%); 15591/250167 objects misplaced (6.232%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/347036284' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:22:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:33 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:22:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:33 smithi027 ceph-mon[93921]: pgmap v797: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/250167 objects degraded (12.033%); 15591/250167 objects misplaced (6.232%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/347036284' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:22:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:33 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:22:34.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:34.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:34.152 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:34.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:34.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:34.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:33 smithi171 ceph-mon[90031]: pgmap v797: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/250167 objects degraded (12.033%); 15591/250167 objects misplaced (6.232%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/347036284' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:22:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:33 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:22:34.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:34.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:34.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:34.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:34.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:34.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:34.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:34.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:34.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:34.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:34.844 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:34.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:34.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:34.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:34.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:35.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:35.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:35.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:35.200 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:34 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:35.200 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:34 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:22:35.200 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:34 smithi027 ceph-mon[93921]: osdmap e253: 8 total, 8 up, 8 in 2024-08-29T15:22:35.200 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:34 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:35.200 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:34 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:22:35.201 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:34 smithi027 ceph-mon[98970]: osdmap e253: 8 total, 8 up, 8 in 2024-08-29T15:22:35.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:35.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:35.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:34 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:34 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:22:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:34 smithi171 ceph-mon[90031]: osdmap e253: 8 total, 8 up, 8 in 2024-08-29T15:22:35.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:35.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:35.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:35.484 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:35.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:35.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:35.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:35.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:35.735 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:35.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:35.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:35.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:35.912 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:35.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:35.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:35.991 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:35.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:36.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:36.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:36.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:36.117 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:35 smithi171 ceph-mon[90031]: pgmap v799: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/249924 objects degraded (12.044%); 15591/249924 objects misplaced (6.238%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:36.138 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:22:36.138 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:22:36.138 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:22:36.138 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:22:36.138 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:220847 5.e 5:70ceb708:::benchmark_data_smithi027_135490_object87095:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 532.04205641500005, 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095839788999999995, 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 220847 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.139 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073331+0000", 2024-08-29T15:22:36.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073333+0000", 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2689999999999998e-06 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073338+0000", 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4009999999999996e-06 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.073342+0000", 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0799999999999999e-06 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.077819+0000", 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044769440000000001 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.077934+0000", 2024-08-29T15:22:36.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000115091 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.078049+0000", 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000114052 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.082327+0000", 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042779640000000004 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.086356+0000", 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040299300000000001 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169109+0000", 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082753023999999994 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169144+0000", 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5011000000000001e-05 2024-08-29T15:22:36.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:44.169171+0000", 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6423000000000001e-05 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239381 5.e 5:711fbf86:::benchmark_data_smithi027_135490_object105629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 510.401316313, 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.21797998, 2024-08-29T15:22:36.145 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239381 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.146 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714071+0000", 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714074+0000", 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.54e-06 2024-08-29T15:22:36.147 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714081+0000", 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8430000000000004e-06 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714086+0000", 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6070000000000001e-06 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714321+0000", 2024-08-29T15:22:36.148 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000235135 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.714648+0000", 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00032731600000000002 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.715035+0000", 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000386602 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728628+0000", 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013593190999999999 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728671+0000", 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2849000000000003e-05 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755904+0000", 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027233362000000001 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755912+0000", 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9650000000000008e-06 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693075+0000", 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.937162461 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693084+0000", 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3929999999999997e-06 2024-08-29T15:22:36.151 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.931934+0000", 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2388502020000001 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932051+0000", 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117007 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.152 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239384 5.e 5:760d1bed:::benchmark_data_smithi027_135490_object105632:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 510.386911086, 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2039694540000001, 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239384 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.153 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728476+0000", 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728475+0000", 2024-08-29T15:22:36.154 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728482+0000", 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5410000000000003e-06 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728496+0000", 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4047e-05 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.155 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728504+0000", 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3010000000000001e-06 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728714+0000", 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021033 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728818+0000", 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000103436 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.156 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728983+0000", 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016501800000000001 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755958+0000", 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026975717 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755964+0000", 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2480000000000004e-06 2024-08-29T15:22:36.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693203+0000", 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.937239665 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693213+0000", 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7189999999999996e-06 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.158 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932372+0000", 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2391595259999999 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932446+0000", 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3459000000000003e-05 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.159 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239387 5.e 5:742a0183:::benchmark_data_smithi027_135490_object105635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 510.386874998, 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.204054685, 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239387 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.160 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728513+0000", 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.161 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728512+0000", 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728515+0000", 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5450000000000002e-06 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728521+0000", 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.703e-06 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.728526+0000", 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4220000000000002e-06 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.732840+0000", 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043146590000000002 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.733088+0000", 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00024788399999999997 2024-08-29T15:22:36.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.733205+0000", 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011640400000000001 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.755994+0000", 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022789411999999998 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.756004+0000", 2024-08-29T15:22:36.164 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7180000000000008e-06 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693242+0000", 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93723825599999999 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693251+0000", 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9320000000000002e-06 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.165 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932479+0000", 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.239228132 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932567+0000", 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8008000000000002e-05 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239382 5.f 5:fda737cc:::benchmark_data_smithi027_135490_object105630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:22:36.166 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 510.367236392, 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2015833570000001, 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239382 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.167 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748151+0000", 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748149+0000", 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.168 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748157+0000", 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.109e-06 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748168+0000", 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1785e-05 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.169 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748177+0000", 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6270000000000006e-06 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748229+0000", 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2408999999999999e-05 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.748239+0000", 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9629999999999993e-06 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693049+0000", 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.944809695 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693062+0000", 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.29e-05 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.171 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.947700+0000", 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.254637813 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949734+0000", 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020346959999999999 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.172 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239402 5.f 5:f2c183c5:::benchmark_data_smithi027_135490_object105650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 510.35644438700001, 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2791852029999999, 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239402 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.173 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758943+0000", 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758942+0000", 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758947+0000", 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5459999999999999e-06 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.175 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758958+0000", 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0305e-05 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758964+0000", 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0689999999999998e-06 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.758998+0000", 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4258999999999998e-05 2024-08-29T15:22:36.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:05.759005+0000", 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5520000000000001e-06 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693436+0000", 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93443171199999997 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:36.177 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:06.693448+0000", 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1666e-05 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.949915+0000", 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2564665989999999 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.178 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.950510+0000", 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00059576300000000002 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5,6", 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.950770+0000", 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00025979900000000001 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980049+0000", 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029278985 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980078+0000", 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9043e-05 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.981086+0000", 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0010076250000000001 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038008+0000", 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.056922683000000002 2024-08-29T15:22:36.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038051+0000", 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2484000000000002e-05 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038128+0000", 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7443999999999999e-05 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.181 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239405 5.e 5:7ac1943d:::benchmark_data_smithi027_135490_object105653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 508.18312040900003, 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.149048244, 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.182 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239405 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932267+0000", 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932265+0000", 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932278+0000", 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2337999999999999e-05 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.184 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932292+0000", 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3723000000000001e-05 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932301+0000", 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8660000000000001e-06 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932591+0000", 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00028943000000000003 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.932743+0000", 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015197400000000001 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,2,5", 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.933306+0000", 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00056278000000000003 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.941711+0000", 2024-08-29T15:22:36.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0084057430000000002 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.941847+0000", 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136174 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.955062+0000", 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013214889 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081140+0000", 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12607742999999999 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081195+0000", 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4777000000000001e-05 2024-08-29T15:22:36.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.081315+0000", 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012078000000000001 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:239409 5.f 5:f1f503e2:::benchmark_data_smithi027_135490_object105657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 508.170458559, 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093393523000000006, 2024-08-29T15:22:36.189 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239409 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944929+0000", 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944927+0000", 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944936+0000", 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3550000000000005e-06 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944950+0000", 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3862e-05 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.944956+0000", 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3049999999999998e-06 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.976857+0000", 2024-08-29T15:22:36.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031901438999999997 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.976996+0000", 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000138603 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5,6", 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.977150+0000", 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000154559 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980087+0000", 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029361729999999998 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.980644+0000", 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00055748400000000002 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:07.981163+0000", 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00051858500000000001 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038192+0000", 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.057029253000000002 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038240+0000", 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7635e-05 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:08.038322+0000", 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2867999999999993e-05 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.195 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:242790 5.e 5:7ba30800:::benchmark_data_smithi027_135490_object109038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 503.25925879099998, 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100980449, 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:36.196 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242790 2024-08-29T15:22:36.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.197 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.197 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.197 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.197 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:22:36.197 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856129+0000", 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856128+0000", 2024-08-29T15:22:36.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856134+0000", 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.669e-06 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856145+0000", 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0298e-05 2024-08-29T15:22:36.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856150+0000", 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2220000000000001e-06 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856187+0000", 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7452e-05 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856270+0000", 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2522999999999996e-05 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,2,5", 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.856349+0000", 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9153000000000006e-05 2024-08-29T15:22:36.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.859142+0000", 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027929640000000002 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.859413+0000", 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00027162699999999998 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954345+0000", 2024-08-29T15:22:36.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.094931353999999996 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956931+0000", 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025865459999999999 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.956981+0000", 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9325e-05 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.957109+0000", 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012847100000000001 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44573.0:242790 5.e e143/140 ondisk, result = 0)", 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 503.25480599500003, 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093766336000000006, 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:22:36.204 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860582+0000", 2024-08-29T15:22:36.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860581+0000", 2024-08-29T15:22:36.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:36.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.860586+0000", 2024-08-29T15:22:36.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9599999999999999e-06 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954254+0000", 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093668543000000007 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954271+0000", 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6373999999999999e-05 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954325+0000", 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4342999999999998e-05 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954337+0000", 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1558e-05 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954348+0000", 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1318000000000001e-05 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44573.0:242791 5.7 e143/140 5:e93ed9d7:::benchmark_data_smithi027_135490_object109039:head v 143'15344, mlcod=143'15341)", 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 503.24939935600003, 2024-08-29T15:22:36.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.120497535, 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865988+0000", 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865987+0000", 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.865994+0000", 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8990000000000004e-06 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954486+0000", 2024-08-29T15:22:36.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088492092999999994 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954496+0000", 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5550000000000008e-06 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954810+0000", 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031430999999999999 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.954830+0000", 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0162000000000001e-05 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.986452+0000", 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031621761999999998 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:14:12.986486+0000", 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3630000000000002e-05 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:154446 6.2 6:49f57ff2:::benchmark_data_smithi027_169445_object29019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:45.735650+0000", 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 110.379737913, 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021025988999999998, 2024-08-29T15:22:36.215 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.216 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:35 smithi027 ceph-mon[93921]: pgmap v799: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/249924 objects degraded (12.044%); 15591/249924 objects misplaced (6.238%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:36.217 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:35 smithi027 ceph-mon[98970]: pgmap v799: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/249924 objects degraded (12.044%); 15591/249924 objects misplaced (6.238%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:36.217 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.217 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 154446 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735650+0000", 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735650+0000", 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735648+0000", 2024-08-29T15:22:36.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:36.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735657+0000", 2024-08-29T15:22:36.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0589999999999992e-06 2024-08-29T15:22:36.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735673+0000", 2024-08-29T15:22:36.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6521999999999999e-05 2024-08-29T15:22:36.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735683+0000", 2024-08-29T15:22:36.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3279999999999995e-06 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735764+0000", 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1403999999999995e-05 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735853+0000", 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9492000000000003e-05 2024-08-29T15:22:36.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735922+0000", 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8484000000000004e-05 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.738928+0000", 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030056670000000001 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.738967+0000", 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9134000000000002e-05 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.756607+0000", 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017640427 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.756645+0000", 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8203e-05 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.756676+0000", 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0215000000000001e-05 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:159001 6.2 6:43e3c641:::benchmark_data_smithi027_169445_object33574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:24.412053+0000", 2024-08-29T15:22:36.225 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 71.703334937999998, 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022453205, 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 159001 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.226 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412053+0000", 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412053+0000", 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412051+0000", 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412058+0000", 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1799999999999999e-06 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412071+0000", 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2894000000000001e-05 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412077+0000", 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0820000000000004e-06 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412141+0000", 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3800999999999994e-05 2024-08-29T15:22:36.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412232+0000", 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0545999999999997e-05 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412314+0000", 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2047999999999998e-05 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.230 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.415477+0000", 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031632259999999999 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.415885+0000", 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00040792600000000002 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.434420+0000", 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018534761 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.434463+0000", 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3479000000000002e-05 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.434506+0000", 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2688999999999997e-05 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.232 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:159421 6.e 6:73864256:::benchmark_data_smithi027_169445_object33994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:28.183950+0000", 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 67.931437360000004, 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014066255999999999, 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 159421 2024-08-29T15:22:36.233 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183950+0000", 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183950+0000", 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.234 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183949+0000", 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183955+0000", 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5939999999999999e-06 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183969+0000", 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.328e-05 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183976+0000", 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.878e-06 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.184006+0000", 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0870000000000001e-05 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.236 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.184097+0000", 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0328000000000007e-05 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.184173+0000", 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6341000000000007e-05 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.237 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.187293+0000", 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00312013 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.190702+0000", 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034091999999999998 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.196255+0000", 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055525920000000003 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.196292+0000", 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.752e-05 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.198016+0000", 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017239309999999999 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.239 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:159451 6.2 6:41bc5701:::benchmark_data_smithi027_169445_object34024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:28.267758+0000", 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 67.847629182000006, 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013994127, 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 159451 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.240 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267758+0000", 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267758+0000", 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267757+0000", 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267764+0000", 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2840000000000001e-06 2024-08-29T15:22:36.242 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267779+0000", 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.469e-05 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267786+0000", 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7610000000000007e-06 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267839+0000", 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3044000000000002e-05 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.267973+0000", 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000133122 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.268090+0000", 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117823 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.271274+0000", 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003183885 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.271426+0000", 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000151466 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.281639+0000", 2024-08-29T15:22:36.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010213711 2024-08-29T15:22:36.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.281695+0000", 2024-08-29T15:22:36.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5198000000000002e-05 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.281752+0000", 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7840999999999997e-05 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.247 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:161957 6.e 6:7da24517:::benchmark_data_smithi027_169445_object36530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:48.349106+0000", 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 47.766281438999997, 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016709833, 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.248 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.249 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 161957 2024-08-29T15:22:36.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.249 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.249 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.249 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.249 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349106+0000", 2024-08-29T15:22:36.249 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349106+0000", 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349104+0000", 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349114+0000", 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.662e-06 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349148+0000", 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4424000000000001e-05 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349159+0000", 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0261000000000001e-05 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349207+0000", 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8610999999999999e-05 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349312+0000", 2024-08-29T15:22:36.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000104647 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349399+0000", 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7540000000000006e-05 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.352696+0000", 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003296326 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.352734+0000", 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8361000000000001e-05 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.364229+0000", 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011494881 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.364264+0000", 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5108999999999999e-05 2024-08-29T15:22:36.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.365816+0000", 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015519189999999999 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:162115 6.e 6:735d1fd6:::benchmark_data_smithi027_169445_object36688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:49.300856+0000", 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 46.814531746999997, 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012498881, 2024-08-29T15:22:36.255 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 162115 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.257 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.257 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.300856+0000", 2024-08-29T15:22:36.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.300856+0000", 2024-08-29T15:22:36.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.300855+0000", 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.300861+0000", 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4649999999999999e-06 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.300872+0000", 2024-08-29T15:22:36.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1053000000000001e-05 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.300878+0000", 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2530000000000001e-06 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.303559+0000", 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026802599999999998 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.303664+0000", 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010579800000000001 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.303766+0000", 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0001015 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.307568+0000", 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003801909 2024-08-29T15:22:36.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.311808+0000", 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042403689999999999 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.311889+0000", 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1029e-05 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.311938+0000", 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9008999999999998e-05 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:49.313355+0000", 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001416446 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:164550 6.e 6:7edb1c1a:::benchmark_data_smithi027_169445_object39123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:08.491336+0000", 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 27.624051477999998, 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014258033999999999, 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 164550 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491336+0000", 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491336+0000", 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491334+0000", 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:36.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491342+0000", 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1869999999999997e-06 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491358+0000", 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6212e-05 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491366+0000", 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.43e-06 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491414+0000", 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8507000000000002e-05 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491520+0000", 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010611300000000001 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491609+0000", 2024-08-29T15:22:36.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8382999999999997e-05 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.497448+0000", 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0058391609999999998 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.497464+0000", 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6127e-05 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.503546+0000", 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006082222 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.503602+0000", 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6103999999999997e-05 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.505594+0000", 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001991629 2024-08-29T15:22:36.269 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:164551 6.e 6:75c46948:::benchmark_data_smithi027_169445_object39124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:08.493575+0000", 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 27.621812168000002, 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013903413999999999, 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.270 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 164551 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.493575+0000", 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.493575+0000", 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.493574+0000", 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.493580+0000", 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4720000000000003e-06 2024-08-29T15:22:36.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.493592+0000", 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.224e-05 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.493598+0000", 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.784e-06 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.494695+0000", 2024-08-29T15:22:36.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0010967469999999999 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.494791+0000", 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6358000000000001e-05 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.494874+0000", 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2757999999999996e-05 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.497594+0000", 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027202889999999999 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.498780+0000", 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001185911 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.505659+0000", 2024-08-29T15:22:36.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0068792150000000002 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.505726+0000", 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6406000000000001e-05 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.507479+0000", 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017532450000000001 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.276 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:165316 6.e 6:7833455b:::benchmark_data_smithi027_169445_object39889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:14.864936+0000", 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 21.250451971, 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012378768, 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:36.277 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 165316 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.864936+0000", 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.864936+0000", 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.864935+0000", 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.864938+0000", 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3079999999999999e-06 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.864943+0000", 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0889999999999999e-06 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.864947+0000", 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4649999999999999e-06 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.868216+0000", 2024-08-29T15:22:36.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032691280000000001 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.868301+0000", 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5088999999999998e-05 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:22:36.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.868383+0000", 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1428999999999996e-05 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.871001+0000", 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002618263 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.871847+0000", 2024-08-29T15:22:36.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00084646600000000004 2024-08-29T15:22:36.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:36.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.876139+0000", 2024-08-29T15:22:36.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042913839999999997 2024-08-29T15:22:36.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.876174+0000", 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4904999999999999e-05 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:14.877314+0000", 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001140781 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.285 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.286 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.286 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:36.286 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:36.286 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:22:36.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:36.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:36.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:36.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:36.348 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:36.349 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:22:36.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:36.435 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:36.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:36.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:36.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:36.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:36.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:36.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:36.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:36.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:36.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:36.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:36.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:36.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:22:36.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:36.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:36.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:36 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/249924 objects degraded (12.044%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:37.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:37.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:37.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:22:37.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:37.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:37.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:37.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:37.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:37.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:37.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:36 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/249924 objects degraded (12.044%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:37.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:36 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/249924 objects degraded (12.044%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:37.344 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:37.412 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":799,"stamp":"2024-08-29T15:22:34.575056+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5462562813,"num_objects":83308,"num_object_clones":0,"num_object_copies":249924,"num_objects_missing_on_primary":6231,"num_objects_missing":6231,"num_objects_degraded":30102,"num_objects_misplaced":15591,"num_objects_unfound":0,"num_objects_dirty":83308,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":168693,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15236,"num_bytes_recovered":877333709,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124629,"ondisk_log_size":124629,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17793928,"kb_used_data":15969608,"kb_used_omap":43,"kb_used_meta":1824212,"kb_avail":732167288,"statfs":{"total":767960285184,"available":749739302912,"internally_reserved":0,"allocated":16352878592,"data_stored":16341580750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1867993580},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-85393408,"num_objects":-1303,"num_object_clones":0,"num_object_copies":-3909,"num_objects_missing_on_primary":-74,"num_objects_missing":-74,"num_objects_degraded":0,"num_objects_misplaced":-239,"num_objects_unfound":0,"num_objects_dirty":-1303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1303,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":5046272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002855"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":388,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.775752+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:22:22.775752+0000","last_peered":"2024-08-29T15:22:22.775752+0000","last_clean":"2024-08-29T15:22:22.775752+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:22:22.775752+0000","last_undegraded":"2024-08-29T15:22:22.775752+0000","last_fullsized":"2024-08-29T15:22:22.775752+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":29,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:13.726168+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:22:13.726168+0000","last_peered":"2024-08-29T15:22:13.726168+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:22:13.726168+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:22:13.726168+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744885+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:22:22.744885+0000","last_peered":"2024-08-29T15:22:22.744885+0000","last_clean":"2024-08-29T15:22:22.744885+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:22:22.744885+0000","last_undegraded":"2024-08-29T15:22:22.744885+0000","last_fullsized":"2024-08-29T15:22:22.744885+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":356,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874888+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:22:33.874888+0000","last_peered":"2024-08-29T15:22:33.874888+0000","last_clean":"2024-08-29T15:22:33.874888+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:22:33.874888+0000","last_undegraded":"2024-08-29T15:22:33.874888+0000","last_fullsized":"2024-08-29T15:22:33.874888+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":356,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874662+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:22:33.874662+0000","last_peered":"2024-08-29T15:22:33.874662+0000","last_clean":"2024-08-29T15:22:33.874662+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:22:33.874662+0000","last_undegraded":"2024-08-29T15:22:33.874662+0000","last_fullsized":"2024-08-29T15:22:33.874662+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723737+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:22:13.723737+0000","last_peered":"2024-08-29T15:22:13.723737+0000","last_clean":"2024-08-29T15:22:13.723737+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:22:13.723737+0000","last_undegraded":"2024-08-29T15:22:13.723737+0000","last_fullsized":"2024-08-29T15:22:13.723737+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":165,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.745273+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:22:22.745273+0000","last_peered":"2024-08-29T15:22:22.745273+0000","last_clean":"2024-08-29T15:22:22.745273+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:22:22.745273+0000","last_undegraded":"2024-08-29T15:22:22.745273+0000","last_fullsized":"2024-08-29T15:22:22.745273+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":120,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.730934+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:22:13.730934+0000","last_peered":"2024-08-29T15:22:13.730934+0000","last_clean":"2024-08-29T15:22:13.730934+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:22:13.730934+0000","last_undegraded":"2024-08-29T15:22:13.730934+0000","last_fullsized":"2024-08-29T15:22:13.730934+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"252'10627","reported_seq":18657,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:31.035037+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:22:31.035037+0000","last_peered":"2024-08-29T15:22:31.035037+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:22:31.035037+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'600","ondisk_log_start":"175'600","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10027,"log_dups_size":0,"ondisk_log_size":10027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346357760,"num_objects":5285,"num_object_clones":0,"num_object_copies":15855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10627,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2360}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":28,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:13.725951+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:22:13.725951+0000","last_peered":"2024-08-29T15:22:13.725951+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:22:13.725951+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:22:13.725951+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":137,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723442+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:22:13.723442+0000","last_peered":"2024-08-29T15:22:13.723442+0000","last_clean":"2024-08-29T15:22:13.723442+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:22:13.723442+0000","last_undegraded":"2024-08-29T15:22:13.723442+0000","last_fullsized":"2024-08-29T15:22:13.723442+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"252'10360","reported_seq":10542,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:31.695892+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:22:31.695892+0000","last_peered":"2024-08-29T15:22:31.695892+0000","last_clean":"2024-08-29T15:22:31.695892+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:22:31.695892+0000","last_undegraded":"2024-08-29T15:22:31.695892+0000","last_fullsized":"2024-08-29T15:22:31.695892+0000","mapping_epoch":168,"log_start":"179'3628","ondisk_log_start":"179'3628","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6732,"log_dups_size":0,"ondisk_log_size":6732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340131840,"num_objects":5190,"num_object_clones":0,"num_object_copies":15570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10360,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":28,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:13.726866+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:22:13.726866+0000","last_peered":"2024-08-29T15:22:13.726866+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:22:13.726866+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:22:13.726866+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723537+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:22:13.723537+0000","last_peered":"2024-08-29T15:22:13.723537+0000","last_clean":"2024-08-29T15:22:13.723537+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:22:13.723537+0000","last_undegraded":"2024-08-29T15:22:13.723537+0000","last_fullsized":"2024-08-29T15:22:13.723537+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"252'10512","reported_seq":10664,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:31.622831+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:22:31.622831+0000","last_peered":"2024-08-29T15:22:31.622831+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:22:31.622831+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"179'3774","ondisk_log_start":"179'3774","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6738,"log_dups_size":0,"ondisk_log_size":6738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340787200,"num_objects":5200,"num_object_clones":0,"num_object_copies":15600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10512,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2723}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":172,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875175+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:22:33.875175+0000","last_peered":"2024-08-29T15:22:33.875175+0000","last_clean":"2024-08-29T15:22:33.875175+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:22:33.875175+0000","last_undegraded":"2024-08-29T15:22:33.875175+0000","last_fullsized":"2024-08-29T15:22:33.875175+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.731286+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:22:13.731286+0000","last_peered":"2024-08-29T15:22:13.731286+0000","last_clean":"2024-08-29T15:22:13.731286+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:22:13.731286+0000","last_undegraded":"2024-08-29T15:22:13.731286+0000","last_fullsized":"2024-08-29T15:22:13.731286+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"252'10644","reported_seq":16048,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:33.874006+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:22:33.874006+0000","last_peered":"2024-08-29T15:22:33.874006+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:22:33.874006+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"177'2462","ondisk_log_start":"177'2462","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8182,"log_dups_size":0,"ondisk_log_size":8182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336461824,"num_objects":5134,"num_object_clones":0,"num_object_copies":15402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10644,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5134}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":137,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.724459+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:22:13.724459+0000","last_peered":"2024-08-29T15:22:13.724459+0000","last_clean":"2024-08-29T15:22:13.724459+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:22:13.724459+0000","last_undegraded":"2024-08-29T15:22:13.724459+0000","last_fullsized":"2024-08-29T15:22:13.724459+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":370,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874372+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:22:33.874372+0000","last_peered":"2024-08-29T15:22:33.874372+0000","last_clean":"2024-08-29T15:22:33.874372+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:22:33.874372+0000","last_undegraded":"2024-08-29T15:22:33.874372+0000","last_fullsized":"2024-08-29T15:22:33.874372+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"252'10506","reported_seq":10644,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:29.483292+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:22:29.483292+0000","last_peered":"2024-08-29T15:22:29.483292+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:22:29.483292+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:22:29.483292+0000","mapping_epoch":234,"log_start":"177'2533","ondisk_log_start":"177'2533","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7973,"log_dups_size":0,"ondisk_log_size":7973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347865088,"num_objects":5308,"num_object_clones":0,"num_object_copies":15924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10506,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2831},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":363,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.775887+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:22:22.775887+0000","last_peered":"2024-08-29T15:22:22.775887+0000","last_clean":"2024-08-29T15:22:22.775887+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:22:22.775887+0000","last_undegraded":"2024-08-29T15:22:22.775887+0000","last_fullsized":"2024-08-29T15:22:22.775887+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.725873+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:22:13.725873+0000","last_peered":"2024-08-29T15:22:13.725873+0000","last_clean":"2024-08-29T15:22:13.725873+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:22:13.725873+0000","last_undegraded":"2024-08-29T15:22:13.725873+0000","last_fullsized":"2024-08-29T15:22:13.725873+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"252'10545","reported_seq":10743,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:31.632871+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:22:31.632871+0000","last_peered":"2024-08-29T15:22:31.632871+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:22:31.632871+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:22:31.632871+0000","mapping_epoch":234,"log_start":"179'3868","ondisk_log_start":"179'3868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6677,"log_dups_size":0,"ondisk_log_size":6677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347144192,"num_objects":5297,"num_object_clones":0,"num_object_copies":15891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10545,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2832},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.731162+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:22:13.731162+0000","last_peered":"2024-08-29T15:22:13.731162+0000","last_clean":"2024-08-29T15:22:13.731162+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:22:13.731162+0000","last_undegraded":"2024-08-29T15:22:13.731162+0000","last_fullsized":"2024-08-29T15:22:13.731162+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.724377+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:22:13.724377+0000","last_peered":"2024-08-29T15:22:13.724377+0000","last_clean":"2024-08-29T15:22:13.724377+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:22:13.724377+0000","last_undegraded":"2024-08-29T15:22:13.724377+0000","last_fullsized":"2024-08-29T15:22:13.724377+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"252'10397","reported_seq":10780,"reported_epoch":252,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:22:31.412627+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:22:31.412627+0000","last_peered":"2024-08-29T15:22:31.412627+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:22:31.412627+0000","last_undegraded":"2024-08-29T15:22:31.412627+0000","last_fullsized":"2024-08-29T15:22:31.412627+0000","mapping_epoch":202,"log_start":"179'3667","ondisk_log_start":"179'3667","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344784896,"num_objects":5261,"num_object_clones":0,"num_object_copies":15783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5261,"num_objects_unfound":0,"num_objects_dirty":5261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10397,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5261}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723604+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:22:13.723604+0000","last_peered":"2024-08-29T15:22:13.723604+0000","last_clean":"2024-08-29T15:22:13.723604+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:22:13.723604+0000","last_undegraded":"2024-08-29T15:22:13.723604+0000","last_fullsized":"2024-08-29T15:22:13.723604+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":235,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.726984+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:22:13.726984+0000","last_peered":"2024-08-29T15:22:13.726984+0000","last_clean":"2024-08-29T15:22:13.726984+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:22:13.726984+0000","last_undegraded":"2024-08-29T15:22:13.726984+0000","last_fullsized":"2024-08-29T15:22:13.726984+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"252'10507","reported_seq":10639,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:30.806158+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:22:30.806158+0000","last_peered":"2024-08-29T15:22:30.806158+0000","last_clean":"2024-08-29T15:22:30.806158+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:22:30.806158+0000","last_undegraded":"2024-08-29T15:22:30.806158+0000","last_fullsized":"2024-08-29T15:22:30.806158+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9229,"log_dups_size":0,"ondisk_log_size":9229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344195096,"num_objects":5253,"num_object_clones":0,"num_object_copies":15759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5253,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10507,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":366,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.776334+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:22:22.776334+0000","last_peered":"2024-08-29T15:22:22.776334+0000","last_clean":"2024-08-29T15:22:22.776334+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:22:22.776334+0000","last_undegraded":"2024-08-29T15:22:22.776334+0000","last_fullsized":"2024-08-29T15:22:22.776334+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":358,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.745353+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:22:22.745353+0000","last_peered":"2024-08-29T15:22:22.745353+0000","last_clean":"2024-08-29T15:22:22.745353+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:22:22.745353+0000","last_undegraded":"2024-08-29T15:22:22.745353+0000","last_fullsized":"2024-08-29T15:22:22.745353+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"252'10178","reported_seq":10358,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:31.045817+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:22:31.045817+0000","last_peered":"2024-08-29T15:22:31.045817+0000","last_clean":"2024-08-29T15:22:31.045817+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:22:31.045817+0000","last_undegraded":"2024-08-29T15:22:31.045817+0000","last_fullsized":"2024-08-29T15:22:31.045817+0000","mapping_epoch":201,"log_start":"174'100","ondisk_log_start":"174'100","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10078,"log_dups_size":0,"ondisk_log_size":10078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329646080,"num_objects":5030,"num_object_clones":0,"num_object_copies":15090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10178,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":459,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874205+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:22:33.874205+0000","last_peered":"2024-08-29T15:22:33.874205+0000","last_clean":"2024-08-29T15:22:33.874205+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:22:33.874205+0000","last_undegraded":"2024-08-29T15:22:33.874205+0000","last_fullsized":"2024-08-29T15:22:33.874205+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":721,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723879+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:22:13.723879+0000","last_peered":"2024-08-29T15:22:13.723879+0000","last_clean":"2024-08-29T15:22:13.723879+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:22:13.723879+0000","last_undegraded":"2024-08-29T15:22:13.723879+0000","last_fullsized":"2024-08-29T15:22:13.723879+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":229,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.724994+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:22:13.724994+0000","last_peered":"2024-08-29T15:22:13.724994+0000","last_clean":"2024-08-29T15:22:13.724994+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:22:13.724994+0000","last_undegraded":"2024-08-29T15:22:13.724994+0000","last_fullsized":"2024-08-29T15:22:13.724994+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.731183+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:22:13.731183+0000","last_peered":"2024-08-29T15:22:13.731183+0000","last_clean":"2024-08-29T15:22:13.731183+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:22:13.731183+0000","last_undegraded":"2024-08-29T15:22:13.731183+0000","last_fullsized":"2024-08-29T15:22:13.731183+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"252'10527","reported_seq":10907,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:30.488247+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:22:30.488247+0000","last_peered":"2024-08-29T15:22:30.488247+0000","last_clean":"2024-08-29T15:22:30.488247+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:22:30.488247+0000","last_undegraded":"2024-08-29T15:22:30.488247+0000","last_fullsized":"2024-08-29T15:22:30.488247+0000","mapping_epoch":201,"log_start":"178'3453","ondisk_log_start":"178'3453","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337838080,"num_objects":5155,"num_object_clones":0,"num_object_copies":15465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10527,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":831,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744429+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:22:22.744429+0000","last_peered":"2024-08-29T15:22:22.744429+0000","last_clean":"2024-08-29T15:22:22.744429+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:22:22.744429+0000","last_undegraded":"2024-08-29T15:22:22.744429+0000","last_fullsized":"2024-08-29T15:22:22.744429+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":371,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874506+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:22:33.874506+0000","last_peered":"2024-08-29T15:22:33.874506+0000","last_clean":"2024-08-29T15:22:33.874506+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:22:33.874506+0000","last_undegraded":"2024-08-29T15:22:33.874506+0000","last_fullsized":"2024-08-29T15:22:33.874506+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":379,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874912+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:22:33.874912+0000","last_peered":"2024-08-29T15:22:33.874912+0000","last_clean":"2024-08-29T15:22:33.874912+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:22:33.874912+0000","last_undegraded":"2024-08-29T15:22:33.874912+0000","last_fullsized":"2024-08-29T15:22:33.874912+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"252'10429","reported_seq":16232,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:31.613940+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:22:31.613940+0000","last_peered":"2024-08-29T15:22:31.613940+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:22:31.613940+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"179'3730","ondisk_log_start":"179'3730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344784896,"num_objects":5261,"num_object_clones":0,"num_object_copies":15783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10429,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":827},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":639,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.775850+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:22:22.775850+0000","last_peered":"2024-08-29T15:22:22.775850+0000","last_clean":"2024-08-29T15:22:22.775850+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:22:22.775850+0000","last_undegraded":"2024-08-29T15:22:22.775850+0000","last_fullsized":"2024-08-29T15:22:22.775850+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":82,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:19.999920+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:22:19.999920+0000","last_peered":"2024-08-29T15:22:19.999920+0000","last_clean":"2024-08-29T15:22:19.999920+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:22:19.999920+0000","last_undegraded":"2024-08-29T15:22:19.999920+0000","last_fullsized":"2024-08-29T15:22:19.999920+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":403,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.745142+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:22:22.745142+0000","last_peered":"2024-08-29T15:22:22.745142+0000","last_clean":"2024-08-29T15:22:22.745142+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:22:22.745142+0000","last_undegraded":"2024-08-29T15:22:22.745142+0000","last_fullsized":"2024-08-29T15:22:22.745142+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"252'10498","reported_seq":10690,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:31.515017+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:22:31.515017+0000","last_peered":"2024-08-29T15:22:31.515017+0000","last_clean":"2024-08-29T15:22:31.515017+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:22:31.515017+0000","last_undegraded":"2024-08-29T15:22:31.515017+0000","last_fullsized":"2024-08-29T15:22:31.515017+0000","mapping_epoch":234,"log_start":"179'3797","ondisk_log_start":"179'3797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6701,"log_dups_size":0,"ondisk_log_size":6701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341704704,"num_objects":5214,"num_object_clones":0,"num_object_copies":15642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10498,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":189,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723760+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:22:13.723760+0000","last_peered":"2024-08-29T15:22:13.723760+0000","last_clean":"2024-08-29T15:22:13.723760+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:22:13.723760+0000","last_undegraded":"2024-08-29T15:22:13.723760+0000","last_fullsized":"2024-08-29T15:22:13.723760+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.725885+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:22:13.725885+0000","last_peered":"2024-08-29T15:22:13.725885+0000","last_clean":"2024-08-29T15:22:13.725885+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:22:13.725885+0000","last_undegraded":"2024-08-29T15:22:13.725885+0000","last_fullsized":"2024-08-29T15:22:13.725885+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.724099+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:22:13.724099+0000","last_peered":"2024-08-29T15:22:13.724099+0000","last_clean":"2024-08-29T15:22:13.724099+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:22:13.724099+0000","last_undegraded":"2024-08-29T15:22:13.724099+0000","last_fullsized":"2024-08-29T15:22:13.724099+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"252'10563","reported_seq":10755,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:33.873836+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:22:33.873836+0000","last_peered":"2024-08-29T15:22:33.873836+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:22:33.873836+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"177'2442","ondisk_log_start":"177'2442","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8121,"log_dups_size":0,"ondisk_log_size":8121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344915968,"num_objects":5263,"num_object_clones":0,"num_object_copies":15789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10563,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2840}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":121,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723648+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:22:13.723648+0000","last_peered":"2024-08-29T15:22:13.723648+0000","last_clean":"2024-08-29T15:22:13.723648+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:22:13.723648+0000","last_undegraded":"2024-08-29T15:22:13.723648+0000","last_fullsized":"2024-08-29T15:22:13.723648+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":796,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.745784+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:22:22.745784+0000","last_peered":"2024-08-29T15:22:22.745784+0000","last_clean":"2024-08-29T15:22:22.745784+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:22:22.745784+0000","last_undegraded":"2024-08-29T15:22:22.745784+0000","last_fullsized":"2024-08-29T15:22:22.745784+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":373,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744729+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:22:22.744729+0000","last_peered":"2024-08-29T15:22:22.744729+0000","last_clean":"2024-08-29T15:22:22.744729+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:22:22.744729+0000","last_undegraded":"2024-08-29T15:22:22.744729+0000","last_fullsized":"2024-08-29T15:22:22.744729+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"252'10478","reported_seq":11224,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:30.634915+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:22:30.634915+0000","last_peered":"2024-08-29T15:22:30.634915+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:22:30.634915+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"178'3480","ondisk_log_start":"178'3480","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340131840,"num_objects":5190,"num_object_clones":0,"num_object_copies":15570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5156,"num_objects_unfound":0,"num_objects_dirty":5190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10478,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":3039}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":378,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744636+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:22:22.744636+0000","last_peered":"2024-08-29T15:22:22.744636+0000","last_clean":"2024-08-29T15:22:22.744636+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:22:22.744636+0000","last_undegraded":"2024-08-29T15:22:22.744636+0000","last_fullsized":"2024-08-29T15:22:22.744636+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":678,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:13.726444+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:22:13.726444+0000","last_peered":"2024-08-29T15:22:13.726444+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:22:13.726444+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:22:13.726444+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":371,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874293+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:22:33.874293+0000","last_peered":"2024-08-29T15:22:33.874293+0000","last_clean":"2024-08-29T15:22:33.874293+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:22:33.874293+0000","last_undegraded":"2024-08-29T15:22:33.874293+0000","last_fullsized":"2024-08-29T15:22:33.874293+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"252'10376","reported_seq":17512,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:29.691491+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:22:29.691491+0000","last_peered":"2024-08-29T15:22:29.691491+0000","last_clean":"2024-08-29T15:22:29.691491+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:22:29.691491+0000","last_undegraded":"2024-08-29T15:22:29.691491+0000","last_fullsized":"2024-08-29T15:22:29.691491+0000","mapping_epoch":234,"log_start":"177'2399","ondisk_log_start":"177'2399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331087872,"num_objects":5052,"num_object_clones":0,"num_object_copies":15156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10376,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":363,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.781890+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:22:22.781890+0000","last_peered":"2024-08-29T15:22:22.781890+0000","last_clean":"2024-08-29T15:22:22.781890+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:22:22.781890+0000","last_undegraded":"2024-08-29T15:22:22.781890+0000","last_fullsized":"2024-08-29T15:22:22.781890+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":985,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:22.745491+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:22:22.745491+0000","last_peered":"2024-08-29T15:22:22.745491+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:22:22.745491+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":360,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.775625+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:22:22.775625+0000","last_peered":"2024-08-29T15:22:22.775625+0000","last_clean":"2024-08-29T15:22:22.775625+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:22:22.775625+0000","last_undegraded":"2024-08-29T15:22:22.775625+0000","last_fullsized":"2024-08-29T15:22:22.775625+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"252'10423","reported_seq":13573,"reported_epoch":252,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:30.231570+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:22:30.231570+0000","last_peered":"2024-08-29T15:22:30.231570+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:22:30.231570+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'2912","ondisk_log_start":"178'2912","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339935232,"num_objects":5187,"num_object_clones":0,"num_object_copies":15561,"num_objects_missing_on_primary":6223,"num_objects_missing":6223,"num_objects_degraded":2313,"num_objects_misplaced":5174,"num_objects_unfound":0,"num_objects_dirty":5187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10423,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1500,"num_bytes_recovered":98172928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":12},{"shards":"1,2,7","objects":2803},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":361,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874560+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:22:33.874560+0000","last_peered":"2024-08-29T15:22:33.874560+0000","last_clean":"2024-08-29T15:22:33.874560+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:22:33.874560+0000","last_undegraded":"2024-08-29T15:22:33.874560+0000","last_fullsized":"2024-08-29T15:22:33.874560+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":102,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.731490+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:22:13.731490+0000","last_peered":"2024-08-29T15:22:13.731490+0000","last_clean":"2024-08-29T15:22:13.731490+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:22:13.731490+0000","last_undegraded":"2024-08-29T15:22:13.731490+0000","last_fullsized":"2024-08-29T15:22:13.731490+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.723664+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:22:13.723664+0000","last_peered":"2024-08-29T15:22:13.723664+0000","last_clean":"2024-08-29T15:22:13.723664+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:22:13.723664+0000","last_undegraded":"2024-08-29T15:22:13.723664+0000","last_fullsized":"2024-08-29T15:22:13.723664+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.725975+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:22:13.725975+0000","last_peered":"2024-08-29T15:22:13.725975+0000","last_clean":"2024-08-29T15:22:13.725975+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:22:13.725975+0000","last_undegraded":"2024-08-29T15:22:13.725975+0000","last_fullsized":"2024-08-29T15:22:13.725975+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":28,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:13.725500+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:22:13.725500+0000","last_peered":"2024-08-29T15:22:13.725500+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:22:13.725500+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:22:13.725500+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":367,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744545+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:22:22.744545+0000","last_peered":"2024-08-29T15:22:22.744545+0000","last_clean":"2024-08-29T15:22:22.744545+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:22:22.744545+0000","last_undegraded":"2024-08-29T15:22:22.744545+0000","last_fullsized":"2024-08-29T15:22:22.744545+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":117,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:22.776341+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:22:22.776341+0000","last_peered":"2024-08-29T15:22:22.776341+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:22:22.776341+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:22:22.776341+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":119,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744902+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:22:22.744902+0000","last_peered":"2024-08-29T15:22:22.744902+0000","last_clean":"2024-08-29T15:22:22.744902+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:22:22.744902+0000","last_undegraded":"2024-08-29T15:22:22.744902+0000","last_fullsized":"2024-08-29T15:22:22.744902+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":326,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.775513+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:22:22.775513+0000","last_peered":"2024-08-29T15:22:22.775513+0000","last_clean":"2024-08-29T15:22:22.775513+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:22:22.775513+0000","last_undegraded":"2024-08-29T15:22:22.775513+0000","last_fullsized":"2024-08-29T15:22:22.775513+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":382,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875020+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:22:33.875020+0000","last_peered":"2024-08-29T15:22:33.875020+0000","last_clean":"2024-08-29T15:22:33.875020+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:22:33.875020+0000","last_undegraded":"2024-08-29T15:22:33.875020+0000","last_fullsized":"2024-08-29T15:22:33.875020+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":378,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.775532+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:22:22.775532+0000","last_peered":"2024-08-29T15:22:22.775532+0000","last_clean":"2024-08-29T15:22:22.775532+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:22:22.775532+0000","last_undegraded":"2024-08-29T15:22:22.775532+0000","last_fullsized":"2024-08-29T15:22:22.775532+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":90,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.731557+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:22:13.731557+0000","last_peered":"2024-08-29T15:22:13.731557+0000","last_clean":"2024-08-29T15:22:13.731557+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:22:13.731557+0000","last_undegraded":"2024-08-29T15:22:13.731557+0000","last_fullsized":"2024-08-29T15:22:13.731557+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874065+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:22:33.874065+0000","last_peered":"2024-08-29T15:22:33.874065+0000","last_clean":"2024-08-29T15:22:33.874065+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:22:33.874065+0000","last_undegraded":"2024-08-29T15:22:33.874065+0000","last_fullsized":"2024-08-29T15:22:33.874065+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":357,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874813+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:22:33.874813+0000","last_peered":"2024-08-29T15:22:33.874813+0000","last_clean":"2024-08-29T15:22:33.874813+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:22:33.874813+0000","last_undegraded":"2024-08-29T15:22:33.874813+0000","last_fullsized":"2024-08-29T15:22:33.874813+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":359,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746045+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:22:22.746045+0000","last_peered":"2024-08-29T15:22:22.746045+0000","last_clean":"2024-08-29T15:22:22.746045+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:22:22.746045+0000","last_undegraded":"2024-08-29T15:22:22.746045+0000","last_fullsized":"2024-08-29T15:22:22.746045+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":363,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.776202+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:22:22.776202+0000","last_peered":"2024-08-29T15:22:22.776202+0000","last_clean":"2024-08-29T15:22:22.776202+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:22:22.776202+0000","last_undegraded":"2024-08-29T15:22:22.776202+0000","last_fullsized":"2024-08-29T15:22:22.776202+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:19.789784+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:22:19.789784+0000","last_peered":"2024-08-29T15:22:19.789784+0000","last_clean":"2024-08-29T15:22:19.789784+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:22:19.789784+0000","last_undegraded":"2024-08-29T15:22:19.789784+0000","last_fullsized":"2024-08-29T15:22:19.789784+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":383,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.745451+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:22:22.745451+0000","last_peered":"2024-08-29T15:22:22.745451+0000","last_clean":"2024-08-29T15:22:22.745451+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:22:22.745451+0000","last_undegraded":"2024-08-29T15:22:22.745451+0000","last_fullsized":"2024-08-29T15:22:22.745451+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":349,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874197+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:22:33.874197+0000","last_peered":"2024-08-29T15:22:33.874197+0000","last_clean":"2024-08-29T15:22:33.874197+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:22:33.874197+0000","last_undegraded":"2024-08-29T15:22:33.874197+0000","last_fullsized":"2024-08-29T15:22:33.874197+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:19.789817+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:22:19.789817+0000","last_peered":"2024-08-29T15:22:19.789817+0000","last_clean":"2024-08-29T15:22:19.789817+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:22:19.789817+0000","last_undegraded":"2024-08-29T15:22:19.789817+0000","last_fullsized":"2024-08-29T15:22:19.789817+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":142,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:13.724227+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:22:13.724227+0000","last_peered":"2024-08-29T15:22:13.724227+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:22:13.724227+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:22:13.724227+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:19.789800+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:22:19.789800+0000","last_peered":"2024-08-29T15:22:19.789800+0000","last_clean":"2024-08-29T15:22:19.789800+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:22:19.789800+0000","last_undegraded":"2024-08-29T15:22:19.789800+0000","last_fullsized":"2024-08-29T15:22:19.789800+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":148,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:13.725216+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:22:13.725216+0000","last_peered":"2024-08-29T15:22:13.725216+0000","last_clean":"2024-08-29T15:22:13.725216+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:22:13.725216+0000","last_undegraded":"2024-08-29T15:22:13.725216+0000","last_fullsized":"2024-08-29T15:22:13.725216+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":394,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.775953+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:22:22.775953+0000","last_peered":"2024-08-29T15:22:22.775953+0000","last_clean":"2024-08-29T15:22:22.775953+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:22:22.775953+0000","last_undegraded":"2024-08-29T15:22:22.775953+0000","last_fullsized":"2024-08-29T15:22:22.775953+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":395,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.776172+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:22:22.776172+0000","last_peered":"2024-08-29T15:22:22.776172+0000","last_clean":"2024-08-29T15:22:22.776172+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:22:22.776172+0000","last_undegraded":"2024-08-29T15:22:22.776172+0000","last_fullsized":"2024-08-29T15:22:22.776172+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":30,"reported_epoch":251,"state":"active+clean","last_fresh":"2024-08-29T15:22:14.789170+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:22:14.789170+0000","last_peered":"2024-08-29T15:22:14.789170+0000","last_clean":"2024-08-29T15:22:14.789170+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:22:14.789170+0000","last_undegraded":"2024-08-29T15:22:14.789170+0000","last_fullsized":"2024-08-29T15:22:14.789170+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":405,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874068+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:22:33.874068+0000","last_peered":"2024-08-29T15:22:33.874068+0000","last_clean":"2024-08-29T15:22:33.874068+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:22:33.874068+0000","last_undegraded":"2024-08-29T15:22:33.874068+0000","last_fullsized":"2024-08-29T15:22:33.874068+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5457772568,"num_objects":83280,"num_object_clones":0,"num_object_copies":249840,"num_objects_missing_on_primary":6223,"num_objects_missing":6223,"num_objects_degraded":30073,"num_objects_misplaced":15591,"num_objects_unfound":0,"num_objects_dirty":83280,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167570,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15127,"num_bytes_recovered":871366656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16302616576,"data_stored":16302604360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123447,"ondisk_log_size":123447,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234334,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2993176,"kb_used_data":2683152,"kb_used_omap":1,"kb_used_meta":310014,"kb_avail":90751976,"statfs":{"total":95995035648,"available":92930023424,"internally_reserved":0,"allocated":2747547648,"data_stored":2746123771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":317454795},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495966,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":374804,"kb_used_data":217356,"kb_used_omap":1,"kb_used_meta":157438,"kb_avail":93370348,"statfs":{"total":95995035648,"available":95611236352,"internally_reserved":0,"allocated":222572544,"data_stored":221152467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":161216967},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426552,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1723464,"kb_used_data":1668352,"kb_used_omap":8,"kb_used_meta":55095,"kb_avail":92021688,"statfs":{"total":95995035648,"available":94230208512,"internally_reserved":0,"allocated":1708392448,"data_stored":1706979837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":56418186},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019244,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989752,"kb_used_data":1665776,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":91755400,"statfs":{"total":95995035648,"available":93957529600,"internally_reserved":0,"allocated":1705754624,"data_stored":1704357796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435429,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3621508,"kb_used_data":3324732,"kb_used_omap":4,"kb_used_meta":296763,"kb_avail":90123644,"statfs":{"total":95995035648,"available":92286611456,"internally_reserved":0,"allocated":3404525568,"data_stored":3403116897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":303885577},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903281,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3078600,"kb_used_data":2960768,"kb_used_omap":10,"kb_used_meta":117813,"kb_avail":90666552,"statfs":{"total":95995035648,"available":92842549248,"internally_reserved":0,"allocated":3031826432,"data_stored":3030417697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":120641040},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762482,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2360772,"kb_used_data":2104188,"kb_used_omap":7,"kb_used_meta":256568,"kb_avail":91384380,"statfs":{"total":95995035648,"available":93577605120,"internally_reserved":0,"allocated":2154688512,"data_stored":2153278601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":262725986},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926009,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1651852,"kb_used_data":1345284,"kb_used_omap":7,"kb_used_meta":306552,"kb_avail":92093300,"statfs":{"total":95995035648,"available":94303539200,"internally_reserved":0,"allocated":1377570816,"data_stored":1376153684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":313909391},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1368723456,"data_stored":1368719384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2145062912,"data_stored":2145058840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3027238912,"data_stored":3027238912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3401121792,"data_stored":3401121792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1702100992,"data_stored":1702100992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1702563840,"data_stored":1702559768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":217710592,"data_stored":217710592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2738094080,"data_stored":2738094080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:22:37.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:37.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:37.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:37.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:37.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:37.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:37.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:37.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:37.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:37.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:37.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:37.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:37 smithi171 ceph-mon[90031]: from='client.35622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:37 smithi171 ceph-mon[90031]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/248976 objects degraded (12.090%); 15509/248976 objects misplaced (6.229%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:22:38.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:37 smithi027 ceph-mon[98970]: from='client.35622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:37 smithi027 ceph-mon[98970]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/248976 objects degraded (12.090%); 15509/248976 objects misplaced (6.229%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:22:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:37 smithi027 ceph-mon[93921]: from='client.35622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:37 smithi027 ceph-mon[93921]: pgmap v800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/248976 objects degraded (12.090%); 15509/248976 objects misplaced (6.229%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:22:38.070 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:38.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:38.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:38.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:38.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:38.318 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:38.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:38.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:38.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:38.572 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:38.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:38.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:38.712 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:38.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:38.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:38.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:38.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:38.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:39.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:39.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:39.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:39.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:39.203 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:39.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:39.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:39.313 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:22:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:22:39] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-08-29T15:22:39.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:39.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:39.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:39.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:39.411 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:22:39.411 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:22:39.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.411 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168262 6.3 6:c56fe3c1:::benchmark_data_smithi027_169445_object42835:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.411 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:37.820739+0000", 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.5877994549999999, 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.587827324, 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168262 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.820739+0000", 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.820739+0000", 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.820737+0000", 2024-08-29T15:22:39.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.820746+0000", 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5400000000000001e-06 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.820760+0000", 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3665e-05 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.820767+0000", 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7859999999999999e-06 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.921257+0000", 2024-08-29T15:22:39.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10049058499999999 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.934614+0000", 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013356498 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.992321+0000", 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.057707660000000001 2024-08-29T15:22:39.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.001678+0000", 2024-08-29T15:22:39.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0093562839999999994 2024-08-29T15:22:39.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.390184+0000", 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3885066020000001 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.396441+0000", 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0062566849999999997 2024-08-29T15:22:39.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.408426+0000", 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011985179 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168435 6.3 6:c8d8b303:::benchmark_data_smithi027_169445_object43008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:38.686870+0000", 2024-08-29T15:22:39.419 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.72166894299999995, 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.72178146700000001, 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168435 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.686870+0000", 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.686870+0000", 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.686868+0000", 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.686877+0000", 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0289999999999994e-06 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.686895+0000", 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8260000000000001e-05 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.686904+0000", 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0469999999999999e-06 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.705778+0000", 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018873319999999999 2024-08-29T15:22:39.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.714341+0000", 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0085637160000000007 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.780758+0000", 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066417087 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.787783+0000", 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007024728 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168283 6.3 6:c3835e2f:::benchmark_data_smithi027_169445_object42856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:37.863063+0000", 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.545475347, 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5456595819999999, 2024-08-29T15:22:39.425 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168283 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.863063+0000", 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.863063+0000", 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.863062+0000", 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.863070+0000", 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5690000000000005e-06 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.863084+0000", 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4123e-05 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.863094+0000", 2024-08-29T15:22:39.429 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3859999999999996e-06 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.936409+0000", 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.073314988999999997 2024-08-29T15:22:39.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.950363+0000", 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013954713000000001 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.034223+0000", 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083860158000000004 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.041876+0000", 2024-08-29T15:22:39.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007652613 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168486 6.3 6:c692e1fd:::benchmark_data_smithi027_169445_object43059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:39.296344+0000", 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.11219425199999999, 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11242271700000001, 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for missing object", 2024-08-29T15:22:39.432 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168486 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.296344+0000", 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.296344+0000", 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.296343+0000", 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.296350+0000", 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5830000000000001e-06 2024-08-29T15:22:39.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.296363+0000", 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2116999999999999e-05 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.296371+0000", 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5129999999999996e-06 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.356274+0000", 2024-08-29T15:22:39.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.059902756000000001 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.367461+0000", 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011187420999999999 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168446 6.3 6:cd884caa:::benchmark_data_smithi027_169445_object43019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.436 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:38.705724+0000", 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.70281501800000001, 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.70307858599999995, 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168446 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.705724+0000", 2024-08-29T15:22:39.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.705724+0000", 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.705722+0000", 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.705729+0000", 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3579999999999997e-06 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.705741+0000", 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1941e-05 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.705747+0000", 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4450000000000001e-06 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.829215+0000", 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12346744 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.837261+0000", 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0080462280000000008 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.893658+0000", 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.056397220999999997 2024-08-29T15:22:39.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.903253+0000", 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0095951839999999997 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPush(6.3 253/218 [PushOp(6:c692e1fd:::benchmark_data_smithi027_169445_object43059:head, version: 177'2702, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:c692e1fd:::benchmark_data_smithi027_169445_object43059:head@177'2702, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:39.368977+0000", 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.039561853000000001, 2024-08-29T15:22:39.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039878159000000003, 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.368977+0000", 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.368977+0000", 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.368976+0000", 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.369047+0000", 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0962000000000003e-05 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.369065+0000", 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8695999999999999e-05 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.369079+0000", 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3760999999999999e-05 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168292 6.3 6:c7fac34d:::benchmark_data_smithi027_169445_object42865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.444 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:37.881555+0000", 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.5269834119999999, 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5273275559999999, 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168292 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.881555+0000", 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.881555+0000", 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.881553+0000", 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.881563+0000", 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8500000000000006e-06 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.881585+0000", 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2648999999999999e-05 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.881595+0000", 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6819999999999998e-06 2024-08-29T15:22:39.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.952028+0000", 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.070433486000000003 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.960311+0000", 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0082822739999999992 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.448 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.072371+0000", 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.112060512 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.079742+0000", 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0073706939999999997 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168234 6.3 6:c874b0ea:::benchmark_data_smithi027_169445_object42807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.449 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:37.327637+0000", 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.0809010859999999, 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0812857400000002, 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168234 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.327637+0000", 2024-08-29T15:22:39.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.327637+0000", 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.327636+0000", 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.327642+0000", 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3609999999999997e-06 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.327654+0000", 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1806e-05 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.327660+0000", 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4009999999999996e-06 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.353003+0000", 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025343075999999999 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.364429+0000", 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011426472999999999 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.463520+0000", 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.099091243999999995 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.470977+0000", 2024-08-29T15:22:39.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0074562020000000003 2024-08-29T15:22:39.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.262315+0000", 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.791338037 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.269002+0000", 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0066874780000000002 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:22:39.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.288477+0000", 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019474768999999999 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.351392+0000", 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062915203000000003 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.354980+0000", 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035882100000000001 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.457 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168456 6.3 6:c98189f6:::benchmark_data_smithi027_169445_object43029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:38.733345+0000", 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.67519396700000001, 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.67563288600000004, 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168456 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.733345+0000", 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.733345+0000", 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.733343+0000", 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.733351+0000", 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2209999999999997e-06 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.733364+0000", 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2835e-05 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.733373+0000", 2024-08-29T15:22:39.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8380000000000008e-06 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.838445+0000", 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.105071941 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.848915+0000", 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010469869999999999 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.929264+0000", 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080349201999999995 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.936530+0000", 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072662259999999998 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168301 6.3 6:c58171f0:::benchmark_data_smithi027_169445_object42874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.462 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:37.903905+0000", 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.5046335749999999, 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.505112316, 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168301 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.903905+0000", 2024-08-29T15:22:39.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.903905+0000", 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.903904+0000", 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.903911+0000", 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7510000000000002e-06 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.903922+0000", 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1527e-05 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.903928+0000", 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3249999999999996e-06 2024-08-29T15:22:39.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.082400+0000", 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17847155100000001 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.089786+0000", 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0073854860000000001 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.466 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.132277+0000", 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042491864999999997 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.139380+0000", 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0071026070000000004 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168309 6.3 6:c9cea70c:::benchmark_data_smithi027_169445_object42882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:37.925418+0000", 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.4831200899999999, 2024-08-29T15:22:39.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.483639046, 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168309 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.925418+0000", 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.925418+0000", 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.925417+0000", 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.469 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.925426+0000", 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0429999999999996e-06 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.925441+0000", 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5221999999999999e-05 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:37.925450+0000", 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8349999999999993e-06 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.090466+0000", 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.165015681 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.097725+0000", 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072589430000000003 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.200041+0000", 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.102315875 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.207210+0000", 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007169941 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168459 6.3 6:ccd329e6:::benchmark_data_smithi027_169445_object43032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:38.745059+0000", 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.66347983899999996, 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.66403888600000005, 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.472 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168459 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.745059+0000", 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.745059+0000", 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.745057+0000", 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.745064+0000", 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0890000000000002e-06 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.745076+0000", 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1846e-05 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.745083+0000", 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5549999999999999e-06 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.852479+0000", 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10739576300000001 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.860445+0000", 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079664969999999995 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.962113+0000", 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.101668021 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.969545+0000", 2024-08-29T15:22:39.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0074322670000000002 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168372 6.3 6:c87ef878:::benchmark_data_smithi027_169445_object42945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:38.551063+0000", 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.85747511899999995, 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.85807856100000002, 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.477 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168372 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.551063+0000", 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.551063+0000", 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.551061+0000", 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.551072+0000", 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1578e-05 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.551090+0000", 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7989999999999999e-05 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.551100+0000", 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4490000000000005e-06 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.682965+0000", 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13186535399999999 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.692874+0000", 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099088330000000006 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.745547+0000", 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.052672515000000003 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.752784+0000", 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072374819999999999 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.481 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168383 6.3 6:c1ca4150:::benchmark_data_smithi027_169445_object42956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:38.573119+0000", 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.83541955800000001, 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.83606299900000003, 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168383 2024-08-29T15:22:39.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.573119+0000", 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.573119+0000", 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.573117+0000", 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.573124+0000", 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1210000000000001e-06 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.573140+0000", 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5960999999999998e-05 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.573149+0000", 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6309999999999992e-06 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.694174+0000", 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.121024783 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.704952+0000", 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010777709 2024-08-29T15:22:39.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.814826+0000", 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.109874019 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.825685+0000", 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010859743 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.487 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168463 6.3 6:cb8313a6:::benchmark_data_smithi027_169445_object43036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.487 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:39.087548+0000", 2024-08-29T15:22:39.487 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.32099011999999999, 2024-08-29T15:22:39.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.32167758699999999, 2024-08-29T15:22:39.487 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:39.488 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168463 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.087548+0000", 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.087548+0000", 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.087547+0000", 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.087554+0000", 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2759999999999997e-06 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.087566+0000", 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2483000000000001e-05 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.087575+0000", 2024-08-29T15:22:39.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6009999999999995e-06 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.112791+0000", 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025216912000000001 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.124621+0000", 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011829335 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.199451+0000", 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.074830363999999996 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.206772+0000", 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0073212629999999997 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168339 6.3 6:c61242f6:::benchmark_data_smithi027_169445_object42912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.493 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:38.064173+0000", 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.344365416, 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3451004010000001, 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168339 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.494 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.064173+0000", 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.064173+0000", 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.064171+0000", 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:39.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.064179+0000", 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4200000000000001e-06 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.064193+0000", 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4440999999999999e-05 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.064200+0000", 2024-08-29T15:22:39.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0280000000000003e-06 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.098470+0000", 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034270084999999999 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.105847+0000", 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007376184 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.165004+0000", 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.059157578000000002 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:38.172299+0000", 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072950910000000001 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:168471 6.3 6:cc94bac6:::benchmark_data_smithi027_169445_object43044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:39.134246+0000", 2024-08-29T15:22:39.498 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.274292551, 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.275067966, 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 168471 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.134246+0000", 2024-08-29T15:22:39.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.134246+0000", 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.134244+0000", 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:39.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.134252+0000", 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1110000000000008e-06 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.134266+0000", 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3326000000000001e-05 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.134273+0000", 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3150000000000003e-06 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.232750+0000", 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098477312999999997 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.240153+0000", 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0074029780000000002 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.335365+0000", 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095211445000000006 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:39.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:39.347195+0000", 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011830284 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 17 2024-08-29T15:22:39.503 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:22:39.504 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:22:39.504 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:22:39.504 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:22:39.504 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:22:39.505 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:22:39.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:22:39.505 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:22:39.513 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:22:39.513 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:22:39.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:39.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:39.532 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:22:39.532 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:22:39.554 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:22:39.554 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:22:39.627 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:39.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:39.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:39.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:39.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:39.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:39.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:39.877 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:39.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:39.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:39.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:40.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:40.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:40.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:40.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:40.146 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:39 smithi027 ceph-mon[93921]: pgmap v801: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/248424 objects degraded (12.117%); 15479/248424 objects misplaced (6.231%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:22:40.146 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:39 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1854086580' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:40.147 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:39 smithi027 ceph-mon[98970]: pgmap v801: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/248424 objects degraded (12.117%); 15479/248424 objects misplaced (6.231%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:22:40.147 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:39 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1854086580' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:40.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:40.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:39 smithi171 ceph-mon[90031]: pgmap v801: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/248424 objects degraded (12.117%); 15479/248424 objects misplaced (6.231%); 473 KiB/s, 7 objects/s recovering 2024-08-29T15:22:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:39 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1854086580' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:40.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:40.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:40.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:22:40.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:40.404 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:40.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:22:40.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:40.530 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:40.556 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:40.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:40.658 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:40.760 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:40.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:40.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:40.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:40.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:40.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:41.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:41.036 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:41.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:41.160 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:41.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:41.262 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:41.285 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:41.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:41.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:41.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:41.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:41.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:41.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:41.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:41.659 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:41.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:41.761 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:41.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:41.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:41.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:41.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:42.010 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:41 smithi171 ceph-mon[90031]: pgmap v802: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/247995 objects degraded (12.138%); 15479/247995 objects misplaced (6.242%); 230 KiB/s, 3 objects/s recovering 2024-08-29T15:22:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:41 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/247995 objects degraded (12.138%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:42.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:42.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:42.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:42.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:42.262 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:42.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:42.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:41 smithi027 ceph-mon[98970]: pgmap v802: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/247995 objects degraded (12.138%); 15479/247995 objects misplaced (6.242%); 230 KiB/s, 3 objects/s recovering 2024-08-29T15:22:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:41 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/247995 objects degraded (12.138%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:41 smithi027 ceph-mon[93921]: pgmap v802: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 30102/247995 objects degraded (12.138%); 15479/247995 objects misplaced (6.242%); 230 KiB/s, 3 objects/s recovering 2024-08-29T15:22:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:41 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/247995 objects degraded (12.138%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:42.319 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:22:42.319 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:22:42.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:42.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.410 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:42.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:42.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:42.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:42.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:42.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:42.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:42.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:42.787 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:42.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:42.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:42.912 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:42.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:42.937 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:42.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:22:43.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:43.038 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:43.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:43.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:43.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:43.182 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:43.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:43.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:43.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:43.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:43.414 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:43.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:43.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:43.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:43.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:43.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:22:43.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:43.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:43.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:43.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:43.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:43.898 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:43.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:43.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:43.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:44.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:44.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:44.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:44.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:44.175 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:43 smithi171 ceph-mon[90031]: pgmap v803: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/246423 objects degraded (12.216%); 15349/246423 objects misplaced (6.229%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:44.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:43 smithi027 ceph-mon[98970]: pgmap v803: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/246423 objects degraded (12.216%); 15349/246423 objects misplaced (6.229%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:44.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:44.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:44.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:43 smithi027 ceph-mon[93921]: pgmap v803: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/246423 objects degraded (12.216%); 15349/246423 objects misplaced (6.229%); 493 KiB/s, 7 objects/s recovering 2024-08-29T15:22:44.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:44.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:44.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:44.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:44.400 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:44.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:44.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:44.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:44.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.671 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:44.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:44.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:44.797 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:44.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:44.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:44.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:45.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:45.044 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:22:45.044 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:22:45.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:45.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:45.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:45.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:45.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:45.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:45.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:45.271 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:45.271 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:45.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:45.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:45.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:45.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:45.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:45.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:45.517 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:45.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:45.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:45.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:45.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:45.673 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:45.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:45.799 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:45.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:22:45.901 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:45.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:45.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:45.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:45.957 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:45.958 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:22:46.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:46.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:46.133 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:46.146 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:22:46.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:46.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:46.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[98970]: pgmap v804: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/246147 objects degraded (12.229%); 15349/246147 objects misplaced (6.236%); 460 KiB/s, 7 objects/s recovering 2024-08-29T15:22:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:22:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:22:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/752641597' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[93921]: pgmap v804: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/246147 objects degraded (12.229%); 15349/246147 objects misplaced (6.236%); 460 KiB/s, 7 objects/s recovering 2024-08-29T15:22:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:22:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:22:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/752641597' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:46.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:46.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:46.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:46.445 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:46 smithi171 ceph-mon[90031]: pgmap v804: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/246147 objects degraded (12.229%); 15349/246147 objects misplaced (6.236%); 460 KiB/s, 7 objects/s recovering 2024-08-29T15:22:46.445 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:22:46.446 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:22:46.446 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:22:46.446 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/752641597' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:22:46.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:46.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:46.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:46.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:46.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:46.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:46.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:46.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:46.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:46.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:46.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:46.931 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":804,"stamp":"2024-08-29T15:22:44.577862+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5380052989,"num_objects":82049,"num_object_clones":0,"num_object_copies":246147,"num_objects_missing_on_primary":6151,"num_objects_missing":6151,"num_objects_degraded":30102,"num_objects_misplaced":15349,"num_objects_unfound":0,"num_objects_dirty":82049,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":169952,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15313,"num_bytes_recovered":882379981,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124788,"ondisk_log_size":124788,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17550984,"kb_used_data":15734984,"kb_used_omap":43,"kb_used_meta":1815828,"kb_avail":732410232,"statfs":{"total":767960285184,"available":749988077568,"internally_reserved":0,"allocated":16112623616,"data_stored":16101264590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1859408364},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,1,0,0,4,2,9],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-87818240,"num_objects":-1340,"num_object_clones":0,"num_object_copies":-4020,"num_objects_missing_on_primary":-80,"num_objects_missing":-80,"num_objects_degraded":0,"num_objects_misplaced":-242,"num_objects_unfound":0,"num_objects_dirty":-1340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1340,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":5046272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.705031"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":390,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.950459+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:22:33.950459+0000","last_peered":"2024-08-29T15:22:33.950459+0000","last_clean":"2024-08-29T15:22:33.950459+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:22:33.950459+0000","last_undegraded":"2024-08-29T15:22:33.950459+0000","last_fullsized":"2024-08-29T15:22:33.950459+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":30,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:22.746837+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:22:22.746837+0000","last_peered":"2024-08-29T15:22:22.746837+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:22:22.746837+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:22:22.746837+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":120,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:26.164128+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:22:26.164128+0000","last_peered":"2024-08-29T15:22:26.164128+0000","last_clean":"2024-08-29T15:22:26.164128+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:22:26.164128+0000","last_undegraded":"2024-08-29T15:22:26.164128+0000","last_fullsized":"2024-08-29T15:22:26.164128+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":356,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874888+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:22:33.874888+0000","last_peered":"2024-08-29T15:22:33.874888+0000","last_clean":"2024-08-29T15:22:33.874888+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:22:33.874888+0000","last_undegraded":"2024-08-29T15:22:33.874888+0000","last_fullsized":"2024-08-29T15:22:33.874888+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":356,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874662+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:22:33.874662+0000","last_peered":"2024-08-29T15:22:33.874662+0000","last_clean":"2024-08-29T15:22:33.874662+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:22:33.874662+0000","last_undegraded":"2024-08-29T15:22:33.874662+0000","last_fullsized":"2024-08-29T15:22:33.874662+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746529+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:22:22.746529+0000","last_peered":"2024-08-29T15:22:22.746529+0000","last_clean":"2024-08-29T15:22:22.746529+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:22:22.746529+0000","last_undegraded":"2024-08-29T15:22:22.746529+0000","last_fullsized":"2024-08-29T15:22:22.746529+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":167,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.945415+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:22:33.945415+0000","last_peered":"2024-08-29T15:22:33.945415+0000","last_clean":"2024-08-29T15:22:33.945415+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:22:33.945415+0000","last_undegraded":"2024-08-29T15:22:33.945415+0000","last_fullsized":"2024-08-29T15:22:33.945415+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":121,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.773952+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:22:22.773952+0000","last_peered":"2024-08-29T15:22:22.773952+0000","last_clean":"2024-08-29T15:22:22.773952+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:22:22.773952+0000","last_undegraded":"2024-08-29T15:22:22.773952+0000","last_fullsized":"2024-08-29T15:22:22.773952+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"253'10706","reported_seq":18737,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:40.959778+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:22:40.959778+0000","last_peered":"2024-08-29T15:22:40.959778+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:22:40.959778+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'700","ondisk_log_start":"175'700","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341180416,"num_objects":5206,"num_object_clones":0,"num_object_copies":15618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10706,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2281}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":29,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:22.746940+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:22:22.746940+0000","last_peered":"2024-08-29T15:22:22.746940+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:22:22.746940+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:22:22.746940+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":138,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744601+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:22:22.744601+0000","last_peered":"2024-08-29T15:22:22.744601+0000","last_clean":"2024-08-29T15:22:22.744601+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:22:22.744601+0000","last_undegraded":"2024-08-29T15:22:22.744601+0000","last_fullsized":"2024-08-29T15:22:22.744601+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"253'10443","reported_seq":10627,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:41.470596+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:22:41.470596+0000","last_peered":"2024-08-29T15:22:41.470596+0000","last_clean":"2024-08-29T15:22:41.470596+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:22:41.470596+0000","last_undegraded":"2024-08-29T15:22:41.470596+0000","last_fullsized":"2024-08-29T15:22:41.470596+0000","mapping_epoch":168,"log_start":"179'3728","ondisk_log_start":"179'3728","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334692352,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10443,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":29,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:22.750135+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:22:22.750135+0000","last_peered":"2024-08-29T15:22:22.750135+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:22:22.750135+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:22:22.750135+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746435+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:22:22.746435+0000","last_peered":"2024-08-29T15:22:22.746435+0000","last_clean":"2024-08-29T15:22:22.746435+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:22:22.746435+0000","last_undegraded":"2024-08-29T15:22:22.746435+0000","last_fullsized":"2024-08-29T15:22:22.746435+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"253'10590","reported_seq":10744,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:41.589948+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:22:41.589948+0000","last_peered":"2024-08-29T15:22:41.589948+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:22:41.589948+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"179'3874","ondisk_log_start":"179'3874","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10590,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2645}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":172,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875175+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:22:33.875175+0000","last_peered":"2024-08-29T15:22:33.875175+0000","last_clean":"2024-08-29T15:22:33.875175+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:22:33.875175+0000","last_undegraded":"2024-08-29T15:22:33.875175+0000","last_fullsized":"2024-08-29T15:22:33.875175+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":76,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.778130+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:22:22.778130+0000","last_peered":"2024-08-29T15:22:22.778130+0000","last_clean":"2024-08-29T15:22:22.778130+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:22:22.778130+0000","last_undegraded":"2024-08-29T15:22:22.778130+0000","last_fullsized":"2024-08-29T15:22:22.778130+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"253'10736","reported_seq":16140,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:43.860987+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:22:43.860987+0000","last_peered":"2024-08-29T15:22:43.860987+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:22:43.860987+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"177'2562","ondisk_log_start":"177'2562","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8174,"log_dups_size":0,"ondisk_log_size":8174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330432512,"num_objects":5042,"num_object_clones":0,"num_object_copies":15126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10736,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":5042}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":138,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.745158+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:22:22.745158+0000","last_peered":"2024-08-29T15:22:22.745158+0000","last_clean":"2024-08-29T15:22:22.745158+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:22:22.745158+0000","last_undegraded":"2024-08-29T15:22:22.745158+0000","last_fullsized":"2024-08-29T15:22:22.745158+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":370,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874372+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:22:33.874372+0000","last_peered":"2024-08-29T15:22:33.874372+0000","last_clean":"2024-08-29T15:22:33.874372+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:22:33.874372+0000","last_undegraded":"2024-08-29T15:22:33.874372+0000","last_fullsized":"2024-08-29T15:22:33.874372+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"253'10582","reported_seq":10721,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:39.739154+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:22:39.739154+0000","last_peered":"2024-08-29T15:22:39.739154+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:22:39.739154+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:22:39.739154+0000","mapping_epoch":234,"log_start":"177'2633","ondisk_log_start":"177'2633","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7949,"log_dups_size":0,"ondisk_log_size":7949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342884352,"num_objects":5232,"num_object_clones":0,"num_object_copies":15696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10582,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2755},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":365,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.950208+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:22:33.950208+0000","last_peered":"2024-08-29T15:22:33.950208+0000","last_clean":"2024-08-29T15:22:33.950208+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:22:33.950208+0000","last_undegraded":"2024-08-29T15:22:33.950208+0000","last_fullsized":"2024-08-29T15:22:33.950208+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746679+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:22:22.746679+0000","last_peered":"2024-08-29T15:22:22.746679+0000","last_clean":"2024-08-29T15:22:22.746679+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:22:22.746679+0000","last_undegraded":"2024-08-29T15:22:22.746679+0000","last_fullsized":"2024-08-29T15:22:22.746679+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"253'10621","reported_seq":10821,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:41.567976+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:22:41.567976+0000","last_peered":"2024-08-29T15:22:41.567976+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:22:41.567976+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:22:41.567976+0000","mapping_epoch":234,"log_start":"179'3868","ondisk_log_start":"179'3868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342163456,"num_objects":5221,"num_object_clones":0,"num_object_copies":15663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10621,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2756},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":94,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.778083+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:22:22.778083+0000","last_peered":"2024-08-29T15:22:22.778083+0000","last_clean":"2024-08-29T15:22:22.778083+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:22:22.778083+0000","last_undegraded":"2024-08-29T15:22:22.778083+0000","last_fullsized":"2024-08-29T15:22:22.778083+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.747183+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:22:22.747183+0000","last_peered":"2024-08-29T15:22:22.747183+0000","last_clean":"2024-08-29T15:22:22.747183+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:22:22.747183+0000","last_undegraded":"2024-08-29T15:22:22.747183+0000","last_fullsized":"2024-08-29T15:22:22.747183+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"253'10476","reported_seq":10861,"reported_epoch":253,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:22:41.601725+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:22:41.601725+0000","last_peered":"2024-08-29T15:22:41.601725+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:22:41.601725+0000","last_undegraded":"2024-08-29T15:22:41.601725+0000","last_fullsized":"2024-08-29T15:22:41.601725+0000","mapping_epoch":202,"log_start":"179'3767","ondisk_log_start":"179'3767","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339607552,"num_objects":5182,"num_object_clones":0,"num_object_copies":15546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5182,"num_objects_unfound":0,"num_objects_dirty":5182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10476,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5182}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746459+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:22:22.746459+0000","last_peered":"2024-08-29T15:22:22.746459+0000","last_clean":"2024-08-29T15:22:22.746459+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:22:22.746459+0000","last_undegraded":"2024-08-29T15:22:22.746459+0000","last_fullsized":"2024-08-29T15:22:22.746459+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":237,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.876409+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:22:33.876409+0000","last_peered":"2024-08-29T15:22:33.876409+0000","last_clean":"2024-08-29T15:22:33.876409+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:22:33.876409+0000","last_undegraded":"2024-08-29T15:22:33.876409+0000","last_fullsized":"2024-08-29T15:22:33.876409+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"253'10581","reported_seq":10714,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:40.968866+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:22:40.968866+0000","last_peered":"2024-08-29T15:22:40.968866+0000","last_clean":"2024-08-29T15:22:40.968866+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:22:40.968866+0000","last_undegraded":"2024-08-29T15:22:40.968866+0000","last_fullsized":"2024-08-29T15:22:40.968866+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9303,"log_dups_size":0,"ondisk_log_size":9303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339345432,"num_objects":5179,"num_object_clones":0,"num_object_copies":15537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5179,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10581,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":368,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.949944+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:22:33.949944+0000","last_peered":"2024-08-29T15:22:33.949944+0000","last_clean":"2024-08-29T15:22:33.949944+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:22:33.949944+0000","last_undegraded":"2024-08-29T15:22:33.949944+0000","last_fullsized":"2024-08-29T15:22:33.949944+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":360,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.945493+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:22:33.945493+0000","last_peered":"2024-08-29T15:22:33.945493+0000","last_clean":"2024-08-29T15:22:33.945493+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:22:33.945493+0000","last_undegraded":"2024-08-29T15:22:33.945493+0000","last_fullsized":"2024-08-29T15:22:33.945493+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"253'10250","reported_seq":10431,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:40.810716+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:22:40.810716+0000","last_peered":"2024-08-29T15:22:40.810716+0000","last_clean":"2024-08-29T15:22:40.810716+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:22:40.810716+0000","last_undegraded":"2024-08-29T15:22:40.810716+0000","last_fullsized":"2024-08-29T15:22:40.810716+0000","mapping_epoch":201,"log_start":"174'200","ondisk_log_start":"174'200","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10050,"log_dups_size":0,"ondisk_log_size":10050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324927488,"num_objects":4958,"num_object_clones":0,"num_object_copies":14874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10250,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":459,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874205+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:22:33.874205+0000","last_peered":"2024-08-29T15:22:33.874205+0000","last_clean":"2024-08-29T15:22:33.874205+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:22:33.874205+0000","last_undegraded":"2024-08-29T15:22:33.874205+0000","last_fullsized":"2024-08-29T15:22:33.874205+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":722,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744733+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:22:22.744733+0000","last_peered":"2024-08-29T15:22:22.744733+0000","last_clean":"2024-08-29T15:22:22.744733+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:22:22.744733+0000","last_undegraded":"2024-08-29T15:22:22.744733+0000","last_fullsized":"2024-08-29T15:22:22.744733+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":230,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744766+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:22:22.744766+0000","last_peered":"2024-08-29T15:22:22.744766+0000","last_clean":"2024-08-29T15:22:22.744766+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:22:22.744766+0000","last_undegraded":"2024-08-29T15:22:22.744766+0000","last_fullsized":"2024-08-29T15:22:22.744766+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.774392+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:22:22.774392+0000","last_peered":"2024-08-29T15:22:22.774392+0000","last_clean":"2024-08-29T15:22:22.774392+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:22:22.774392+0000","last_undegraded":"2024-08-29T15:22:22.774392+0000","last_fullsized":"2024-08-29T15:22:22.774392+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"253'10610","reported_seq":10992,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:40.278296+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:22:40.278296+0000","last_peered":"2024-08-29T15:22:40.278296+0000","last_clean":"2024-08-29T15:22:40.278296+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:22:40.278296+0000","last_undegraded":"2024-08-29T15:22:40.278296+0000","last_fullsized":"2024-08-29T15:22:40.278296+0000","mapping_epoch":201,"log_start":"178'3553","ondisk_log_start":"178'3553","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332398592,"num_objects":5072,"num_object_clones":0,"num_object_copies":15216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10610,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":833,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.944444+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:22:33.944444+0000","last_peered":"2024-08-29T15:22:33.944444+0000","last_clean":"2024-08-29T15:22:33.944444+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:22:33.944444+0000","last_undegraded":"2024-08-29T15:22:33.944444+0000","last_fullsized":"2024-08-29T15:22:33.944444+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":371,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874506+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:22:33.874506+0000","last_peered":"2024-08-29T15:22:33.874506+0000","last_clean":"2024-08-29T15:22:33.874506+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:22:33.874506+0000","last_undegraded":"2024-08-29T15:22:33.874506+0000","last_fullsized":"2024-08-29T15:22:33.874506+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":379,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874912+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:22:33.874912+0000","last_peered":"2024-08-29T15:22:33.874912+0000","last_clean":"2024-08-29T15:22:33.874912+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:22:33.874912+0000","last_undegraded":"2024-08-29T15:22:33.874912+0000","last_fullsized":"2024-08-29T15:22:33.874912+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"253'10496","reported_seq":16301,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:41.685134+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:22:41.685134+0000","last_peered":"2024-08-29T15:22:41.685134+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:22:41.685134+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"179'3730","ondisk_log_start":"179'3730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6766,"log_dups_size":0,"ondisk_log_size":6766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340393984,"num_objects":5194,"num_object_clones":0,"num_object_copies":15582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10496,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":760},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":641,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.949980+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:22:33.949980+0000","last_peered":"2024-08-29T15:22:33.949980+0000","last_clean":"2024-08-29T15:22:33.949980+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:22:33.949980+0000","last_undegraded":"2024-08-29T15:22:33.949980+0000","last_fullsized":"2024-08-29T15:22:33.949980+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.773841+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:22:22.773841+0000","last_peered":"2024-08-29T15:22:22.773841+0000","last_clean":"2024-08-29T15:22:22.773841+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:22:22.773841+0000","last_undegraded":"2024-08-29T15:22:22.773841+0000","last_fullsized":"2024-08-29T15:22:22.773841+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":405,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.945281+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:22:33.945281+0000","last_peered":"2024-08-29T15:22:33.945281+0000","last_clean":"2024-08-29T15:22:33.945281+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:22:33.945281+0000","last_undegraded":"2024-08-29T15:22:33.945281+0000","last_fullsized":"2024-08-29T15:22:33.945281+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"253'10585","reported_seq":10779,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:41.866163+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:22:41.866163+0000","last_peered":"2024-08-29T15:22:41.866163+0000","last_clean":"2024-08-29T15:22:41.866163+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:22:41.866163+0000","last_undegraded":"2024-08-29T15:22:41.866163+0000","last_fullsized":"2024-08-29T15:22:41.866163+0000","mapping_epoch":234,"log_start":"179'3897","ondisk_log_start":"179'3897","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336003072,"num_objects":5127,"num_object_clones":0,"num_object_copies":15381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10585,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":190,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744638+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:22:22.744638+0000","last_peered":"2024-08-29T15:22:22.744638+0000","last_clean":"2024-08-29T15:22:22.744638+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:22:22.744638+0000","last_undegraded":"2024-08-29T15:22:22.744638+0000","last_fullsized":"2024-08-29T15:22:22.744638+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746735+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:22:22.746735+0000","last_peered":"2024-08-29T15:22:22.746735+0000","last_clean":"2024-08-29T15:22:22.746735+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:22:22.746735+0000","last_undegraded":"2024-08-29T15:22:22.746735+0000","last_fullsized":"2024-08-29T15:22:22.746735+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":97,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.745151+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:22:22.745151+0000","last_peered":"2024-08-29T15:22:22.745151+0000","last_clean":"2024-08-29T15:22:22.745151+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:22:22.745151+0000","last_undegraded":"2024-08-29T15:22:22.745151+0000","last_fullsized":"2024-08-29T15:22:22.745151+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"253'10646","reported_seq":10838,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:43.771038+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:22:43.771038+0000","last_peered":"2024-08-29T15:22:43.771038+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:22:43.771038+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"177'2442","ondisk_log_start":"177'2442","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8204,"log_dups_size":0,"ondisk_log_size":8204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339476480,"num_objects":5180,"num_object_clones":0,"num_object_copies":15540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10646,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2757}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":122,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.744828+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:22:22.744828+0000","last_peered":"2024-08-29T15:22:22.744828+0000","last_clean":"2024-08-29T15:22:22.744828+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:22:22.744828+0000","last_undegraded":"2024-08-29T15:22:22.744828+0000","last_fullsized":"2024-08-29T15:22:22.744828+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":798,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.945058+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:22:33.945058+0000","last_peered":"2024-08-29T15:22:33.945058+0000","last_clean":"2024-08-29T15:22:33.945058+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:22:33.945058+0000","last_undegraded":"2024-08-29T15:22:33.945058+0000","last_fullsized":"2024-08-29T15:22:33.945058+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":375,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.944763+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:22:33.944763+0000","last_peered":"2024-08-29T15:22:33.944763+0000","last_clean":"2024-08-29T15:22:33.944763+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:22:33.944763+0000","last_undegraded":"2024-08-29T15:22:33.944763+0000","last_fullsized":"2024-08-29T15:22:33.944763+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"253'10561","reported_seq":11309,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:40.652004+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:22:40.652004+0000","last_peered":"2024-08-29T15:22:40.652004+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:22:40.652004+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"178'3580","ondisk_log_start":"178'3580","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334692352,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":5073,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10561,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2956}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":380,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.944993+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:22:33.944993+0000","last_peered":"2024-08-29T15:22:33.944993+0000","last_clean":"2024-08-29T15:22:33.944993+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:22:33.944993+0000","last_undegraded":"2024-08-29T15:22:33.944993+0000","last_fullsized":"2024-08-29T15:22:33.944993+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":679,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:22.787413+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:22:22.787413+0000","last_peered":"2024-08-29T15:22:22.787413+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:22:22.787413+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:22:22.787413+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":371,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874293+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:22:33.874293+0000","last_peered":"2024-08-29T15:22:33.874293+0000","last_clean":"2024-08-29T15:22:33.874293+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:22:33.874293+0000","last_undegraded":"2024-08-29T15:22:33.874293+0000","last_fullsized":"2024-08-29T15:22:33.874293+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"253'10445","reported_seq":17582,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:38.732045+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:22:38.732045+0000","last_peered":"2024-08-29T15:22:38.732045+0000","last_clean":"2024-08-29T15:22:38.732045+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:22:38.732045+0000","last_undegraded":"2024-08-29T15:22:38.732045+0000","last_fullsized":"2024-08-29T15:22:38.732045+0000","mapping_epoch":234,"log_start":"177'2499","ondisk_log_start":"177'2499","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326565888,"num_objects":4983,"num_object_clones":0,"num_object_copies":14949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10445,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":365,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.951974+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:22:33.951974+0000","last_peered":"2024-08-29T15:22:33.951974+0000","last_clean":"2024-08-29T15:22:33.951974+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:22:33.951974+0000","last_undegraded":"2024-08-29T15:22:33.951974+0000","last_fullsized":"2024-08-29T15:22:33.951974+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":987,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:33.945460+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:22:33.945460+0000","last_peered":"2024-08-29T15:22:33.945460+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:22:33.945460+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":362,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.949749+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:22:33.949749+0000","last_peered":"2024-08-29T15:22:33.949749+0000","last_clean":"2024-08-29T15:22:33.949749+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:22:33.949749+0000","last_undegraded":"2024-08-29T15:22:33.949749+0000","last_fullsized":"2024-08-29T15:22:33.949749+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"253'10501","reported_seq":13809,"reported_epoch":253,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:40.641680+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:22:40.641680+0000","last_peered":"2024-08-29T15:22:40.641680+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:22:40.641680+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'2912","ondisk_log_start":"178'2912","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7589,"log_dups_size":0,"ondisk_log_size":7589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334823424,"num_objects":5109,"num_object_clones":0,"num_object_copies":15327,"num_objects_missing_on_primary":6143,"num_objects_missing":6143,"num_objects_degraded":2313,"num_objects_misplaced":5094,"num_objects_unfound":0,"num_objects_dirty":5109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10501,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1577,"num_bytes_recovered":103219200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":15},{"shards":"1,2,7","objects":2723},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":361,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874560+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:22:33.874560+0000","last_peered":"2024-08-29T15:22:33.874560+0000","last_clean":"2024-08-29T15:22:33.874560+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:22:33.874560+0000","last_undegraded":"2024-08-29T15:22:33.874560+0000","last_fullsized":"2024-08-29T15:22:33.874560+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":103,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.774442+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:22:22.774442+0000","last_peered":"2024-08-29T15:22:22.774442+0000","last_clean":"2024-08-29T15:22:22.774442+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:22:22.774442+0000","last_undegraded":"2024-08-29T15:22:22.774442+0000","last_fullsized":"2024-08-29T15:22:22.774442+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746507+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:22:22.746507+0000","last_peered":"2024-08-29T15:22:22.746507+0000","last_clean":"2024-08-29T15:22:22.746507+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:22:22.746507+0000","last_undegraded":"2024-08-29T15:22:22.746507+0000","last_fullsized":"2024-08-29T15:22:22.746507+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746978+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:22:22.746978+0000","last_peered":"2024-08-29T15:22:22.746978+0000","last_clean":"2024-08-29T15:22:22.746978+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:22:22.746978+0000","last_undegraded":"2024-08-29T15:22:22.746978+0000","last_fullsized":"2024-08-29T15:22:22.746978+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":29,"reported_epoch":252,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:22.746819+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:22:22.746819+0000","last_peered":"2024-08-29T15:22:22.746819+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:22:22.746819+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:22:22.746819+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":369,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.944609+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:22:33.944609+0000","last_peered":"2024-08-29T15:22:33.944609+0000","last_clean":"2024-08-29T15:22:33.944609+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:22:33.944609+0000","last_undegraded":"2024-08-29T15:22:33.944609+0000","last_fullsized":"2024-08-29T15:22:33.944609+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":119,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:33.950616+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:22:33.950616+0000","last_peered":"2024-08-29T15:22:33.950616+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:22:33.950616+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:22:33.950616+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:26.164165+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:22:26.164165+0000","last_peered":"2024-08-29T15:22:26.164165+0000","last_clean":"2024-08-29T15:22:26.164165+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:22:26.164165+0000","last_undegraded":"2024-08-29T15:22:26.164165+0000","last_fullsized":"2024-08-29T15:22:26.164165+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":328,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.949749+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:22:33.949749+0000","last_peered":"2024-08-29T15:22:33.949749+0000","last_clean":"2024-08-29T15:22:33.949749+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:22:33.949749+0000","last_undegraded":"2024-08-29T15:22:33.949749+0000","last_fullsized":"2024-08-29T15:22:33.949749+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":382,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875020+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:22:33.875020+0000","last_peered":"2024-08-29T15:22:33.875020+0000","last_clean":"2024-08-29T15:22:33.875020+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:22:33.875020+0000","last_undegraded":"2024-08-29T15:22:33.875020+0000","last_fullsized":"2024-08-29T15:22:33.875020+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":380,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.949723+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:22:33.949723+0000","last_peered":"2024-08-29T15:22:33.949723+0000","last_clean":"2024-08-29T15:22:33.949723+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:22:33.949723+0000","last_undegraded":"2024-08-29T15:22:33.949723+0000","last_fullsized":"2024-08-29T15:22:33.949723+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.774526+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:22:22.774526+0000","last_peered":"2024-08-29T15:22:22.774526+0000","last_clean":"2024-08-29T15:22:22.774526+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:22:22.774526+0000","last_undegraded":"2024-08-29T15:22:22.774526+0000","last_fullsized":"2024-08-29T15:22:22.774526+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874065+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:22:33.874065+0000","last_peered":"2024-08-29T15:22:33.874065+0000","last_clean":"2024-08-29T15:22:33.874065+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:22:33.874065+0000","last_undegraded":"2024-08-29T15:22:33.874065+0000","last_fullsized":"2024-08-29T15:22:33.874065+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":357,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874813+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:22:33.874813+0000","last_peered":"2024-08-29T15:22:33.874813+0000","last_clean":"2024-08-29T15:22:33.874813+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:22:33.874813+0000","last_undegraded":"2024-08-29T15:22:33.874813+0000","last_fullsized":"2024-08-29T15:22:33.874813+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":361,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.944995+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:22:33.944995+0000","last_peered":"2024-08-29T15:22:33.944995+0000","last_clean":"2024-08-29T15:22:33.944995+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:22:33.944995+0000","last_undegraded":"2024-08-29T15:22:33.944995+0000","last_fullsized":"2024-08-29T15:22:33.944995+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":365,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.950459+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:22:33.950459+0000","last_peered":"2024-08-29T15:22:33.950459+0000","last_clean":"2024-08-29T15:22:33.950459+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:22:33.950459+0000","last_undegraded":"2024-08-29T15:22:33.950459+0000","last_fullsized":"2024-08-29T15:22:33.950459+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746544+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:22:22.746544+0000","last_peered":"2024-08-29T15:22:22.746544+0000","last_clean":"2024-08-29T15:22:22.746544+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:22:22.746544+0000","last_undegraded":"2024-08-29T15:22:22.746544+0000","last_fullsized":"2024-08-29T15:22:22.746544+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":385,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.945117+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:22:33.945117+0000","last_peered":"2024-08-29T15:22:33.945117+0000","last_clean":"2024-08-29T15:22:33.945117+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:22:33.945117+0000","last_undegraded":"2024-08-29T15:22:33.945117+0000","last_fullsized":"2024-08-29T15:22:33.945117+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":349,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874197+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:22:33.874197+0000","last_peered":"2024-08-29T15:22:33.874197+0000","last_clean":"2024-08-29T15:22:33.874197+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:22:33.874197+0000","last_undegraded":"2024-08-29T15:22:33.874197+0000","last_fullsized":"2024-08-29T15:22:33.874197+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746642+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:22:22.746642+0000","last_peered":"2024-08-29T15:22:22.746642+0000","last_clean":"2024-08-29T15:22:22.746642+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:22:22.746642+0000","last_undegraded":"2024-08-29T15:22:22.746642+0000","last_fullsized":"2024-08-29T15:22:22.746642+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":144,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:33.875973+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:22:33.875973+0000","last_peered":"2024-08-29T15:22:33.875973+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:22:33.875973+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:22:33.875973+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746598+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:22:22.746598+0000","last_peered":"2024-08-29T15:22:22.746598+0000","last_clean":"2024-08-29T15:22:22.746598+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:22:22.746598+0000","last_undegraded":"2024-08-29T15:22:22.746598+0000","last_fullsized":"2024-08-29T15:22:22.746598+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":150,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.876360+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:22:33.876360+0000","last_peered":"2024-08-29T15:22:33.876360+0000","last_clean":"2024-08-29T15:22:33.876360+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:22:33.876360+0000","last_undegraded":"2024-08-29T15:22:33.876360+0000","last_fullsized":"2024-08-29T15:22:33.876360+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":396,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.950058+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:22:33.950058+0000","last_peered":"2024-08-29T15:22:33.950058+0000","last_clean":"2024-08-29T15:22:33.950058+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:22:33.950058+0000","last_undegraded":"2024-08-29T15:22:33.950058+0000","last_fullsized":"2024-08-29T15:22:33.950058+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":397,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.951020+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:22:33.951020+0000","last_peered":"2024-08-29T15:22:33.951020+0000","last_clean":"2024-08-29T15:22:33.951020+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:22:33.951020+0000","last_undegraded":"2024-08-29T15:22:33.951020+0000","last_fullsized":"2024-08-29T15:22:33.951020+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":252,"state":"active+clean","last_fresh":"2024-08-29T15:22:22.746334+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:22:22.746334+0000","last_peered":"2024-08-29T15:22:22.746334+0000","last_clean":"2024-08-29T15:22:22.746334+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:22:22.746334+0000","last_undegraded":"2024-08-29T15:22:22.746334+0000","last_fullsized":"2024-08-29T15:22:22.746334+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":405,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874068+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:22:33.874068+0000","last_peered":"2024-08-29T15:22:33.874068+0000","last_clean":"2024-08-29T15:22:33.874068+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:22:33.874068+0000","last_undegraded":"2024-08-29T15:22:33.874068+0000","last_fullsized":"2024-08-29T15:22:33.874068+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5375262744,"num_objects":82021,"num_object_clones":0,"num_object_copies":246063,"num_objects_missing_on_primary":6143,"num_objects_missing":6143,"num_objects_degraded":30073,"num_objects_misplaced":15349,"num_objects_unfound":0,"num_objects_dirty":82021,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168829,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15204,"num_bytes_recovered":876412928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16060592128,"data_stored":16060579912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123606,"ondisk_log_size":123606,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234336,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2954792,"kb_used_data":2644768,"kb_used_omap":1,"kb_used_meta":310014,"kb_avail":90790360,"statfs":{"total":95995035648,"available":92969328640,"internally_reserved":0,"allocated":2708242432,"data_stored":2706810907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":317454795},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495968,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":363876,"kb_used_data":206428,"kb_used_omap":1,"kb_used_meta":157438,"kb_avail":93381276,"statfs":{"total":95995035648,"available":95622426624,"internally_reserved":0,"allocated":211382272,"data_stored":209954547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":161216967},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,1,0,0,4,2,9],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426554,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702040,"kb_used_data":1642832,"kb_used_omap":8,"kb_used_meta":59191,"kb_avail":92043112,"statfs":{"total":95995035648,"available":94252146688,"internally_reserved":0,"allocated":1682259968,"data_stored":1680839709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":60612490},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019246,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1965192,"kb_used_data":1641216,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":91779960,"statfs":{"total":95995035648,"available":93982679040,"internally_reserved":0,"allocated":1680605184,"data_stored":1679200708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435431,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3556628,"kb_used_data":3276492,"kb_used_omap":4,"kb_used_meta":280123,"kb_avail":90188524,"statfs":{"total":95995035648,"available":92353048576,"internally_reserved":0,"allocated":3355127808,"data_stored":3353711489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":286846217},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903283,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3042136,"kb_used_data":2920144,"kb_used_omap":10,"kb_used_meta":121973,"kb_avail":90703016,"statfs":{"total":95995035648,"available":92879888384,"internally_reserved":0,"allocated":2990227456,"data_stored":2988811073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":124900880},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762484,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2334100,"kb_used_data":2077452,"kb_used_omap":7,"kb_used_meta":256568,"kb_avail":91411052,"statfs":{"total":95995035648,"available":93604917248,"internally_reserved":0,"allocated":2127310848,"data_stored":2125893289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":262725986},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926011,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1632220,"kb_used_data":1325652,"kb_used_omap":7,"kb_used_meta":306552,"kb_avail":92112932,"statfs":{"total":95995035648,"available":94323642368,"internally_reserved":0,"allocated":1357467648,"data_stored":1356042868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":313909391},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1346965504,"data_stored":1346961432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2118258688,"data_stored":2118254616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2985361408,"data_stored":2985361408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3350790144,"data_stored":3350790144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1677197312,"data_stored":1677197312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1677660160,"data_stored":1677656088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":207355904,"data_stored":207355904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2697003008,"data_stored":2697003008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:22:46.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:22:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:46.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:46.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:47.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:47.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:47.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:47.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:47.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:47.301 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:47.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[93921]: from='client.35640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:47.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:47.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/752641597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:47.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[93921]: osdmap e254: 8 total, 8 up, 8 in 2024-08-29T15:22:47.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/246147 objects degraded (12.229%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[98970]: from='client.35640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/752641597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[98970]: osdmap e254: 8 total, 8 up, 8 in 2024-08-29T15:22:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/246147 objects degraded (12.229%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:47.324 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:47.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:47.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:47.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:47.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:47 smithi171 ceph-mon[90031]: from='client.35640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:47.450 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:47 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:47.450 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/752641597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:22:47.450 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:47 smithi171 ceph-mon[90031]: osdmap e254: 8 total, 8 up, 8 in 2024-08-29T15:22:47.450 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/246147 objects degraded (12.229%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:47.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:47.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:47.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:47.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:47.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:47.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:47.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:47.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:47.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:47.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:47.939 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:22:47.939 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:22:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:47.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:47.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:48.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:48.075 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:48.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:48.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:48.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:48.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:48.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.325 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:48.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:48.424 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:48.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:48.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:48 smithi171 ceph-mon[90031]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/245064 objects degraded (12.283%); 15258/245064 objects misplaced (6.226%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:48.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:48.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:48 smithi027 ceph-mon[98970]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/245064 objects degraded (12.283%); 15258/245064 objects misplaced (6.226%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:48.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:48 smithi027 ceph-mon[93921]: pgmap v806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/245064 objects degraded (12.283%); 15258/245064 objects misplaced (6.226%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:48.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:48.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:48.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:22:48.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:48.702 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:48.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:48.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:48.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:48.860 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:48.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:48.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:48.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:48.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:49.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:49.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:22:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:22:49] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.51.0" 2024-08-29T15:22:49.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:49.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:49.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:49.202 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:49.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:49.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:49.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:49.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:49.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:49.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:49.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:49.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:49.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:49.637 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:22:49.637 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:22:49.653 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:22:49.653 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:22:49.669 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:22:49.669 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:22:49.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:49.686 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:22:49.686 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:22:49.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:49.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:49.705 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:22:49.705 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:22:49.721 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:22:49.721 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:22:49.739 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:22:49.739 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:22:49.757 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:22:49.757 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:22:49.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:49.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:49.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:22:49.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:49.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:49.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:50.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:50.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:50.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:50.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:50.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:50 smithi171 ceph-mon[90031]: pgmap v807: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/244188 objects degraded (12.327%); 15211/244188 objects misplaced (6.229%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2038525817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:50.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:50.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:50.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:50.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:50.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:50.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:50.548 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:50 smithi027 ceph-mon[98970]: pgmap v807: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/244188 objects degraded (12.327%); 15211/244188 objects misplaced (6.229%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:50.548 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2038525817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:50.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:50 smithi027 ceph-mon[93921]: pgmap v807: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/244188 objects degraded (12.327%); 15211/244188 objects misplaced (6.229%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:50.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2038525817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:50.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:50.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:50.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:50.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:50.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:50.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:50.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:50.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:50.950 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:50.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:50.959 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:22:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:22:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:22:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:22:50.959 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:22:51.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:51.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:51.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:51.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:51.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:51.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:51.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:51.324 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:51.396 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:51.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:51.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:51.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:51.572 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:51.613 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:51.613 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/243657 objects degraded (12.354%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1153,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":254,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":81219,"data_bytes":5325658109,"bytes_used":17796268032,"bytes_avail":750164017152,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":243657,"degraded_ratio":0.12354252083872,"misplaced_objects":15211,"misplaced_total":243657,"misplaced_ratio":0.062427921217120791,"recovering_objects_per_sec":8,"recovering_bytes_per_sec":550334,"recovering_keys_per_sec":0,"num_objects_recovered":84,"num_bytes_recovered":5505024,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":144},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:22:51.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:51.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:51.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:51.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:51.822 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:51.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:51.830 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:22:51.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:51.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:51.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:51.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:52.038 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:52.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:52.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:52.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:52.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:52.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:52.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:52 smithi027 ceph-mon[98970]: pgmap v808: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 30102/243657 objects degraded (12.354%); 15211/243657 objects misplaced (6.243%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1002225978' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/243657 objects degraded (12.354%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:52 smithi027 ceph-mon[93921]: pgmap v808: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 30102/243657 objects degraded (12.354%); 15211/243657 objects misplaced (6.243%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1002225978' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/243657 objects degraded (12.354%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:52.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:52.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:52.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:52.453 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:52 smithi171 ceph-mon[90031]: pgmap v808: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 30102/243657 objects degraded (12.354%); 15211/243657 objects misplaced (6.243%); 537 KiB/s, 8 objects/s recovering 2024-08-29T15:22:52.454 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1002225978' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:22:52.454 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/243657 objects degraded (12.354%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:52.477 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:22:52.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:22:52.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:52.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:52.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:52.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:52.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:52.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:52.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:52.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:52.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:52.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:22:52.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:52.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:52.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:52.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:53.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:53.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:53.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:53.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:53.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:53.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:53.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:53.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:53.470 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:53.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:53.525 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:53.538 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:53.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:53.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:53.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:53.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:53.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:53.823 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:53.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:53.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44573.0:221741 5.4 5:285b0149:::benchmark_data_smithi027_135490_object87989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:13:45.178817+0000", 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 548.78936114999999, 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.123431307, 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:53.971 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44573", 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221741 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178817+0000", 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:53.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178817+0000", 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178815+0000", 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178822+0000", 2024-08-29T15:22:53.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4949999999999996e-06 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.178837+0000", 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4715999999999999e-05 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179041+0000", 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00020451700000000001 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:53.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179118+0000", 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7063000000000004e-05 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179232+0000", 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000113742 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.179852+0000", 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00061991200000000002 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:53.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.187825+0000", 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079734139999999998 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.190682+0000", 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028564969999999999 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.302155+0000", 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.111472904 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.302207+0000", 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2240999999999997e-05 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:13:45.302248+0000", 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1214000000000002e-05 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.977 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124404 6.3 6:cbf2ce75:::benchmark_data_smithi027_169445_object124403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 355.93891106299998, 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3120749140000001, 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124404 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.978 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:53.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029265+0000", 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029301+0000", 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5939000000000003e-05 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029311+0000", 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5869999999999999e-06 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029318+0000", 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0829999999999998e-06 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.036882+0000", 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075643280000000004 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.036891+0000", 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3100000000000001e-06 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029775+0000", 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99288402499999995 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029786+0000", 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1311000000000001e-05 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.313838+0000", 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2840517499999999 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.313991+0000", 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015349299999999999 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.314109+0000", 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011786300000000001 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.320971+0000", 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0068620870000000002 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325526+0000", 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045548979999999999 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337613+0000", 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012086573999999999 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337664+0000", 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1085999999999998e-05 2024-08-29T15:22:53.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.341342+0000", 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036781280000000001 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124411 6.5 6:a9772c5a:::benchmark_data_smithi027_169445_object124410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:22:53.986 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 355.93092031700002, 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2782022199999998, 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124411 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:53.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037257+0000", 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037284+0000", 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6716e-05 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037290+0000", 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0850000000000002e-06 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:53.990 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037294+0000", 2024-08-29T15:22:53.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4610000000000001e-06 2024-08-29T15:22:53.990 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:53.990 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037613+0000", 2024-08-29T15:22:53.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031895900000000001 2024-08-29T15:22:53.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:53.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037620+0000", 2024-08-29T15:22:53.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7800000000000003e-06 2024-08-29T15:22:53.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:53.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031231+0000", 2024-08-29T15:22:53.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99361049700000004 2024-08-29T15:22:53.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.993 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:53.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031239+0000", 2024-08-29T15:22:53.993 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.473e-06 2024-08-29T15:22:53.993 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.993 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.993 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:53.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.294818+0000", 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2635784000000001 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.294989+0000", 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000171375 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.994 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:22:53.995 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.295176+0000", 2024-08-29T15:22:53.995 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000187002 2024-08-29T15:22:53.995 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.995 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.995 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:53.995 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303052+0000", 2024-08-29T15:22:53.995 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078760210000000004 2024-08-29T15:22:53.996 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.996 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.996 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:53.996 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307520+0000", 2024-08-29T15:22:53.996 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044681469999999996 2024-08-29T15:22:53.996 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.996 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.997 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:53.997 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.312052+0000", 2024-08-29T15:22:53.997 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045315149999999998 2024-08-29T15:22:53.997 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.997 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.997 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.312092+0000", 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0621000000000003e-05 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315460+0000", 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033675900000000002 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:53.998 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124424 6.5 6:a6739d5e:::benchmark_data_smithi027_169445_object124423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 355.90984458200001, 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.268536273, 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124424 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:53.999 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058331+0000", 2024-08-29T15:22:54.000 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058377+0000", 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6134000000000002e-05 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058391+0000", 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3995000000000001e-05 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.001 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058398+0000", 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2060000000000001e-06 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058448+0000", 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9456999999999998e-05 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058460+0000", 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1622e-05 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031289+0000", 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97282904800000003 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031295+0000", 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8249999999999999e-06 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.003 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303104+0000", 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.271809059 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303275+0000", 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017008999999999999 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303459+0000", 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018486800000000001 2024-08-29T15:22:54.004 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315479+0000", 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012019724000000001 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315517+0000", 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7370999999999998e-05 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.322771+0000", 2024-08-29T15:22:54.005 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072540670000000003 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.322827+0000", 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6232e-05 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.006 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326870+0000", 2024-08-29T15:22:54.007 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040427620000000001 2024-08-29T15:22:54.007 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.007 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.007 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.007 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.007 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.008 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124428 6.3 6:cbc20026:::benchmark_data_smithi027_169445_object124427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:22:54.008 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:22:54.008 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 355.90768789999998, 2024-08-29T15:22:54.008 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3095511559999999, 2024-08-29T15:22:54.008 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124428 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.009 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060489+0000", 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.010 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060581+0000", 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1643000000000003e-05 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060594+0000", 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3222e-05 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060601+0000", 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8449999999999997e-06 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060631+0000", 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0372999999999999e-05 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060637+0000", 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4740000000000002e-06 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029808+0000", 2024-08-29T15:22:54.012 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96917036400000001 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029815+0000", 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8190000000000003e-06 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321001+0000", 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.29118635 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.013 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321186+0000", 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018538300000000001 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321355+0000", 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000168238 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337515+0000", 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016160774999999999 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337526+0000", 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1053000000000001e-05 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.366213+0000", 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028686204 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.366281+0000", 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8071999999999995e-05 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370041+0000", 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037604409999999998 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124433 6.3 6:ca9f8931:::benchmark_data_smithi027_169445_object124432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:22:54.016 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 355.90116699100003, 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3129230789999999, 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124433 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.017 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067010+0000", 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.018 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067116+0000", 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106204 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067125+0000", 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.285e-06 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067130+0000", 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1660000000000002e-06 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067165+0000", 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5108000000000003e-05 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067172+0000", 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0650000000000001e-06 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029836+0000", 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96266382100000003 2024-08-29T15:22:54.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029843+0000", 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3280000000000002e-06 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325550+0000", 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2957075179999999 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325689+0000", 2024-08-29T15:22:54.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013860200000000001 2024-08-29T15:22:54.022 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.022 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.022 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:54.022 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325852+0000", 2024-08-29T15:22:54.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016336399999999999 2024-08-29T15:22:54.022 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.023 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.023 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.023 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337576+0000", 2024-08-29T15:22:54.023 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011724399 2024-08-29T15:22:54.023 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347548+0000", 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099717829999999997 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370096+0000", 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0225477 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.024 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370146+0000", 2024-08-29T15:22:54.025 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0419e-05 2024-08-29T15:22:54.025 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.025 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.025 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.025 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.379934+0000", 2024-08-29T15:22:54.025 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0097876270000000001 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124444 6.5 6:af81ef90:::benchmark_data_smithi027_169445_object124443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 355.88915653700002, 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.251856976, 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.026 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124444 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.027 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079020+0000", 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079087+0000", 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6513999999999994e-05 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079099+0000", 2024-08-29T15:22:54.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2391999999999999e-05 2024-08-29T15:22:54.029 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.029 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.029 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.029 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079104+0000", 2024-08-29T15:22:54.029 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3589999999999999e-06 2024-08-29T15:22:54.029 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.030 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.030 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.030 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079138+0000", 2024-08-29T15:22:54.030 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4041000000000002e-05 2024-08-29T15:22:54.030 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.030 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079145+0000", 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9870000000000002e-06 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031398+0000", 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95225246100000005 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031406+0000", 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7789999999999996e-06 2024-08-29T15:22:54.031 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.032 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.032 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.032 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307552+0000", 2024-08-29T15:22:54.032 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.276146349 2024-08-29T15:22:54.032 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.032 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307695+0000", 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014288899999999999 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307892+0000", 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019679800000000001 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.033 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315574+0000", 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0076828130000000001 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315619+0000", 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5049000000000002e-05 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326936+0000", 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011316794 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.034 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326989+0000", 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2293999999999998e-05 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330878+0000", 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038897950000000001 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124462 6.3 6:cf3b5247:::benchmark_data_smithi027_169445_object124461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2024-08-29T15:22:54.035 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 353.80286539799999, 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.21866085199999999, 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124462 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.036 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165309+0000", 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:22:54.037 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165450+0000", 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014043500000000001 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165476+0000", 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5859000000000001e-05 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165489+0000", 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2765999999999999e-05 2024-08-29T15:22:54.038 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165566+0000", 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7019000000000002e-05 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165580+0000", 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4005999999999999e-05 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330157+0000", 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16457785 2024-08-29T15:22:54.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330324+0000", 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016682599999999999 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330523+0000", 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019885899999999999 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.341411+0000", 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010887468000000001 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347578+0000", 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0061672610000000003 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.379991+0000", 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032413320000000002 2024-08-29T15:22:54.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.380174+0000", 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018272399999999999 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.383973+0000", 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037995310000000001 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.042 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124473 6.3 6:c5e714a2:::benchmark_data_smithi027_169445_object124472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 353.77264066999999, 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19243169900000001, 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124473 2024-08-29T15:22:54.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195531+0000", 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999933 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195597+0000", 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6513000000000005e-05 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195613+0000", 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5865000000000001e-05 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195620+0000", 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2259999999999999e-06 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195679+0000", 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8695e-05 2024-08-29T15:22:54.046 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195690+0000", 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1473e-05 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342341+0000", 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.146650369 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.047 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342504+0000", 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000162812 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342677+0000", 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000172874 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347795+0000", 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051186010000000004 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347888+0000", 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2949999999999996e-05 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.384025+0000", 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036137057 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.384081+0000", 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5881000000000003e-05 2024-08-29T15:22:54.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.387969+0000", 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038879259999999999 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPushReply(6.5 191/190 [PushReplyOp(6:a4839e33:::benchmark_data_smithi027_169445_object147:head)])", 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 352.988395309, 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13693580699999999, 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.050 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979781+0000", 2024-08-29T15:22:54.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:54.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979790+0000", 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8929999999999994e-06 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979811+0000", 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0508000000000001e-05 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979820+0000", 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2040000000000003e-06 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:01.107955+0000", 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12813488200000001 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.054 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:01.116718+0000", 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087634329999999993 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137594 6.3 6:c9cad5ba:::benchmark_data_smithi027_169445_object12167:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.26329959499998, 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13802763300000001, 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.055 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137594 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:22:54.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:22:54.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.058 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.058 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.058 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:22:54.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:54.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.059 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.059 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.059 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704881+0000", 2024-08-29T15:22:54.059 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.627e-06 2024-08-29T15:22:54.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.059 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704886+0000", 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1390000000000003e-06 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704890+0000", 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1139999999999999e-06 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711314+0000", 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0064242639999999998 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711480+0000", 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016600600000000001 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:54.061 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711632+0000", 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015182300000000001 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717197+0000", 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055650309999999998 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732501+0000", 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015303654 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.838889+0000", 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10638834899999999 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.838943+0000", 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4279e-05 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.842906+0000", 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039627280000000004 2024-08-29T15:22:54.063 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137598 6.3 6:cf32642d:::benchmark_data_smithi027_169445_object12171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.260911222, 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.139080008, 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.064 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137598 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707265+0000", 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707271+0000", 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.48e-06 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707279+0000", 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8459999999999997e-06 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707285+0000", 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7899999999999996e-06 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717221+0000", 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099359559999999993 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717372+0000", 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015091400000000001 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717520+0000", 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014827699999999999 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.729379+0000", 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011858485 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732510+0000", 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003130678 2024-08-29T15:22:54.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.842959+0000", 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.110449724 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.843014+0000", 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4672000000000001e-05 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846347+0000", 2024-08-29T15:22:54.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033327650000000001 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137609 6.3 6:c98a9743:::benchmark_data_smithi027_169445_object12182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.24697805900001, 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12867278300000001, 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.070 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137609 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:22:54.071 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721199+0000", 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.072 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721202+0000", 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0939999999999999e-06 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721209+0000", 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0800000000000002e-06 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721213+0000", 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5600000000000004e-06 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721607+0000", 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00039427400000000001 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721766+0000", 2024-08-29T15:22:54.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000158667 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721903+0000", 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013681500000000001 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.729409+0000", 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075061219999999996 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732515+0000", 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031063250000000001 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846478+0000", 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.113962623 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846534+0000", 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6375000000000001e-05 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.849873+0000", 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033383330000000002 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:138635 6.5 6:aff223a2:::benchmark_data_smithi027_169445_object13208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 334.986140369, 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14247343300000001, 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.077 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 138635 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982036+0000", 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982044+0000", 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0830000000000004e-06 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982598+0000", 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00055435700000000001 2024-08-29T15:22:54.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982609+0000", 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0635999999999999e-05 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.003985+0000", 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021376350999999998 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.004101+0000", 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011584 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083026+0000", 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078925680999999998 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083103+0000", 2024-08-29T15:22:54.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6506000000000002e-05 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083205+0000", 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000101593 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.091336+0000", 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081309610000000008 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.082 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.095604+0000", 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042681050000000003 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.120534+0000", 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024930548 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.120652+0000", 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011811099999999999 2024-08-29T15:22:54.083 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.124511+0000", 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038586829999999999 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:139269 6.5 6:a8585b14:::benchmark_data_smithi027_169445_object13842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 334.17056144700001, 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.174785774, 2024-08-29T15:22:54.084 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 139269 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.085 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797615+0000", 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.086 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797621+0000", 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9689999999999999e-06 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797637+0000", 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6127999999999999e-05 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797645+0000", 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2130000000000008e-06 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.087 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.821075+0000", 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023429189 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.821189+0000", 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011487799999999999 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.940614+0000", 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.119424749 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.088 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.940879+0000", 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00026527700000000001 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.941261+0000", 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00038113399999999999 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.089 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968451+0000", 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027190783999999999 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968461+0000", 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1959999999999997e-06 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968492+0000", 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1841999999999998e-05 2024-08-29T15:22:54.090 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968542+0000", 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9348000000000003e-05 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.972402+0000", 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038604500000000001 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.091 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:139443 6.5 6:a02b9a8b:::benchmark_data_smithi027_169445_object14016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 333.931004567, 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15171564100000001, 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.092 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 139443 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037176+0000", 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.078e-06 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037182+0000", 2024-08-29T15:22:54.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8540000000000002e-06 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037187+0000", 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8110000000000002e-06 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.061998+0000", 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024811600999999999 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.062127+0000", 2024-08-29T15:22:54.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012901000000000001 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.100824+0000", 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038696362999999998 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.100935+0000", 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011161499999999999 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.104035+0000", 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030998649999999998 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.130597+0000", 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026562129 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.152790+0000", 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022192587999999999 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.184767+0000", 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031977515999999997 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.184820+0000", 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3084999999999997e-05 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.188889+0000", 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040686769999999997 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.098 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:142989 6.3 6.e160cdf3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 284.11310533400001, 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98678860400000001, 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 142989 2024-08-29T15:22:54.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855071+0000", 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855079+0000", 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5390000000000007e-06 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855093+0000", 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3835000000000001e-05 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855102+0000", 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9740000000000008e-06 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855146+0000", 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4082000000000002e-05 2024-08-29T15:22:54.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855154+0000", 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.622e-06 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.841849+0000", 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98669542700000001 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.841861+0000", 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1803e-05 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:142999 6.5 6.8bdf2d15 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 283.808444345, 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.683494037, 2024-08-29T15:22:54.104 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 142999 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.105 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159731+0000", 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159743+0000", 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1389000000000001e-05 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159759+0000", 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6514000000000001e-05 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159769+0000", 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4409999999999999e-06 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159822+0000", 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2806999999999998e-05 2024-08-29T15:22:54.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159834+0000", 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2047e-05 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843215+0000", 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.68338162599999996 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843228+0000", 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2296999999999999e-05 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:143011 6.5 6.1a21b555 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 283.435115258, 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.310191416, 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.109 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:22:54.110 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:22:54.110 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:22:54.110 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:22:54.110 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 143011 2024-08-29T15:22:54.110 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.110 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.111 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.111 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.111 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:22:54.111 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.111 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.111 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.111 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533061+0000", 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533069+0000", 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5210000000000002e-06 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533083+0000", 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4115e-05 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533094+0000", 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0427e-05 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533134+0000", 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0201999999999999e-05 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533144+0000", 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0164e-05 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843246+0000", 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.31010232399999998 2024-08-29T15:22:54.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843254+0000", 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7239999999999992e-06 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44912.0:170085 6.c e254/202 6:30e4c63e:::benchmark_data_smithi027_169445_object44658:head v 254'10546, mlcod=254'10544)", 2024-08-29T15:22:54.115 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 3.541335417, 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0096297399999999995, 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:22:54.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426849+0000", 2024-08-29T15:22:54.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.092e-06 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426872+0000", 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3258e-05 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426879+0000", 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2429999999999999e-06 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.427777+0000", 2024-08-29T15:22:54.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00089768400000000005 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.427805+0000", 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8583000000000001e-05 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.436412+0000", 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086069579999999996 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.436472+0000", 2024-08-29T15:22:54.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9851e-05 2024-08-29T15:22:54.120 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.120 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.120 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.120 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:22:54.120 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:22:54.120 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:22:54.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:54.174 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:54.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:54.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:54.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:54.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:54.400 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:54.423 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:54.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:54.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:54.425 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:54 smithi171 ceph-mon[90031]: pgmap v809: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/242445 objects degraded (12.416%); 15104/242445 objects misplaced (6.230%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:54.526 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:54.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:54.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:54.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:54 smithi027 ceph-mon[98970]: pgmap v809: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/242445 objects degraded (12.416%); 15104/242445 objects misplaced (6.230%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:54 smithi027 ceph-mon[93921]: pgmap v809: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/242445 objects degraded (12.416%); 15104/242445 objects misplaced (6.230%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:54.654 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:54.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:54.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:54.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:54.800 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:54.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:54.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:54.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:54.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:55.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:55.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:55.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:22:55.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:55.168 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:22:55.177 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:55.279 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:55.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:55.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/157494499' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:55.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:55 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/157494499' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:55 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:55.367 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:55.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:55.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:55.490 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/157494499' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:55.490 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:55 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:22:55.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:55.551 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:55.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:55.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:55.779 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:55.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:55.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:55.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:55.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:55.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:56.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:56.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:56.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:56.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:56.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:56.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:56.304 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:56.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[93921]: pgmap v810: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/242229 objects degraded (12.427%); 15104/242229 objects misplaced (6.235%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:22:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[93921]: osdmap e255: 8 total, 8 up, 8 in 2024-08-29T15:22:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[98970]: pgmap v810: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/242229 objects degraded (12.427%); 15104/242229 objects misplaced (6.235%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:22:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:56 smithi027 ceph-mon[98970]: osdmap e255: 8 total, 8 up, 8 in 2024-08-29T15:22:56.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:56.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:56 smithi171 ceph-mon[90031]: pgmap v810: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 130 op/s; 30102/242229 objects degraded (12.427%); 15104/242229 objects misplaced (6.235%); 525 KiB/s, 8 objects/s recovering 2024-08-29T15:22:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:56 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:22:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:56 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:22:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:56 smithi171 ceph-mon[90031]: osdmap e255: 8 total, 8 up, 8 in 2024-08-29T15:22:56.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:56.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:56.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:56.681 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:56.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:56.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:56.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:56.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:56.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:56.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:56.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:57.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:57.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:57.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:57.162 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:22:57.162 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:22:57.162 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:22:57.162 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:22:57.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:57.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:57.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/242229 objects degraded (12.427%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/242229 objects degraded (12.427%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:57.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:57.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/242229 objects degraded (12.427%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:22:57.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:22:57.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:22:57.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:57.688 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:57.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:57.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:57.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:57.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:57.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:22:57.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:57.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:57.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:57.989 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:22:57.989 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:22:58.034 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:22:58.034 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:22:58.051 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:22:58.052 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:22:58.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:58.068 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:22:58.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:22:58.085 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:22:58.085 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:22:58.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:58.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:58.101 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:22:58.102 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:22:58.110 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:22:58.119 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:22:58.119 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:22:58.136 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:22:58.137 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:22:58.153 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:22:58.154 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:22:58.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:58.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:58.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:58.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:58 smithi027 ceph-mon[93921]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 30102/241395 objects degraded (12.470%); 15032/241395 objects misplaced (6.227%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:58.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2837448073' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:58.243 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:58 smithi027 ceph-mon[98970]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 30102/241395 objects degraded (12.470%); 15032/241395 objects misplaced (6.227%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:58.243 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2837448073' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:58.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:58.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:58.435 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:58.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:58.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:58.483 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:58 smithi171 ceph-mon[90031]: pgmap v812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 30102/241395 objects degraded (12.470%); 15032/241395 objects misplaced (6.227%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:22:58.483 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2837448073' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:22:58.560 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:58.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:58.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:22:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:58.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:22:58.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:22:58.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:22:58.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:58.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:58.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:59.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:22:59.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:22:59.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:59.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:22:59.220 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:22:59.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:22:59.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:22:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:59 smithi027 ceph-mon[98970]: from='client.35664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:22:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:59 smithi027 ceph-mon[93921]: from='client.35664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:22:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:22:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:22:59] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:22:59.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:59.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:59.468 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:59.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:59 smithi171 ceph-mon[90031]: from='client.35664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:22:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:22:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:22:59.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:22:59.610 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":812,"stamp":"2024-08-29T15:22:56.581353+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5276243965,"num_objects":80465,"num_object_clones":0,"num_object_copies":241395,"num_objects_missing_on_primary":6029,"num_objects_missing":6029,"num_objects_degraded":30102,"num_objects_misplaced":15032,"num_objects_unfound":0,"num_objects_dirty":80465,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":171536,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15437,"num_bytes_recovered":890506445,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124872,"ondisk_log_size":124872,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17233456,"kb_used_data":15424360,"kb_used_omap":43,"kb_used_meta":1808980,"kb_avail":732727760,"statfs":{"total":767960285184,"available":750313226240,"internally_reserved":0,"allocated":15794544640,"data_stored":15783109134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1852396012},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,0,2,1,1,1,2,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-80150528,"num_objects":-1223,"num_object_clones":0,"num_object_copies":-3669,"num_objects_missing_on_primary":-83,"num_objects_missing":-83,"num_objects_degraded":0,"num_objects_misplaced":-226,"num_objects_unfound":0,"num_objects_dirty":-1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1223,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":81,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002883"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":392,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158894+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:22:46.158894+0000","last_peered":"2024-08-29T15:22:46.158894+0000","last_clean":"2024-08-29T15:22:46.158894+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:22:46.158894+0000","last_undegraded":"2024-08-29T15:22:46.158894+0000","last_fullsized":"2024-08-29T15:22:46.158894+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":31,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:33.875004+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:22:33.875004+0000","last_peered":"2024-08-29T15:22:33.875004+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:22:33.875004+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:22:33.875004+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149343+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:22:46.149343+0000","last_peered":"2024-08-29T15:22:46.149343+0000","last_clean":"2024-08-29T15:22:46.149343+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:22:46.149343+0000","last_undegraded":"2024-08-29T15:22:46.149343+0000","last_fullsized":"2024-08-29T15:22:46.149343+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":358,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147212+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:22:46.147212+0000","last_peered":"2024-08-29T15:22:46.147212+0000","last_clean":"2024-08-29T15:22:46.147212+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:22:46.147212+0000","last_undegraded":"2024-08-29T15:22:46.147212+0000","last_fullsized":"2024-08-29T15:22:46.147212+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":358,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147034+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:22:46.147034+0000","last_peered":"2024-08-29T15:22:46.147034+0000","last_clean":"2024-08-29T15:22:46.147034+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:22:46.147034+0000","last_undegraded":"2024-08-29T15:22:46.147034+0000","last_fullsized":"2024-08-29T15:22:46.147034+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875040+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:22:33.875040+0000","last_peered":"2024-08-29T15:22:33.875040+0000","last_clean":"2024-08-29T15:22:33.875040+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:22:33.875040+0000","last_undegraded":"2024-08-29T15:22:33.875040+0000","last_fullsized":"2024-08-29T15:22:33.875040+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":171,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165723+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:22:55.165723+0000","last_peered":"2024-08-29T15:22:55.165723+0000","last_clean":"2024-08-29T15:22:55.165723+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:22:55.165723+0000","last_undegraded":"2024-08-29T15:22:55.165723+0000","last_fullsized":"2024-08-29T15:22:55.165723+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":123,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.164786+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:22:46.164786+0000","last_peered":"2024-08-29T15:22:46.164786+0000","last_clean":"2024-08-29T15:22:46.164786+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:22:46.164786+0000","last_undegraded":"2024-08-29T15:22:46.164786+0000","last_fullsized":"2024-08-29T15:22:46.164786+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"255'10826","reported_seq":18859,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:56.101517+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:22:56.101517+0000","last_peered":"2024-08-29T15:22:56.101517+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:22:56.101517+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'800","ondisk_log_start":"175'800","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316096,"num_objects":5086,"num_object_clones":0,"num_object_copies":15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10826,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2161}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":30,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:33.875192+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:22:33.875192+0000","last_peered":"2024-08-29T15:22:33.875192+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:22:33.875192+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:22:33.875192+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":140,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.148798+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:22:46.148798+0000","last_peered":"2024-08-29T15:22:46.148798+0000","last_clean":"2024-08-29T15:22:46.148798+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:22:46.148798+0000","last_undegraded":"2024-08-29T15:22:46.148798+0000","last_fullsized":"2024-08-29T15:22:46.148798+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"254'10525","reported_seq":10711,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:51.358086+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:22:51.358086+0000","last_peered":"2024-08-29T15:22:51.358086+0000","last_clean":"2024-08-29T15:22:51.358086+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:22:51.358086+0000","last_undegraded":"2024-08-29T15:22:51.358086+0000","last_fullsized":"2024-08-29T15:22:51.358086+0000","mapping_epoch":168,"log_start":"179'3828","ondisk_log_start":"179'3828","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329318400,"num_objects":5025,"num_object_clones":0,"num_object_copies":15075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10525,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":30,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:33.876203+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:22:33.876203+0000","last_peered":"2024-08-29T15:22:33.876203+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:22:33.876203+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:22:33.876203+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874821+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:22:33.874821+0000","last_peered":"2024-08-29T15:22:33.874821+0000","last_clean":"2024-08-29T15:22:33.874821+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:22:33.874821+0000","last_undegraded":"2024-08-29T15:22:33.874821+0000","last_fullsized":"2024-08-29T15:22:33.874821+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"254'10673","reported_seq":10829,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:50.427574+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:22:50.427574+0000","last_peered":"2024-08-29T15:22:50.427574+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:22:50.427574+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"179'3974","ondisk_log_start":"179'3974","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330235904,"num_objects":5039,"num_object_clones":0,"num_object_copies":15117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10673,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2562}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":174,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.150019+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:22:46.150019+0000","last_peered":"2024-08-29T15:22:46.150019+0000","last_clean":"2024-08-29T15:22:46.150019+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:22:46.150019+0000","last_undegraded":"2024-08-29T15:22:46.150019+0000","last_fullsized":"2024-08-29T15:22:46.150019+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.165691+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:22:46.165691+0000","last_peered":"2024-08-29T15:22:46.165691+0000","last_clean":"2024-08-29T15:22:46.165691+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:22:46.165691+0000","last_undegraded":"2024-08-29T15:22:46.165691+0000","last_fullsized":"2024-08-29T15:22:46.165691+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"254'10826","reported_seq":16232,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:53.817143+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:22:53.817143+0000","last_peered":"2024-08-29T15:22:53.817143+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:22:53.817143+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"177'2662","ondisk_log_start":"177'2662","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8164,"log_dups_size":0,"ondisk_log_size":8164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324534272,"num_objects":4952,"num_object_clones":0,"num_object_copies":14856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10826,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4952}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149767+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:22:46.149767+0000","last_peered":"2024-08-29T15:22:46.149767+0000","last_clean":"2024-08-29T15:22:46.149767+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:22:46.149767+0000","last_undegraded":"2024-08-29T15:22:46.149767+0000","last_fullsized":"2024-08-29T15:22:46.149767+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":372,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.148096+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:22:46.148096+0000","last_peered":"2024-08-29T15:22:46.148096+0000","last_clean":"2024-08-29T15:22:46.148096+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:22:46.148096+0000","last_undegraded":"2024-08-29T15:22:46.148096+0000","last_fullsized":"2024-08-29T15:22:46.148096+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"254'10707","reported_seq":10847,"reported_epoch":254,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:54.418409+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:22:54.418409+0000","last_peered":"2024-08-29T15:22:54.418409+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:22:54.418409+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:22:54.418409+0000","mapping_epoch":234,"log_start":"177'2733","ondisk_log_start":"177'2733","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7974,"log_dups_size":0,"ondisk_log_size":7974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334692352,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10707,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2630},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":367,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158727+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:22:46.158727+0000","last_peered":"2024-08-29T15:22:46.158727+0000","last_clean":"2024-08-29T15:22:46.158727+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:22:46.158727+0000","last_undegraded":"2024-08-29T15:22:46.158727+0000","last_fullsized":"2024-08-29T15:22:46.158727+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875049+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:22:33.875049+0000","last_peered":"2024-08-29T15:22:33.875049+0000","last_clean":"2024-08-29T15:22:33.875049+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:22:33.875049+0000","last_undegraded":"2024-08-29T15:22:33.875049+0000","last_fullsized":"2024-08-29T15:22:33.875049+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"254'10696","reported_seq":10898,"reported_epoch":254,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:51.802107+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:22:51.802107+0000","last_peered":"2024-08-29T15:22:51.802107+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:22:51.802107+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:22:51.802107+0000","mapping_epoch":234,"log_start":"179'3968","ondisk_log_start":"179'3968","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337248256,"num_objects":5146,"num_object_clones":0,"num_object_copies":15438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10696,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2681},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.165577+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:22:46.165577+0000","last_peered":"2024-08-29T15:22:46.165577+0000","last_clean":"2024-08-29T15:22:46.165577+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:22:46.165577+0000","last_undegraded":"2024-08-29T15:22:46.165577+0000","last_fullsized":"2024-08-29T15:22:46.165577+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875310+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:22:33.875310+0000","last_peered":"2024-08-29T15:22:33.875310+0000","last_clean":"2024-08-29T15:22:33.875310+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:22:33.875310+0000","last_undegraded":"2024-08-29T15:22:33.875310+0000","last_fullsized":"2024-08-29T15:22:33.875310+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"254'10550","reported_seq":10937,"reported_epoch":254,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:22:51.421692+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:22:51.421692+0000","last_peered":"2024-08-29T15:22:51.421692+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:22:51.421692+0000","last_undegraded":"2024-08-29T15:22:51.421692+0000","last_fullsized":"2024-08-29T15:22:51.421692+0000","mapping_epoch":202,"log_start":"179'3867","ondisk_log_start":"179'3867","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334757888,"num_objects":5108,"num_object_clones":0,"num_object_copies":15324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5108,"num_objects_unfound":0,"num_objects_dirty":5108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10550,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5108}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874896+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:22:33.874896+0000","last_peered":"2024-08-29T15:22:33.874896+0000","last_clean":"2024-08-29T15:22:33.874896+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:22:33.874896+0000","last_undegraded":"2024-08-29T15:22:33.874896+0000","last_fullsized":"2024-08-29T15:22:33.874896+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":238,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:39.795750+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:22:39.795750+0000","last_peered":"2024-08-29T15:22:39.795750+0000","last_clean":"2024-08-29T15:22:39.795750+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:22:39.795750+0000","last_undegraded":"2024-08-29T15:22:39.795750+0000","last_fullsized":"2024-08-29T15:22:39.795750+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"255'10685","reported_seq":10820,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.935582+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:22:55.935582+0000","last_peered":"2024-08-29T15:22:55.935582+0000","last_clean":"2024-08-29T15:22:55.935582+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:22:55.935582+0000","last_undegraded":"2024-08-29T15:22:55.935582+0000","last_fullsized":"2024-08-29T15:22:55.935582+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9407,"log_dups_size":0,"ondisk_log_size":9407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332529688,"num_objects":5075,"num_object_clones":0,"num_object_copies":15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5075,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10685,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":370,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158429+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:22:46.158429+0000","last_peered":"2024-08-29T15:22:46.158429+0000","last_clean":"2024-08-29T15:22:46.158429+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:22:46.158429+0000","last_undegraded":"2024-08-29T15:22:46.158429+0000","last_fullsized":"2024-08-29T15:22:46.158429+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":364,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165728+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:22:55.165728+0000","last_peered":"2024-08-29T15:22:55.165728+0000","last_clean":"2024-08-29T15:22:55.165728+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:22:55.165728+0000","last_undegraded":"2024-08-29T15:22:55.165728+0000","last_fullsized":"2024-08-29T15:22:55.165728+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"255'10366","reported_seq":10549,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.955142+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:22:55.955142+0000","last_peered":"2024-08-29T15:22:55.955142+0000","last_clean":"2024-08-29T15:22:55.955142+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:22:55.955142+0000","last_undegraded":"2024-08-29T15:22:55.955142+0000","last_fullsized":"2024-08-29T15:22:55.955142+0000","mapping_epoch":201,"log_start":"175'300","ondisk_log_start":"175'300","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317325312,"num_objects":4842,"num_object_clones":0,"num_object_copies":14526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10366,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":461,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147689+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:22:46.147689+0000","last_peered":"2024-08-29T15:22:46.147689+0000","last_clean":"2024-08-29T15:22:46.147689+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:22:46.147689+0000","last_undegraded":"2024-08-29T15:22:46.147689+0000","last_fullsized":"2024-08-29T15:22:46.147689+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":724,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149118+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:22:46.149118+0000","last_peered":"2024-08-29T15:22:46.149118+0000","last_clean":"2024-08-29T15:22:46.149118+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:22:46.149118+0000","last_undegraded":"2024-08-29T15:22:46.149118+0000","last_fullsized":"2024-08-29T15:22:46.149118+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":232,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149947+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:22:46.149947+0000","last_peered":"2024-08-29T15:22:46.149947+0000","last_clean":"2024-08-29T15:22:46.149947+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:22:46.149947+0000","last_undegraded":"2024-08-29T15:22:46.149947+0000","last_fullsized":"2024-08-29T15:22:46.149947+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":94,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.172241+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:22:46.172241+0000","last_peered":"2024-08-29T15:22:46.172241+0000","last_clean":"2024-08-29T15:22:46.172241+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:22:46.172241+0000","last_undegraded":"2024-08-29T15:22:46.172241+0000","last_fullsized":"2024-08-29T15:22:46.172241+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"255'10727","reported_seq":11113,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.514610+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:22:55.514610+0000","last_peered":"2024-08-29T15:22:55.514610+0000","last_clean":"2024-08-29T15:22:55.514610+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:22:55.514610+0000","last_undegraded":"2024-08-29T15:22:55.514610+0000","last_fullsized":"2024-08-29T15:22:55.514610+0000","mapping_epoch":201,"log_start":"178'3653","ondisk_log_start":"178'3653","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324730880,"num_objects":4955,"num_object_clones":0,"num_object_copies":14865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10727,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":837,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.164881+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:22:55.164881+0000","last_peered":"2024-08-29T15:22:55.164881+0000","last_clean":"2024-08-29T15:22:55.164881+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:22:55.164881+0000","last_undegraded":"2024-08-29T15:22:55.164881+0000","last_fullsized":"2024-08-29T15:22:55.164881+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":373,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.148471+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:22:46.148471+0000","last_peered":"2024-08-29T15:22:46.148471+0000","last_clean":"2024-08-29T15:22:46.148471+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:22:46.148471+0000","last_undegraded":"2024-08-29T15:22:46.148471+0000","last_fullsized":"2024-08-29T15:22:46.148471+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":381,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147691+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:22:46.147691+0000","last_peered":"2024-08-29T15:22:46.147691+0000","last_clean":"2024-08-29T15:22:46.147691+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:22:46.147691+0000","last_undegraded":"2024-08-29T15:22:46.147691+0000","last_fullsized":"2024-08-29T15:22:46.147691+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"254'10579","reported_seq":16387,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:50.641947+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:22:50.641947+0000","last_peered":"2024-08-29T15:22:50.641947+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:22:50.641947+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"179'3830","ondisk_log_start":"179'3830","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334954496,"num_objects":5111,"num_object_clones":0,"num_object_copies":15333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10579,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":677},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":644,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158595+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:22:46.158595+0000","last_peered":"2024-08-29T15:22:46.158595+0000","last_clean":"2024-08-29T15:22:46.158595+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:22:46.158595+0000","last_undegraded":"2024-08-29T15:22:46.158595+0000","last_fullsized":"2024-08-29T15:22:46.158595+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":85,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.164543+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:22:46.164543+0000","last_peered":"2024-08-29T15:22:46.164543+0000","last_clean":"2024-08-29T15:22:46.164543+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:22:46.164543+0000","last_undegraded":"2024-08-29T15:22:46.164543+0000","last_fullsized":"2024-08-29T15:22:46.164543+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":409,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165614+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:22:55.165614+0000","last_peered":"2024-08-29T15:22:55.165614+0000","last_clean":"2024-08-29T15:22:55.165614+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:22:55.165614+0000","last_undegraded":"2024-08-29T15:22:55.165614+0000","last_fullsized":"2024-08-29T15:22:55.165614+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"254'10658","reported_seq":10854,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:51.320473+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:22:51.320473+0000","last_peered":"2024-08-29T15:22:51.320473+0000","last_clean":"2024-08-29T15:22:51.320473+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:22:51.320473+0000","last_undegraded":"2024-08-29T15:22:51.320473+0000","last_fullsized":"2024-08-29T15:22:51.320473+0000","mapping_epoch":234,"log_start":"179'3897","ondisk_log_start":"179'3897","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331218944,"num_objects":5054,"num_object_clones":0,"num_object_copies":15162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10658,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":192,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149170+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:22:46.149170+0000","last_peered":"2024-08-29T15:22:46.149170+0000","last_clean":"2024-08-29T15:22:46.149170+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:22:46.149170+0000","last_undegraded":"2024-08-29T15:22:46.149170+0000","last_fullsized":"2024-08-29T15:22:46.149170+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874888+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:22:33.874888+0000","last_peered":"2024-08-29T15:22:33.874888+0000","last_clean":"2024-08-29T15:22:33.874888+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:22:33.874888+0000","last_undegraded":"2024-08-29T15:22:33.874888+0000","last_fullsized":"2024-08-29T15:22:33.874888+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":99,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149377+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:22:46.149377+0000","last_peered":"2024-08-29T15:22:46.149377+0000","last_clean":"2024-08-29T15:22:46.149377+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:22:46.149377+0000","last_undegraded":"2024-08-29T15:22:46.149377+0000","last_fullsized":"2024-08-29T15:22:46.149377+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"254'10716","reported_seq":10910,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:53.878107+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:22:53.878107+0000","last_peered":"2024-08-29T15:22:53.878107+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:22:53.878107+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"177'2542","ondisk_log_start":"177'2542","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8174,"log_dups_size":0,"ondisk_log_size":8174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334888960,"num_objects":5110,"num_object_clones":0,"num_object_copies":15330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10716,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2687}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":124,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149282+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:22:46.149282+0000","last_peered":"2024-08-29T15:22:46.149282+0000","last_clean":"2024-08-29T15:22:46.149282+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:22:46.149282+0000","last_undegraded":"2024-08-29T15:22:46.149282+0000","last_fullsized":"2024-08-29T15:22:46.149282+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":802,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165321+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:22:55.165321+0000","last_peered":"2024-08-29T15:22:55.165321+0000","last_clean":"2024-08-29T15:22:55.165321+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:22:55.165321+0000","last_undegraded":"2024-08-29T15:22:55.165321+0000","last_fullsized":"2024-08-29T15:22:55.165321+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":379,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165456+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:22:55.165456+0000","last_peered":"2024-08-29T15:22:55.165456+0000","last_clean":"2024-08-29T15:22:55.165456+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:22:55.165456+0000","last_undegraded":"2024-08-29T15:22:55.165456+0000","last_fullsized":"2024-08-29T15:22:55.165456+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"255'10682","reported_seq":11434,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:55.168640+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:22:55.168640+0000","last_peered":"2024-08-29T15:22:55.168640+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:22:55.168640+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"179'3680","ondisk_log_start":"179'3680","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326762496,"num_objects":4986,"num_object_clones":0,"num_object_copies":14958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4952,"num_objects_unfound":0,"num_objects_dirty":4986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10682,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2835}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":384,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165230+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:22:55.165230+0000","last_peered":"2024-08-29T15:22:55.165230+0000","last_clean":"2024-08-29T15:22:55.165230+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:22:55.165230+0000","last_undegraded":"2024-08-29T15:22:55.165230+0000","last_fullsized":"2024-08-29T15:22:55.165230+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":680,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:33.964039+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:22:33.964039+0000","last_peered":"2024-08-29T15:22:33.964039+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:22:33.964039+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:22:33.964039+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":373,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147987+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:22:46.147987+0000","last_peered":"2024-08-29T15:22:46.147987+0000","last_clean":"2024-08-29T15:22:46.147987+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:22:46.147987+0000","last_undegraded":"2024-08-29T15:22:46.147987+0000","last_fullsized":"2024-08-29T15:22:46.147987+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"254'10573","reported_seq":17711,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:54.503591+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:22:54.503591+0000","last_peered":"2024-08-29T15:22:54.503591+0000","last_clean":"2024-08-29T15:22:54.503591+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:22:54.503591+0000","last_undegraded":"2024-08-29T15:22:54.503591+0000","last_fullsized":"2024-08-29T15:22:54.503591+0000","mapping_epoch":234,"log_start":"177'2599","ondisk_log_start":"177'2599","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7974,"log_dups_size":0,"ondisk_log_size":7974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318177280,"num_objects":4855,"num_object_clones":0,"num_object_copies":14565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10573,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":367,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.162021+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:22:46.162021+0000","last_peered":"2024-08-29T15:22:46.162021+0000","last_clean":"2024-08-29T15:22:46.162021+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:22:46.162021+0000","last_undegraded":"2024-08-29T15:22:46.162021+0000","last_fullsized":"2024-08-29T15:22:46.162021+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":991,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:55.165718+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:22:55.165718+0000","last_peered":"2024-08-29T15:22:55.165718+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:22:55.165718+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":364,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158248+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:22:46.158248+0000","last_peered":"2024-08-29T15:22:46.158248+0000","last_clean":"2024-08-29T15:22:46.158248+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:22:46.158248+0000","last_undegraded":"2024-08-29T15:22:46.158248+0000","last_fullsized":"2024-08-29T15:22:46.158248+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"255'10624","reported_seq":14180,"reported_epoch":255,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:22:56.102908+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:22:56.102908+0000","last_peered":"2024-08-29T15:22:56.102908+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:22:56.102908+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'3112","ondisk_log_start":"178'3112","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7512,"log_dups_size":0,"ondisk_log_size":7512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326762496,"num_objects":4986,"num_object_clones":0,"num_object_copies":14958,"num_objects_missing_on_primary":6021,"num_objects_missing":6021,"num_objects_degraded":2313,"num_objects_misplaced":4972,"num_objects_unfound":0,"num_objects_dirty":4986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10624,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1701,"num_bytes_recovered":111345664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":2601},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":363,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147006+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:22:46.147006+0000","last_peered":"2024-08-29T15:22:46.147006+0000","last_clean":"2024-08-29T15:22:46.147006+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:22:46.147006+0000","last_undegraded":"2024-08-29T15:22:46.147006+0000","last_fullsized":"2024-08-29T15:22:46.147006+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":105,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.165668+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:22:46.165668+0000","last_peered":"2024-08-29T15:22:46.165668+0000","last_clean":"2024-08-29T15:22:46.165668+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:22:46.165668+0000","last_undegraded":"2024-08-29T15:22:46.165668+0000","last_fullsized":"2024-08-29T15:22:46.165668+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874966+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:22:33.874966+0000","last_peered":"2024-08-29T15:22:33.874966+0000","last_clean":"2024-08-29T15:22:33.874966+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:22:33.874966+0000","last_undegraded":"2024-08-29T15:22:33.874966+0000","last_fullsized":"2024-08-29T15:22:33.874966+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875246+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:22:33.875246+0000","last_peered":"2024-08-29T15:22:33.875246+0000","last_clean":"2024-08-29T15:22:33.875246+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:22:33.875246+0000","last_undegraded":"2024-08-29T15:22:33.875246+0000","last_fullsized":"2024-08-29T15:22:33.875246+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":30,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:33.875172+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:22:33.875172+0000","last_peered":"2024-08-29T15:22:33.875172+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:22:33.875172+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:22:33.875172+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":373,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165292+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:22:55.165292+0000","last_peered":"2024-08-29T15:22:55.165292+0000","last_clean":"2024-08-29T15:22:55.165292+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:22:55.165292+0000","last_undegraded":"2024-08-29T15:22:55.165292+0000","last_fullsized":"2024-08-29T15:22:55.165292+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":121,"reported_epoch":254,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:46.161890+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:22:46.161890+0000","last_peered":"2024-08-29T15:22:46.161890+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:22:46.161890+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:22:46.161890+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":122,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149046+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:22:46.149046+0000","last_peered":"2024-08-29T15:22:46.149046+0000","last_clean":"2024-08-29T15:22:46.149046+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:22:46.149046+0000","last_undegraded":"2024-08-29T15:22:46.149046+0000","last_fullsized":"2024-08-29T15:22:46.149046+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":330,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158379+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:22:46.158379+0000","last_peered":"2024-08-29T15:22:46.158379+0000","last_clean":"2024-08-29T15:22:46.158379+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:22:46.158379+0000","last_undegraded":"2024-08-29T15:22:46.158379+0000","last_fullsized":"2024-08-29T15:22:46.158379+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":384,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147318+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:22:46.147318+0000","last_peered":"2024-08-29T15:22:46.147318+0000","last_clean":"2024-08-29T15:22:46.147318+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:22:46.147318+0000","last_undegraded":"2024-08-29T15:22:46.147318+0000","last_fullsized":"2024-08-29T15:22:46.147318+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":382,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158193+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:22:46.158193+0000","last_peered":"2024-08-29T15:22:46.158193+0000","last_clean":"2024-08-29T15:22:46.158193+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:22:46.158193+0000","last_undegraded":"2024-08-29T15:22:46.158193+0000","last_fullsized":"2024-08-29T15:22:46.158193+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":93,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.165751+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:22:46.165751+0000","last_peered":"2024-08-29T15:22:46.165751+0000","last_clean":"2024-08-29T15:22:46.165751+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:22:46.165751+0000","last_undegraded":"2024-08-29T15:22:46.165751+0000","last_fullsized":"2024-08-29T15:22:46.165751+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":239,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147599+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:22:46.147599+0000","last_peered":"2024-08-29T15:22:46.147599+0000","last_clean":"2024-08-29T15:22:46.147599+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:22:46.147599+0000","last_undegraded":"2024-08-29T15:22:46.147599+0000","last_fullsized":"2024-08-29T15:22:46.147599+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":359,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147620+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:22:46.147620+0000","last_peered":"2024-08-29T15:22:46.147620+0000","last_clean":"2024-08-29T15:22:46.147620+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:22:46.147620+0000","last_undegraded":"2024-08-29T15:22:46.147620+0000","last_fullsized":"2024-08-29T15:22:46.147620+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":365,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.164980+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:22:55.164980+0000","last_peered":"2024-08-29T15:22:55.164980+0000","last_clean":"2024-08-29T15:22:55.164980+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:22:55.164980+0000","last_undegraded":"2024-08-29T15:22:55.164980+0000","last_fullsized":"2024-08-29T15:22:55.164980+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":367,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.161664+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:22:46.161664+0000","last_peered":"2024-08-29T15:22:46.161664+0000","last_clean":"2024-08-29T15:22:46.161664+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:22:46.161664+0000","last_undegraded":"2024-08-29T15:22:46.161664+0000","last_fullsized":"2024-08-29T15:22:46.161664+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874947+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:22:33.874947+0000","last_peered":"2024-08-29T15:22:33.874947+0000","last_clean":"2024-08-29T15:22:33.874947+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:22:33.874947+0000","last_undegraded":"2024-08-29T15:22:33.874947+0000","last_fullsized":"2024-08-29T15:22:33.874947+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":389,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.165410+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:22:55.165410+0000","last_peered":"2024-08-29T15:22:55.165410+0000","last_clean":"2024-08-29T15:22:55.165410+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:22:55.165410+0000","last_undegraded":"2024-08-29T15:22:55.165410+0000","last_fullsized":"2024-08-29T15:22:55.165410+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":351,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147926+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:22:46.147926+0000","last_peered":"2024-08-29T15:22:46.147926+0000","last_clean":"2024-08-29T15:22:46.147926+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:22:46.147926+0000","last_undegraded":"2024-08-29T15:22:46.147926+0000","last_fullsized":"2024-08-29T15:22:46.147926+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875169+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:22:33.875169+0000","last_peered":"2024-08-29T15:22:33.875169+0000","last_clean":"2024-08-29T15:22:33.875169+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:22:33.875169+0000","last_undegraded":"2024-08-29T15:22:33.875169+0000","last_fullsized":"2024-08-29T15:22:33.875169+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":146,"reported_epoch":254,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:46.149532+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:22:46.149532+0000","last_peered":"2024-08-29T15:22:46.149532+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:22:46.149532+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:22:46.149532+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.875023+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:22:33.875023+0000","last_peered":"2024-08-29T15:22:33.875023+0000","last_clean":"2024-08-29T15:22:33.875023+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:22:33.875023+0000","last_undegraded":"2024-08-29T15:22:33.875023+0000","last_fullsized":"2024-08-29T15:22:33.875023+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":152,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.149412+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:22:46.149412+0000","last_peered":"2024-08-29T15:22:46.149412+0000","last_clean":"2024-08-29T15:22:46.149412+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:22:46.149412+0000","last_undegraded":"2024-08-29T15:22:46.149412+0000","last_fullsized":"2024-08-29T15:22:46.149412+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":398,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.158643+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:22:46.158643+0000","last_peered":"2024-08-29T15:22:46.158643+0000","last_clean":"2024-08-29T15:22:46.158643+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:22:46.158643+0000","last_undegraded":"2024-08-29T15:22:46.158643+0000","last_fullsized":"2024-08-29T15:22:46.158643+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":399,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.159509+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:22:46.159509+0000","last_peered":"2024-08-29T15:22:46.159509+0000","last_clean":"2024-08-29T15:22:46.159509+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:22:46.159509+0000","last_undegraded":"2024-08-29T15:22:46.159509+0000","last_fullsized":"2024-08-29T15:22:46.159509+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":32,"reported_epoch":253,"state":"active+clean","last_fresh":"2024-08-29T15:22:33.874740+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:22:33.874740+0000","last_peered":"2024-08-29T15:22:33.874740+0000","last_clean":"2024-08-29T15:22:33.874740+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:22:33.874740+0000","last_undegraded":"2024-08-29T15:22:33.874740+0000","last_fullsized":"2024-08-29T15:22:33.874740+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":407,"reported_epoch":254,"state":"active+clean","last_fresh":"2024-08-29T15:22:46.147234+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:22:46.147234+0000","last_peered":"2024-08-29T15:22:46.147234+0000","last_clean":"2024-08-29T15:22:46.147234+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:22:46.147234+0000","last_undegraded":"2024-08-29T15:22:46.147234+0000","last_fullsized":"2024-08-29T15:22:46.147234+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5271453720,"num_objects":80437,"num_object_clones":0,"num_object_copies":241311,"num_objects_missing_on_primary":6021,"num_objects_missing":6021,"num_objects_degraded":30073,"num_objects_misplaced":15032,"num_objects_unfound":0,"num_objects_dirty":80437,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170413,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15328,"num_bytes_recovered":884539392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15747395584,"data_stored":15747383368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123690,"ondisk_log_size":123690,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234339,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2875640,"kb_used_data":2577904,"kb_used_omap":1,"kb_used_meta":297726,"kb_avail":90869512,"statfs":{"total":95995035648,"available":93050380288,"internally_reserved":0,"allocated":2639773696,"data_stored":2638334523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":304871883},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495971,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":352648,"kb_used_data":191100,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":93392504,"statfs":{"total":95995035648,"available":95633924096,"internally_reserved":0,"allocated":195686400,"data_stored":194243379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":165411271},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,1,1,1,2,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426557,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1665788,"kb_used_data":1606576,"kb_used_omap":8,"kb_used_meta":59191,"kb_avail":92079364,"statfs":{"total":95995035648,"available":94289268736,"internally_reserved":0,"allocated":1645133824,"data_stored":1643698269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":60612490},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019249,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1929048,"kb_used_data":1600976,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":91816104,"statfs":{"total":95995035648,"available":94019690496,"internally_reserved":0,"allocated":1639399424,"data_stored":1637987300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":335935942},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435433,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3510116,"kb_used_data":3225820,"kb_used_omap":4,"kb_used_meta":284283,"kb_avail":90235036,"statfs":{"total":95995035648,"available":92400676864,"internally_reserved":0,"allocated":3303239680,"data_stored":3301815713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":291106057},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903285,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2983208,"kb_used_data":2877856,"kb_used_omap":10,"kb_used_meta":105333,"kb_avail":90761944,"statfs":{"total":95995035648,"available":92940230656,"internally_reserved":0,"allocated":2946924544,"data_stored":2945500513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":107861520},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762487,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2301028,"kb_used_data":2038876,"kb_used_omap":7,"kb_used_meta":262136,"kb_avail":91444124,"statfs":{"total":95995035648,"available":93638782976,"internally_reserved":0,"allocated":2087809024,"data_stored":2086383817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":268427618},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926013,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1615980,"kb_used_data":1305252,"kb_used_omap":7,"kb_used_meta":310712,"kb_avail":92129172,"statfs":{"total":95995035648,"available":94340272128,"internally_reserved":0,"allocated":1336578048,"data_stored":1335145620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":318169231},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1327108096,"data_stored":1327104024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2079985664,"data_stored":2079981592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2942763008,"data_stored":2942763008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3299672064,"data_stored":3299672064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1635713024,"data_stored":1635713024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1640697856,"data_stored":1640693784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":191234048,"data_stored":191234048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2630221824,"data_stored":2630221824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:22:59.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:22:59.717 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:22:59.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:59.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:22:59.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:22:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:59.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:22:59.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:22:59.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:22:59.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:22:59.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:00.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:00.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:00.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:00.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:00.295 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:23:00.295 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:23:00.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:00.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:00.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:00.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:00.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:00.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:00 smithi171 ceph-mon[90031]: pgmap v813: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 30102/240717 objects degraded (12.505%); 14995/240717 objects misplaced (6.229%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:00.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:00 smithi027 ceph-mon[98970]: pgmap v813: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 30102/240717 objects degraded (12.505%); 14995/240717 objects misplaced (6.229%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:00.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:00 smithi027 ceph-mon[93921]: pgmap v813: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 30102/240717 objects degraded (12.505%); 14995/240717 objects misplaced (6.229%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:00.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:00.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:00.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:00.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:00.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:00.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:00.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:00.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:00.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:00.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:00.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:00.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:01.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:23:01.077 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:23:01.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:01.119 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:01.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:01.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:01.244 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:01.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:01.321 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:23:01.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:01.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:01.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:01.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:01.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:01.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:01.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:01.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:01.602 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:01.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:01.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:01.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:01.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:01.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:01.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:01.877 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:01.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:01.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:01.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:02.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:02.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:02.126 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:02.168 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:23:02.228 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:02.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:02 smithi027 ceph-mon[93921]: pgmap v814: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/240258 objects degraded (12.529%); 14995/240258 objects misplaced (6.241%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1433480826' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/240258 objects degraded (12.529%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:02 smithi027 ceph-mon[98970]: pgmap v814: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/240258 objects degraded (12.529%); 14995/240258 objects misplaced (6.241%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1433480826' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/240258 objects degraded (12.529%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:02.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:02.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:02.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:02.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:02.498 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:02 smithi171 ceph-mon[90031]: pgmap v814: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 30102/240258 objects degraded (12.529%); 14995/240258 objects misplaced (6.241%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:02.498 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1433480826' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:02.498 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/240258 objects degraded (12.529%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:02.503 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:02.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:02.605 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:02.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:23:02.628 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:02.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:02.754 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:02.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:02.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:02.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:02.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:02.986 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:03.008 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:03.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:03.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:03.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:03.230 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:23:03.230 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:23:03.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:03.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:03.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:03.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.385 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:03.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:03.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:03.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:03.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:03 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:23:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:03 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1433480826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:03 smithi171 ceph-mon[90031]: osdmap e256: 8 total, 8 up, 8 in 2024-08-29T15:23:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:03 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:23:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:03 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1433480826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:03 smithi027 ceph-mon[93921]: osdmap e256: 8 total, 8 up, 8 in 2024-08-29T15:23:03.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:03 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:23:03.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:03 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1433480826' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:03.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:03 smithi027 ceph-mon[98970]: osdmap e256: 8 total, 8 up, 8 in 2024-08-29T15:23:03.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:03.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:03.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:03.757 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:03.766 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:03.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:03.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:03.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:03.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:04.008 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:04.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:23:04.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:04.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:04.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:04.240 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:04.264 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:04 smithi027 ceph-mon[98970]: pgmap v816: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 30102/238734 objects degraded (12.609%); 14875/238734 objects misplaced (6.231%); 648 KiB/s, 10 objects/s recovering 2024-08-29T15:23:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:04 smithi027 ceph-mon[93921]: pgmap v816: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 30102/238734 objects degraded (12.609%); 14875/238734 objects misplaced (6.231%); 648 KiB/s, 10 objects/s recovering 2024-08-29T15:23:04.366 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:04.388 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:04.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:04.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:04 smithi171 ceph-mon[90031]: pgmap v816: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 30102/238734 objects degraded (12.609%); 14875/238734 objects misplaced (6.231%); 648 KiB/s, 10 objects/s recovering 2024-08-29T15:23:04.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:04.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.638 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:04.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:04.739 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:04.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:04.868 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:04.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:04.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:04.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:04.905 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:23:04.905 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:23:04.909 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:23:04.909 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:23:04.927 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:23:04.927 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:23:04.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:23:04.944 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:23:04.960 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:23:04.960 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:23:04.977 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:23:04.977 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:23:04.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:04.993 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:23:04.993 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:23:05.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:23:05.010 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:23:05.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:05.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:05.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:05.139 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:05.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:23:05.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:05.288 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:05.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:05.413 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:05.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:05.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4134521071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:05.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:05.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4134521071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:05.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4134521071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:05.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:05.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:05.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:05.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:05.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:05.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:05.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:05.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:06.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:06.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:06.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:06.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:06.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:06.288 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:06 smithi027 ceph-mon[93921]: pgmap v817: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/238479 objects degraded (12.622%); 14875/238479 objects misplaced (6.237%); 550 KiB/s, 8 objects/s recovering 2024-08-29T15:23:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:06 smithi027 ceph-mon[98970]: pgmap v817: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/238479 objects degraded (12.622%); 14875/238479 objects misplaced (6.237%); 550 KiB/s, 8 objects/s recovering 2024-08-29T15:23:06.390 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:06.412 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:06.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:06.509 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:06.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:06 smithi171 ceph-mon[90031]: pgmap v817: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/238479 objects degraded (12.622%); 14875/238479 objects misplaced (6.237%); 550 KiB/s, 8 objects/s recovering 2024-08-29T15:23:06.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:06.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:06.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:06.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:06.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:06.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:06.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:06.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:06.912 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:06.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:07.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:07.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:07.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:07.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:07.160 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:07.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:07.283 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:07.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:07.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:07.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/238479 objects degraded (12.622%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:07.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:07.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:07.566 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/238479 objects degraded (12.622%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:07.566 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/238479 objects degraded (12.622%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:07.633 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:23:07.634 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:23:07.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:07.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:07.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:07.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:07.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:07.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:07.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:07.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:08.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:08.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:08.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:08.202 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:08.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:08.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:08 smithi027 ceph-mon[93921]: pgmap v818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/237258 objects degraded (12.687%); 14780/237258 objects misplaced (6.230%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:08 smithi027 ceph-mon[98970]: pgmap v818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/237258 objects degraded (12.687%); 14780/237258 objects misplaced (6.230%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:08.328 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:08.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:08.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:08.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:23:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.454 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:08.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:08 smithi171 ceph-mon[90031]: pgmap v818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/237258 objects degraded (12.687%); 14780/237258 objects misplaced (6.230%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:08.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:08.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:08.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:08.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:08.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:08.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:08.757 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":818,"stamp":"2024-08-29T15:23:06.584544+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5185869821,"num_objects":79086,"num_object_clones":0,"num_object_copies":237258,"num_objects_missing_on_primary":5948,"num_objects_missing":5948,"num_objects_degraded":30102,"num_objects_misplaced":14780,"num_objects_unfound":0,"num_objects_dirty":79086,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":172915,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15518,"num_bytes_recovered":895814861,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124951,"ondisk_log_size":124951,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17018288,"kb_used_data":15170408,"kb_used_omap":43,"kb_used_meta":1847636,"kb_avail":732942928,"statfs":{"total":767960285184,"available":750533558272,"internally_reserved":0,"allocated":15534497792,"data_stored":15522970510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1891979756},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-90374144,"num_objects":-1379,"num_object_clones":0,"num_object_copies":-4137,"num_objects_missing_on_primary":-81,"num_objects_missing":-81,"num_objects_degraded":0,"num_objects_misplaced":-252,"num_objects_unfound":0,"num_objects_dirty":-1379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1379,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":81,"num_bytes_recovered":5308416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003190"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":395,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.167272+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:22:55.167272+0000","last_peered":"2024-08-29T15:22:55.167272+0000","last_clean":"2024-08-29T15:22:55.167272+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:22:55.167272+0000","last_undegraded":"2024-08-29T15:22:55.167272+0000","last_fullsized":"2024-08-29T15:22:55.167272+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":33,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:55.163864+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:22:55.163864+0000","last_peered":"2024-08-29T15:22:55.163864+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:22:55.163864+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:22:55.163864+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.162170+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:22:55.162170+0000","last_peered":"2024-08-29T15:22:55.162170+0000","last_clean":"2024-08-29T15:22:55.162170+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:22:55.162170+0000","last_undegraded":"2024-08-29T15:22:55.162170+0000","last_fullsized":"2024-08-29T15:22:55.162170+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":362,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242328+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:23:02.242328+0000","last_peered":"2024-08-29T15:23:02.242328+0000","last_clean":"2024-08-29T15:23:02.242328+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:23:02.242328+0000","last_undegraded":"2024-08-29T15:23:02.242328+0000","last_fullsized":"2024-08-29T15:23:02.242328+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":362,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242751+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:23:02.242751+0000","last_peered":"2024-08-29T15:23:02.242751+0000","last_clean":"2024-08-29T15:23:02.242751+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:23:02.242751+0000","last_undegraded":"2024-08-29T15:23:02.242751+0000","last_fullsized":"2024-08-29T15:23:02.242751+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163580+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:22:55.163580+0000","last_peered":"2024-08-29T15:22:55.163580+0000","last_clean":"2024-08-29T15:22:55.163580+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:22:55.163580+0000","last_undegraded":"2024-08-29T15:22:55.163580+0000","last_fullsized":"2024-08-29T15:22:55.163580+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":174,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173850+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:23:02.173850+0000","last_peered":"2024-08-29T15:23:02.173850+0000","last_clean":"2024-08-29T15:23:02.173850+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:23:02.173850+0000","last_undegraded":"2024-08-29T15:23:02.173850+0000","last_fullsized":"2024-08-29T15:23:02.173850+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":124,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.264553+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:22:55.264553+0000","last_peered":"2024-08-29T15:22:55.264553+0000","last_clean":"2024-08-29T15:22:55.264553+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:22:55.264553+0000","last_undegraded":"2024-08-29T15:22:55.264553+0000","last_fullsized":"2024-08-29T15:22:55.264553+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"256'10922","reported_seq":18956,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:06.017272+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:23:06.017272+0000","last_peered":"2024-08-29T15:23:06.017272+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:23:06.017272+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'900","ondisk_log_start":"175'900","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327024640,"num_objects":4990,"num_object_clones":0,"num_object_copies":14970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10922,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":2065}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":32,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:55.164355+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:22:55.164355+0000","last_peered":"2024-08-29T15:22:55.164355+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:22:55.164355+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:22:55.164355+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":141,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.161484+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:22:55.161484+0000","last_peered":"2024-08-29T15:22:55.161484+0000","last_clean":"2024-08-29T15:22:55.161484+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:22:55.161484+0000","last_undegraded":"2024-08-29T15:22:55.161484+0000","last_fullsized":"2024-08-29T15:22:55.161484+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"255'10610","reported_seq":10798,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:23:01.639437+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:23:01.639437+0000","last_peered":"2024-08-29T15:23:01.639437+0000","last_clean":"2024-08-29T15:23:01.639437+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:23:01.639437+0000","last_undegraded":"2024-08-29T15:23:01.639437+0000","last_fullsized":"2024-08-29T15:23:01.639437+0000","mapping_epoch":168,"log_start":"179'3928","ondisk_log_start":"179'3928","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6682,"log_dups_size":0,"ondisk_log_size":6682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323747840,"num_objects":4940,"num_object_clones":0,"num_object_copies":14820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10610,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":32,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:55.167288+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:22:55.167288+0000","last_peered":"2024-08-29T15:22:55.167288+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:22:55.167288+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:22:55.167288+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163467+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:22:55.163467+0000","last_peered":"2024-08-29T15:22:55.163467+0000","last_clean":"2024-08-29T15:22:55.163467+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:22:55.163467+0000","last_undegraded":"2024-08-29T15:22:55.163467+0000","last_fullsized":"2024-08-29T15:22:55.163467+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"255'10766","reported_seq":10924,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:01.312099+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:23:01.312099+0000","last_peered":"2024-08-29T15:23:01.312099+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:23:01.312099+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"179'4074","ondisk_log_start":"179'4074","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324141056,"num_objects":4946,"num_object_clones":0,"num_object_copies":14838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10766,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2469}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":178,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242587+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:23:02.242587+0000","last_peered":"2024-08-29T15:23:02.242587+0000","last_clean":"2024-08-29T15:23:02.242587+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:23:02.242587+0000","last_undegraded":"2024-08-29T15:23:02.242587+0000","last_fullsized":"2024-08-29T15:23:02.242587+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":79,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.265020+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:22:55.265020+0000","last_peered":"2024-08-29T15:22:55.265020+0000","last_clean":"2024-08-29T15:22:55.265020+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:22:55.265020+0000","last_undegraded":"2024-08-29T15:22:55.265020+0000","last_fullsized":"2024-08-29T15:22:55.265020+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"256'10893","reported_seq":16303,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:03.818042+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:23:03.818042+0000","last_peered":"2024-08-29T15:23:03.818042+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:23:03.818042+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"177'2762","ondisk_log_start":"177'2762","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8131,"log_dups_size":0,"ondisk_log_size":8131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320143360,"num_objects":4885,"num_object_clones":0,"num_object_copies":14655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10893,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4885}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":141,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.162037+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:22:55.162037+0000","last_peered":"2024-08-29T15:22:55.162037+0000","last_clean":"2024-08-29T15:22:55.162037+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:22:55.162037+0000","last_undegraded":"2024-08-29T15:22:55.162037+0000","last_fullsized":"2024-08-29T15:22:55.162037+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":377,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242660+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:23:02.242660+0000","last_peered":"2024-08-29T15:23:02.242660+0000","last_clean":"2024-08-29T15:23:02.242660+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:23:02.242660+0000","last_undegraded":"2024-08-29T15:23:02.242660+0000","last_fullsized":"2024-08-29T15:23:02.242660+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"256'10793","reported_seq":10935,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:04.691549+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:23:04.691549+0000","last_peered":"2024-08-29T15:23:04.691549+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:23:04.691549+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:23:04.691549+0000","mapping_epoch":234,"log_start":"178'2833","ondisk_log_start":"178'2833","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7960,"log_dups_size":0,"ondisk_log_size":7960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329056256,"num_objects":5021,"num_object_clones":0,"num_object_copies":15063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10793,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2544},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":369,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.166972+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:22:55.166972+0000","last_peered":"2024-08-29T15:22:55.166972+0000","last_clean":"2024-08-29T15:22:55.166972+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:22:55.166972+0000","last_undegraded":"2024-08-29T15:22:55.166972+0000","last_fullsized":"2024-08-29T15:22:55.166972+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163889+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:22:55.163889+0000","last_peered":"2024-08-29T15:22:55.163889+0000","last_clean":"2024-08-29T15:22:55.163889+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:22:55.163889+0000","last_undegraded":"2024-08-29T15:22:55.163889+0000","last_fullsized":"2024-08-29T15:22:55.163889+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"255'10776","reported_seq":10980,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:01.466607+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:23:01.466607+0000","last_peered":"2024-08-29T15:23:01.466607+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:23:01.466607+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:23:01.466607+0000","mapping_epoch":234,"log_start":"179'4068","ondisk_log_start":"179'4068","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6708,"log_dups_size":0,"ondisk_log_size":6708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332005376,"num_objects":5066,"num_object_clones":0,"num_object_copies":15198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10776,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2601},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":97,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.264881+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:22:55.264881+0000","last_peered":"2024-08-29T15:22:55.264881+0000","last_clean":"2024-08-29T15:22:55.264881+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:22:55.264881+0000","last_undegraded":"2024-08-29T15:22:55.264881+0000","last_fullsized":"2024-08-29T15:22:55.264881+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.164240+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:22:55.164240+0000","last_peered":"2024-08-29T15:22:55.164240+0000","last_clean":"2024-08-29T15:22:55.164240+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:22:55.164240+0000","last_undegraded":"2024-08-29T15:22:55.164240+0000","last_fullsized":"2024-08-29T15:22:55.164240+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"255'10629","reported_seq":11018,"reported_epoch":255,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:23:01.603184+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:23:01.603184+0000","last_peered":"2024-08-29T15:23:01.603184+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:23:01.603184+0000","last_undegraded":"2024-08-29T15:23:01.603184+0000","last_fullsized":"2024-08-29T15:23:01.603184+0000","mapping_epoch":202,"log_start":"179'3867","ondisk_log_start":"179'3867","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6762,"log_dups_size":0,"ondisk_log_size":6762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329580544,"num_objects":5029,"num_object_clones":0,"num_object_copies":15087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5029,"num_objects_unfound":0,"num_objects_dirty":5029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10629,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5029}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163502+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:22:55.163502+0000","last_peered":"2024-08-29T15:22:55.163502+0000","last_clean":"2024-08-29T15:22:55.163502+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:22:55.163502+0000","last_undegraded":"2024-08-29T15:22:55.163502+0000","last_fullsized":"2024-08-29T15:22:55.163502+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":240,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.167397+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:22:55.167397+0000","last_peered":"2024-08-29T15:22:55.167397+0000","last_clean":"2024-08-29T15:22:55.167397+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:22:55.167397+0000","last_undegraded":"2024-08-29T15:22:55.167397+0000","last_fullsized":"2024-08-29T15:22:55.167397+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"256'10784","reported_seq":10920,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:05.968095+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:23:05.968095+0000","last_peered":"2024-08-29T15:23:05.968095+0000","last_clean":"2024-08-29T15:23:05.968095+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:23:05.968095+0000","last_undegraded":"2024-08-29T15:23:05.968095+0000","last_fullsized":"2024-08-29T15:23:05.968095+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9506,"log_dups_size":0,"ondisk_log_size":9506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326041624,"num_objects":4976,"num_object_clones":0,"num_object_copies":14928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4976,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10784,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":372,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.166512+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:22:55.166512+0000","last_peered":"2024-08-29T15:22:55.166512+0000","last_clean":"2024-08-29T15:22:55.166512+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:22:55.166512+0000","last_undegraded":"2024-08-29T15:22:55.166512+0000","last_fullsized":"2024-08-29T15:22:55.166512+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":366,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173987+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:23:02.173987+0000","last_peered":"2024-08-29T15:23:02.173987+0000","last_clean":"2024-08-29T15:23:02.173987+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:23:02.173987+0000","last_undegraded":"2024-08-29T15:23:02.173987+0000","last_fullsized":"2024-08-29T15:23:02.173987+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"256'10454","reported_seq":10638,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:05.945681+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:23:05.945681+0000","last_peered":"2024-08-29T15:23:05.945681+0000","last_clean":"2024-08-29T15:23:05.945681+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:23:05.945681+0000","last_undegraded":"2024-08-29T15:23:05.945681+0000","last_fullsized":"2024-08-29T15:23:05.945681+0000","mapping_epoch":201,"log_start":"175'400","ondisk_log_start":"175'400","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311558144,"num_objects":4754,"num_object_clones":0,"num_object_copies":14262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10454,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":465,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242243+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:23:02.242243+0000","last_peered":"2024-08-29T15:23:02.242243+0000","last_clean":"2024-08-29T15:23:02.242243+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:23:02.242243+0000","last_undegraded":"2024-08-29T15:23:02.242243+0000","last_fullsized":"2024-08-29T15:23:02.242243+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":725,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.161729+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:22:55.161729+0000","last_peered":"2024-08-29T15:22:55.161729+0000","last_clean":"2024-08-29T15:22:55.161729+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:22:55.161729+0000","last_undegraded":"2024-08-29T15:22:55.161729+0000","last_fullsized":"2024-08-29T15:22:55.161729+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":233,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.162283+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:22:55.162283+0000","last_peered":"2024-08-29T15:22:55.162283+0000","last_clean":"2024-08-29T15:22:55.162283+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:22:55.162283+0000","last_undegraded":"2024-08-29T15:22:55.162283+0000","last_fullsized":"2024-08-29T15:22:55.162283+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":95,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.265232+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:22:55.265232+0000","last_peered":"2024-08-29T15:22:55.265232+0000","last_clean":"2024-08-29T15:22:55.265232+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:22:55.265232+0000","last_undegraded":"2024-08-29T15:22:55.265232+0000","last_fullsized":"2024-08-29T15:22:55.265232+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"256'10815","reported_seq":11203,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:05.414674+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:23:05.414674+0000","last_peered":"2024-08-29T15:23:05.414674+0000","last_clean":"2024-08-29T15:23:05.414674+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:23:05.414674+0000","last_undegraded":"2024-08-29T15:23:05.414674+0000","last_fullsized":"2024-08-29T15:23:05.414674+0000","mapping_epoch":201,"log_start":"179'3753","ondisk_log_start":"179'3753","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318963712,"num_objects":4867,"num_object_clones":0,"num_object_copies":14601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10815,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":839,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173450+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:23:02.173450+0000","last_peered":"2024-08-29T15:23:02.173450+0000","last_clean":"2024-08-29T15:23:02.173450+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:23:02.173450+0000","last_undegraded":"2024-08-29T15:23:02.173450+0000","last_fullsized":"2024-08-29T15:23:02.173450+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":377,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242216+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:23:02.242216+0000","last_peered":"2024-08-29T15:23:02.242216+0000","last_clean":"2024-08-29T15:23:02.242216+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:23:02.242216+0000","last_undegraded":"2024-08-29T15:23:02.242216+0000","last_fullsized":"2024-08-29T15:23:02.242216+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":385,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242996+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:23:02.242996+0000","last_peered":"2024-08-29T15:23:02.242996+0000","last_clean":"2024-08-29T15:23:02.242996+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:23:02.242996+0000","last_undegraded":"2024-08-29T15:23:02.242996+0000","last_fullsized":"2024-08-29T15:23:02.242996+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"255'10650","reported_seq":16460,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:01.689013+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:23:01.689013+0000","last_peered":"2024-08-29T15:23:01.689013+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:23:01.689013+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"179'3930","ondisk_log_start":"179'3930","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330301440,"num_objects":5040,"num_object_clones":0,"num_object_copies":15120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10650,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":606},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":646,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.166573+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:22:55.166573+0000","last_peered":"2024-08-29T15:22:55.166573+0000","last_clean":"2024-08-29T15:22:55.166573+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:22:55.166573+0000","last_undegraded":"2024-08-29T15:22:55.166573+0000","last_fullsized":"2024-08-29T15:22:55.166573+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":86,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.264271+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:22:55.264271+0000","last_peered":"2024-08-29T15:22:55.264271+0000","last_clean":"2024-08-29T15:22:55.264271+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:22:55.264271+0000","last_undegraded":"2024-08-29T15:22:55.264271+0000","last_fullsized":"2024-08-29T15:22:55.264271+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":412,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173693+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:23:02.173693+0000","last_peered":"2024-08-29T15:23:02.173693+0000","last_clean":"2024-08-29T15:23:02.173693+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:23:02.173693+0000","last_undegraded":"2024-08-29T15:23:02.173693+0000","last_fullsized":"2024-08-29T15:23:02.173693+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"255'10746","reported_seq":10944,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:23:01.644628+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:23:01.644628+0000","last_peered":"2024-08-29T15:23:01.644628+0000","last_clean":"2024-08-29T15:23:01.644628+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:23:01.644628+0000","last_undegraded":"2024-08-29T15:23:01.644628+0000","last_fullsized":"2024-08-29T15:23:01.644628+0000","mapping_epoch":234,"log_start":"179'3997","ondisk_log_start":"179'3997","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325451776,"num_objects":4966,"num_object_clones":0,"num_object_copies":14898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10746,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":193,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.161650+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:22:55.161650+0000","last_peered":"2024-08-29T15:22:55.161650+0000","last_clean":"2024-08-29T15:22:55.161650+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:22:55.161650+0000","last_undegraded":"2024-08-29T15:22:55.161650+0000","last_fullsized":"2024-08-29T15:22:55.161650+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163822+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:22:55.163822+0000","last_peered":"2024-08-29T15:22:55.163822+0000","last_clean":"2024-08-29T15:22:55.163822+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:22:55.163822+0000","last_undegraded":"2024-08-29T15:22:55.163822+0000","last_fullsized":"2024-08-29T15:22:55.163822+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.161587+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:22:55.161587+0000","last_peered":"2024-08-29T15:22:55.161587+0000","last_clean":"2024-08-29T15:22:55.161587+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:22:55.161587+0000","last_undegraded":"2024-08-29T15:22:55.161587+0000","last_fullsized":"2024-08-29T15:22:55.161587+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"256'10810","reported_seq":11008,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:03.944046+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:23:03.944046+0000","last_peered":"2024-08-29T15:23:03.944046+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:23:03.944046+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"177'2642","ondisk_log_start":"177'2642","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8168,"log_dups_size":0,"ondisk_log_size":8168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328728576,"num_objects":5016,"num_object_clones":0,"num_object_copies":15048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10810,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2593}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":125,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.162104+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:22:55.162104+0000","last_peered":"2024-08-29T15:22:55.162104+0000","last_clean":"2024-08-29T15:22:55.162104+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:22:55.162104+0000","last_undegraded":"2024-08-29T15:22:55.162104+0000","last_fullsized":"2024-08-29T15:22:55.162104+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":804,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173851+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:23:02.173851+0000","last_peered":"2024-08-29T15:23:02.173851+0000","last_clean":"2024-08-29T15:23:02.173851+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:23:02.173851+0000","last_undegraded":"2024-08-29T15:23:02.173851+0000","last_fullsized":"2024-08-29T15:23:02.173851+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":381,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173234+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:23:02.173234+0000","last_peered":"2024-08-29T15:23:02.173234+0000","last_clean":"2024-08-29T15:23:02.173234+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:23:02.173234+0000","last_undegraded":"2024-08-29T15:23:02.173234+0000","last_fullsized":"2024-08-29T15:23:02.173234+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"256'10775","reported_seq":11529,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:05.419089+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:23:05.419089+0000","last_peered":"2024-08-29T15:23:05.419089+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:23:05.419089+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"179'3780","ondisk_log_start":"179'3780","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320667648,"num_objects":4893,"num_object_clones":0,"num_object_copies":14679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4860,"num_objects_unfound":0,"num_objects_dirty":4893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10775,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2742}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":386,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173735+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:23:02.173735+0000","last_peered":"2024-08-29T15:23:02.173735+0000","last_clean":"2024-08-29T15:23:02.173735+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:23:02.173735+0000","last_undegraded":"2024-08-29T15:23:02.173735+0000","last_fullsized":"2024-08-29T15:23:02.173735+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":682,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:55.242556+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:22:55.242556+0000","last_peered":"2024-08-29T15:22:55.242556+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:22:55.242556+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:22:55.242556+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":377,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242534+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:23:02.242534+0000","last_peered":"2024-08-29T15:23:02.242534+0000","last_clean":"2024-08-29T15:23:02.242534+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:23:02.242534+0000","last_undegraded":"2024-08-29T15:23:02.242534+0000","last_fullsized":"2024-08-29T15:23:02.242534+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"256'10664","reported_seq":17804,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:04.711883+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:23:04.711883+0000","last_peered":"2024-08-29T15:23:04.711883+0000","last_clean":"2024-08-29T15:23:04.711883+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:23:04.711883+0000","last_undegraded":"2024-08-29T15:23:04.711883+0000","last_fullsized":"2024-08-29T15:23:04.711883+0000","mapping_epoch":234,"log_start":"177'2699","ondisk_log_start":"177'2699","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7965,"log_dups_size":0,"ondisk_log_size":7965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312213504,"num_objects":4764,"num_object_clones":0,"num_object_copies":14292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10664,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":369,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.170488+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:22:55.170488+0000","last_peered":"2024-08-29T15:22:55.170488+0000","last_clean":"2024-08-29T15:22:55.170488+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:22:55.170488+0000","last_undegraded":"2024-08-29T15:22:55.170488+0000","last_fullsized":"2024-08-29T15:22:55.170488+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":994,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:02.172914+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:23:02.172914+0000","last_peered":"2024-08-29T15:23:02.172914+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:23:02.172914+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":367,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.166393+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:22:55.166393+0000","last_peered":"2024-08-29T15:22:55.166393+0000","last_clean":"2024-08-29T15:22:55.166393+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:22:55.166393+0000","last_undegraded":"2024-08-29T15:22:55.166393+0000","last_fullsized":"2024-08-29T15:22:55.166393+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"256'10705","reported_seq":14423,"reported_epoch":256,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:06.321617+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:23:06.321617+0000","last_peered":"2024-08-29T15:23:06.321617+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:23:06.321617+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'3112","ondisk_log_start":"178'3112","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7593,"log_dups_size":0,"ondisk_log_size":7593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321454080,"num_objects":4905,"num_object_clones":0,"num_object_copies":14715,"num_objects_missing_on_primary":5940,"num_objects_missing":5940,"num_objects_degraded":2313,"num_objects_misplaced":4891,"num_objects_unfound":0,"num_objects_dirty":4905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10705,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1782,"num_bytes_recovered":116654080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":2520},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":367,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242650+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:23:02.242650+0000","last_peered":"2024-08-29T15:23:02.242650+0000","last_clean":"2024-08-29T15:23:02.242650+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:23:02.242650+0000","last_undegraded":"2024-08-29T15:23:02.242650+0000","last_fullsized":"2024-08-29T15:23:02.242650+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":106,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.265614+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:22:55.265614+0000","last_peered":"2024-08-29T15:22:55.265614+0000","last_clean":"2024-08-29T15:22:55.265614+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:22:55.265614+0000","last_undegraded":"2024-08-29T15:22:55.265614+0000","last_fullsized":"2024-08-29T15:22:55.265614+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163537+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:22:55.163537+0000","last_peered":"2024-08-29T15:22:55.163537+0000","last_clean":"2024-08-29T15:22:55.163537+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:22:55.163537+0000","last_undegraded":"2024-08-29T15:22:55.163537+0000","last_fullsized":"2024-08-29T15:22:55.163537+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163897+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:22:55.163897+0000","last_peered":"2024-08-29T15:22:55.163897+0000","last_clean":"2024-08-29T15:22:55.163897+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:22:55.163897+0000","last_undegraded":"2024-08-29T15:22:55.163897+0000","last_fullsized":"2024-08-29T15:22:55.163897+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":32,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:55.164009+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:22:55.164009+0000","last_peered":"2024-08-29T15:22:55.164009+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:22:55.164009+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:22:55.164009+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":375,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173082+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:23:02.173082+0000","last_peered":"2024-08-29T15:23:02.173082+0000","last_clean":"2024-08-29T15:23:02.173082+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:23:02.173082+0000","last_undegraded":"2024-08-29T15:23:02.173082+0000","last_fullsized":"2024-08-29T15:23:02.173082+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":124,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:55.170338+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:22:55.170338+0000","last_peered":"2024-08-29T15:22:55.170338+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:22:55.170338+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:22:55.170338+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":123,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.162018+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:22:55.162018+0000","last_peered":"2024-08-29T15:22:55.162018+0000","last_clean":"2024-08-29T15:22:55.162018+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:22:55.162018+0000","last_undegraded":"2024-08-29T15:22:55.162018+0000","last_fullsized":"2024-08-29T15:22:55.162018+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":333,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.166356+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:22:55.166356+0000","last_peered":"2024-08-29T15:22:55.166356+0000","last_clean":"2024-08-29T15:22:55.166356+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:22:55.166356+0000","last_undegraded":"2024-08-29T15:22:55.166356+0000","last_fullsized":"2024-08-29T15:22:55.166356+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":388,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242813+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:23:02.242813+0000","last_peered":"2024-08-29T15:23:02.242813+0000","last_clean":"2024-08-29T15:23:02.242813+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:23:02.242813+0000","last_undegraded":"2024-08-29T15:23:02.242813+0000","last_fullsized":"2024-08-29T15:23:02.242813+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":385,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.166344+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:22:55.166344+0000","last_peered":"2024-08-29T15:22:55.166344+0000","last_clean":"2024-08-29T15:22:55.166344+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:22:55.166344+0000","last_undegraded":"2024-08-29T15:22:55.166344+0000","last_fullsized":"2024-08-29T15:22:55.166344+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":94,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.265639+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:22:55.265639+0000","last_peered":"2024-08-29T15:22:55.265639+0000","last_clean":"2024-08-29T15:22:55.265639+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:22:55.265639+0000","last_undegraded":"2024-08-29T15:22:55.265639+0000","last_fullsized":"2024-08-29T15:22:55.265639+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":243,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242129+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:23:02.242129+0000","last_peered":"2024-08-29T15:23:02.242129+0000","last_clean":"2024-08-29T15:23:02.242129+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:23:02.242129+0000","last_undegraded":"2024-08-29T15:23:02.242129+0000","last_fullsized":"2024-08-29T15:23:02.242129+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":363,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242916+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:23:02.242916+0000","last_peered":"2024-08-29T15:23:02.242916+0000","last_clean":"2024-08-29T15:23:02.242916+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:23:02.242916+0000","last_undegraded":"2024-08-29T15:23:02.242916+0000","last_fullsized":"2024-08-29T15:23:02.242916+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":367,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173302+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:23:02.173302+0000","last_peered":"2024-08-29T15:23:02.173302+0000","last_clean":"2024-08-29T15:23:02.173302+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:23:02.173302+0000","last_undegraded":"2024-08-29T15:23:02.173302+0000","last_fullsized":"2024-08-29T15:23:02.173302+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":369,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.170005+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:22:55.170005+0000","last_peered":"2024-08-29T15:22:55.170005+0000","last_clean":"2024-08-29T15:22:55.170005+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:22:55.170005+0000","last_undegraded":"2024-08-29T15:22:55.170005+0000","last_fullsized":"2024-08-29T15:22:55.170005+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163692+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:22:55.163692+0000","last_peered":"2024-08-29T15:22:55.163692+0000","last_clean":"2024-08-29T15:22:55.163692+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:22:55.163692+0000","last_undegraded":"2024-08-29T15:22:55.163692+0000","last_fullsized":"2024-08-29T15:22:55.163692+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":391,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.173979+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:23:02.173979+0000","last_peered":"2024-08-29T15:23:02.173979+0000","last_clean":"2024-08-29T15:23:02.173979+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:23:02.173979+0000","last_undegraded":"2024-08-29T15:23:02.173979+0000","last_fullsized":"2024-08-29T15:23:02.173979+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":355,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242507+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:23:02.242507+0000","last_peered":"2024-08-29T15:23:02.242507+0000","last_clean":"2024-08-29T15:23:02.242507+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:23:02.242507+0000","last_undegraded":"2024-08-29T15:23:02.242507+0000","last_fullsized":"2024-08-29T15:23:02.242507+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163735+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:22:55.163735+0000","last_peered":"2024-08-29T15:22:55.163735+0000","last_clean":"2024-08-29T15:22:55.163735+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:22:55.163735+0000","last_undegraded":"2024-08-29T15:22:55.163735+0000","last_fullsized":"2024-08-29T15:22:55.163735+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":147,"reported_epoch":255,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:22:55.161757+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:22:55.161757+0000","last_peered":"2024-08-29T15:22:55.161757+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:22:55.161757+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:22:55.161757+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163736+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:22:55.163736+0000","last_peered":"2024-08-29T15:22:55.163736+0000","last_clean":"2024-08-29T15:22:55.163736+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:22:55.163736+0000","last_undegraded":"2024-08-29T15:22:55.163736+0000","last_fullsized":"2024-08-29T15:22:55.163736+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":153,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.162258+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:22:55.162258+0000","last_peered":"2024-08-29T15:22:55.162258+0000","last_clean":"2024-08-29T15:22:55.162258+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:22:55.162258+0000","last_undegraded":"2024-08-29T15:22:55.162258+0000","last_fullsized":"2024-08-29T15:22:55.162258+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":400,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.166628+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:22:55.166628+0000","last_peered":"2024-08-29T15:22:55.166628+0000","last_clean":"2024-08-29T15:22:55.166628+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:22:55.166628+0000","last_undegraded":"2024-08-29T15:22:55.166628+0000","last_fullsized":"2024-08-29T15:22:55.166628+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":402,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.167793+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:22:55.167793+0000","last_peered":"2024-08-29T15:22:55.167793+0000","last_clean":"2024-08-29T15:22:55.167793+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:22:55.167793+0000","last_undegraded":"2024-08-29T15:22:55.167793+0000","last_fullsized":"2024-08-29T15:22:55.167793+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":34,"reported_epoch":255,"state":"active+clean","last_fresh":"2024-08-29T15:22:55.163389+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:22:55.163389+0000","last_peered":"2024-08-29T15:22:55.163389+0000","last_clean":"2024-08-29T15:22:55.163389+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:22:55.163389+0000","last_undegraded":"2024-08-29T15:22:55.163389+0000","last_fullsized":"2024-08-29T15:22:55.163389+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":411,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.242760+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:23:02.242760+0000","last_peered":"2024-08-29T15:23:02.242760+0000","last_clean":"2024-08-29T15:23:02.242760+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:23:02.242760+0000","last_undegraded":"2024-08-29T15:23:02.242760+0000","last_fullsized":"2024-08-29T15:23:02.242760+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5181079576,"num_objects":79058,"num_object_clones":0,"num_object_copies":237174,"num_objects_missing_on_primary":5940,"num_objects_missing":5940,"num_objects_degraded":30073,"num_objects_misplaced":14780,"num_objects_unfound":0,"num_objects_dirty":79058,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171792,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15409,"num_bytes_recovered":889847808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15480598528,"data_stored":15480586312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123769,"ondisk_log_size":123769,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234341,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2834904,"kb_used_data":2533072,"kb_used_omap":1,"kb_used_meta":301822,"kb_avail":90910248,"statfs":{"total":95995035648,"available":93092093952,"internally_reserved":0,"allocated":2593865728,"data_stored":2592411259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":309066187},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495973,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362452,"kb_used_data":178764,"kb_used_omap":1,"kb_used_meta":183678,"kb_avail":93382700,"statfs":{"total":95995035648,"available":95623884800,"internally_reserved":0,"allocated":183054336,"data_stored":181603667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188086727},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426559,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1639496,"kb_used_data":1576128,"kb_used_omap":8,"kb_used_meta":63287,"kb_avail":92105656,"statfs":{"total":95995035648,"available":94316191744,"internally_reserved":0,"allocated":1613955072,"data_stored":1612511869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":64806794},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019251,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1900732,"kb_used_data":1572656,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":91844420,"statfs":{"total":95995035648,"available":94048686080,"internally_reserved":0,"allocated":1610399744,"data_stored":1608972324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":335935942},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435435,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3464116,"kb_used_data":3175596,"kb_used_omap":4,"kb_used_meta":288443,"kb_avail":90281036,"statfs":{"total":95995035648,"available":92447780864,"internally_reserved":0,"allocated":3251810304,"data_stored":3250378689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":295365897},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903287,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2948344,"kb_used_data":2838832,"kb_used_omap":10,"kb_used_meta":109493,"kb_avail":90796808,"statfs":{"total":95995035648,"available":92975931392,"internally_reserved":0,"allocated":2906963968,"data_stored":2905532289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":112121360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762489,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2273092,"kb_used_data":2010940,"kb_used_omap":7,"kb_used_meta":262136,"kb_avail":91472060,"statfs":{"total":95995035648,"available":93667389440,"internally_reserved":0,"allocated":2059202560,"data_stored":2057762057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":268427618},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926015,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595152,"kb_used_data":1284420,"kb_used_omap":7,"kb_used_meta":310712,"kb_avail":92150000,"statfs":{"total":95995035648,"available":94361600000,"internally_reserved":0,"allocated":1315246080,"data_stored":1313798356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":318169231},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1304891392,"data_stored":1304887320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2049773568,"data_stored":2049769496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2901082112,"data_stored":2901082112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3245604864,"data_stored":3245604864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1606352896,"data_stored":1606352896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1609568256,"data_stored":1609564184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":178847744,"data_stored":178847744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2584477696,"data_stored":2584477696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:23:08.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:23:08.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:08.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:08.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:08.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:08.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:08.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:09.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:09.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:09.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:09.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:09.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:09 smithi027 ceph-mon[98970]: from='client.35688 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:09.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:09 smithi027 ceph-mon[93921]: from='client.35688 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:09.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:23:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:23:09] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:23:09.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:09.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:09.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:09.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:09.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:09 smithi171 ceph-mon[90031]: from='client.35688 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:09.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:09.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:09.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:09.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:09.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:09.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:09.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:09.878 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:09.879 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:09.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:09.965 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:09.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:09.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:10.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:10.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:10.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:10.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:10.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:10.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:10.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:10.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:10.404 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:10 smithi027 ceph-mon[98970]: pgmap v819: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 30102/236289 objects degraded (12.739%); 14723/236289 objects misplaced (6.231%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:10.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:10 smithi027 ceph-mon[93921]: pgmap v819: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 30102/236289 objects degraded (12.739%); 14723/236289 objects misplaced (6.231%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:10.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:10.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:10 smithi171 ceph-mon[90031]: pgmap v819: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 30102/236289 objects degraded (12.739%); 14723/236289 objects misplaced (6.231%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:10.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:10.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:10.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:10.656 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:23:10.657 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:23:10.657 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:23:10.657 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:23:10.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:10.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:10.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:10.839 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:10.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:10.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:10.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:10.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:10.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:11.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:11.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:11.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:11.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:11.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:11.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:11.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:11.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:11.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:23:11.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:11.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:11.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:11.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:11.717 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:11.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:11.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:11.845 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:11.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:11.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:11.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:11.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:12.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:12.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:12.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:12.195 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:12.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:12.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:12.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:12.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:12.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:12.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:12.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:12 smithi171 ceph-mon[90031]: pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 30102/235740 objects degraded (12.769%); 14723/235740 objects misplaced (6.245%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/235740 objects degraded (12.769%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:12.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:12.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:12 smithi027 ceph-mon[98970]: pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 30102/235740 objects degraded (12.769%); 14723/235740 objects misplaced (6.245%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:12.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/235740 objects degraded (12.769%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:12 smithi027 ceph-mon[93921]: pgmap v820: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 30102/235740 objects degraded (12.769%); 14723/235740 objects misplaced (6.245%); 518 KiB/s, 8 objects/s recovering 2024-08-29T15:23:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/235740 objects degraded (12.769%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:12.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:12.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:12.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:12.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:12.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:12.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:12.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:12.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:12.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:12.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:12.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:13.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.089 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:13.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:13.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:13.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:13.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:13.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:13.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:13.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:13.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:13.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:13.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:13.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:13.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:13.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:13.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:13.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:13.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:13.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:13.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:13.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:13.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:14.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:14.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:14.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:14.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:14.225 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:14.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:14.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:14.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.349 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:14.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:14.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:14.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:14.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:14 smithi027 ceph-mon[98970]: pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 30102/233793 objects degraded (12.875%); 14571/233793 objects misplaced (6.232%); 522 KiB/s, 8 objects/s recovering 2024-08-29T15:23:14.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:14.486 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:14.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:14 smithi027 ceph-mon[93921]: pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 30102/233793 objects degraded (12.875%); 14571/233793 objects misplaced (6.232%); 522 KiB/s, 8 objects/s recovering 2024-08-29T15:23:14.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:14.486 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:14 smithi171 ceph-mon[90031]: pgmap v821: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 158 op/s; 30102/233793 objects degraded (12.875%); 14571/233793 objects misplaced (6.232%); 522 KiB/s, 8 objects/s recovering 2024-08-29T15:23:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:14.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:14.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:14.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:14.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:14.785 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:14.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:14.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:14.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:14.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:14.970 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:14.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:15.072 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:15.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:15.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:15.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:15.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:15.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:15.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:15.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:15.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:15.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:15.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.598 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:15.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:15.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:15.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:15.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:15.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:15.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:15.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:15.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:15.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:15.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:16.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:16.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:16.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:16.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:16.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:16.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:16 smithi027 ceph-mon[93921]: pgmap v822: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 30102/233577 objects degraded (12.887%); 14571/233577 objects misplaced (6.238%); 453 KiB/s, 7 objects/s recovering 2024-08-29T15:23:16.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:16 smithi027 ceph-mon[98970]: pgmap v822: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 30102/233577 objects degraded (12.887%); 14571/233577 objects misplaced (6.238%); 453 KiB/s, 7 objects/s recovering 2024-08-29T15:23:16.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:16.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:16.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:16.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:16 smithi171 ceph-mon[90031]: pgmap v822: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 30102/233577 objects degraded (12.887%); 14571/233577 objects misplaced (6.238%); 453 KiB/s, 7 objects/s recovering 2024-08-29T15:23:16.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:16.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:16.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:16.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:16.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:16.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:16.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:16.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:16.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:16.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:16.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:17.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:17.097 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:17.199 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:17.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:17.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:17.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/233577 objects degraded (12.887%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:17.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/233577 objects degraded (12.887%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:17.323 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:17.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:17.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:17.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:17.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/233577 objects degraded (12.887%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:17.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:17.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:17.704 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:17.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:17.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:17.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:17.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:17.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:17.906 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:23:17.906 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:23:17.906 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:23:17.907 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:23:17.917 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:23:17.917 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:23:17.937 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:23:17.937 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:23:17.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:17.959 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:23:17.959 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:23:17.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:17.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:17.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:17.980 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:23:17.980 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:23:18.005 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:23:18.006 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:23:18.029 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:23:18.030 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:23:18.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:18.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:18.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:18.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:18.212 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:23:18.238 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:18.340 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:18.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:18.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:18.488 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:18 smithi171 ceph-mon[90031]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 30102/232740 objects degraded (12.934%); 14484/232740 objects misplaced (6.223%); 698 KiB/s, 10 objects/s recovering 2024-08-29T15:23:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:18 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3677926409' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:23:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:18 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3571389632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:18 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1379976097' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:23:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[93921]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 30102/232740 objects degraded (12.934%); 14484/232740 objects misplaced (6.223%); 698 KiB/s, 10 objects/s recovering 2024-08-29T15:23:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3677926409' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:23:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3571389632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1379976097' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:23:18.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[98970]: pgmap v823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 30102/232740 objects degraded (12.934%); 14484/232740 objects misplaced (6.223%); 698 KiB/s, 10 objects/s recovering 2024-08-29T15:23:18.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3677926409' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:23:18.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3571389632' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:18.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:18 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1379976097' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:23:18.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:18.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:18.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:18.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:18.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:18.863 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:18.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:18.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:18.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:18.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:19.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:19.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:19.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:19.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:19.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:19.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:19.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:19.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:19.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:19 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:19.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3677926409' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:23:19.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:19 smithi027 ceph-mon[93921]: osdmap e257: 8 total, 8 up, 8 in 2024-08-29T15:23:19.297 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:23:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:23:19] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:23:19.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:19 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:19.298 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3677926409' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:23:19.298 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:19 smithi027 ceph-mon[98970]: osdmap e257: 8 total, 8 up, 8 in 2024-08-29T15:23:19.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:19.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:19.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:19.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:19.397 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:23:19.397 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/232740 objects degraded (12.934%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1179,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":256,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":77580,"data_bytes":5087172605,"bytes_used":17133084672,"bytes_avail":750827200512,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":232740,"degraded_ratio":0.12933745810775973,"misplaced_objects":14484,"misplaced_total":232740,"misplaced_ratio":0.062232534158288215,"recovering_objects_per_sec":10,"recovering_bytes_per_sec":715216,"recovering_keys_per_sec":0,"num_objects_recovered":131,"num_bytes_recovered":8585216,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":159},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:23:19.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:19.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:19.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:19.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:19 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3677926409' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:23:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:19 smithi171 ceph-mon[90031]: osdmap e257: 8 total, 8 up, 8 in 2024-08-29T15:23:19.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:19.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:19.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:19.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:19.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:19.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:19.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:19.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:19.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:19.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:19.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:19.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:19.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:20.008 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:20.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:20.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:20.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:20.175 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:23:20.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:23:20.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:20.257 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:20.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:20.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:20.380 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:20.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:20.482 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:20.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:20 smithi171 ceph-mon[90031]: pgmap v825: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 30102/231975 objects degraded (12.976%); 14439/231975 objects misplaced (6.224%); 569 KiB/s, 8 objects/s recovering 2024-08-29T15:23:20.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:20 smithi027 ceph-mon[98970]: pgmap v825: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 30102/231975 objects degraded (12.976%); 14439/231975 objects misplaced (6.224%); 569 KiB/s, 8 objects/s recovering 2024-08-29T15:23:20.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:20 smithi027 ceph-mon[93921]: pgmap v825: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 30102/231975 objects degraded (12.976%); 14439/231975 objects misplaced (6.224%); 569 KiB/s, 8 objects/s recovering 2024-08-29T15:23:20.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:20.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:20.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:20.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:20.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:20.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:20.834 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:23:20.834 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:23:20.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:20.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:20.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:20.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:21.012 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:21.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:23:21.115 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:21.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:21.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:21.264 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:21.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:21.392 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:21.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:21.516 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:21.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:21.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:21.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:21.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:23:21.701 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:21.732 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:21.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:21.768 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:21.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:21.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:21.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:21.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:21.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:22.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:22.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:22.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:22.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:22.269 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:22 smithi027 ceph-mon[98970]: pgmap v826: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 30102/231384 objects degraded (13.010%); 14439/231384 objects misplaced (6.240%); 569 KiB/s, 8 objects/s recovering 2024-08-29T15:23:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/231384 objects degraded (13.010%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:22 smithi027 ceph-mon[93921]: pgmap v826: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 30102/231384 objects degraded (13.010%); 14439/231384 objects misplaced (6.240%); 569 KiB/s, 8 objects/s recovering 2024-08-29T15:23:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/231384 objects degraded (13.010%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:22.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:22.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:22.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:22 smithi171 ceph-mon[90031]: pgmap v826: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 30102/231384 objects degraded (13.010%); 14439/231384 objects misplaced (6.240%); 569 KiB/s, 8 objects/s recovering 2024-08-29T15:23:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/231384 objects degraded (13.010%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:22.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:22.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:22.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:22.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:22.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:23:22.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:22.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:22.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:22.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:22.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:22.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:23.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:23.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:23.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:23.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:23.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:23.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:23.268 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:23.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:23.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:23.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:23.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:23.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:23.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:23.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:23.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:23.768 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:23.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:23.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:23.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:23.962 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:24.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:24.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:24.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:24.140 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:23:24.142 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:23:24.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:24.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:24.293 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:24.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:24.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:24.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:24.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:24.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:24.475 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:24 smithi027 ceph-mon[98970]: pgmap v827: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/229554 objects degraded (13.113%); 14298/229554 objects misplaced (6.229%); 563 KiB/s, 8 objects/s recovering 2024-08-29T15:23:24.475 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:24 smithi027 ceph-mon[93921]: pgmap v827: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/229554 objects degraded (13.113%); 14298/229554 objects misplaced (6.229%); 563 KiB/s, 8 objects/s recovering 2024-08-29T15:23:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:24 smithi171 ceph-mon[90031]: pgmap v827: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/229554 objects degraded (13.113%); 14298/229554 objects misplaced (6.229%); 563 KiB/s, 8 objects/s recovering 2024-08-29T15:23:24.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:24.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:24.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:24.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:24.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:24.816 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:24.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:24.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:24.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:25.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:25.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:25.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:25.183 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":827,"stamp":"2024-08-29T15:23:22.589306+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5017573373,"num_objects":76518,"num_object_clones":0,"num_object_copies":229554,"num_objects_missing_on_primary":5816,"num_objects_missing":5816,"num_objects_degraded":30102,"num_objects_misplaced":14298,"num_objects_unfound":0,"num_objects_dirty":76518,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":175483,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15649,"num_bytes_recovered":904400077,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125119,"ondisk_log_size":125119,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16534672,"kb_used_data":14695112,"kb_used_omap":43,"kb_used_meta":1839444,"kb_avail":733426544,"statfs":{"total":767960285184,"available":751028781056,"internally_reserved":0,"allocated":15047794688,"data_stored":15036190926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1883591148},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,1,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-92602368,"num_objects":-1413,"num_object_clones":0,"num_object_copies":-4239,"num_objects_missing_on_primary":-89,"num_objects_missing":-89,"num_objects_degraded":0,"num_objects_misplaced":-273,"num_objects_unfound":0,"num_objects_dirty":-1413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1413,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":5767168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002745"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":399,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217294+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:23:18.217294+0000","last_peered":"2024-08-29T15:23:18.217294+0000","last_clean":"2024-08-29T15:23:18.217294+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:23:18.217294+0000","last_undegraded":"2024-08-29T15:23:18.217294+0000","last_fullsized":"2024-08-29T15:23:18.217294+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":34,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:02.171894+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:23:02.171894+0000","last_peered":"2024-08-29T15:23:02.171894+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:23:02.171894+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:23:02.171894+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":124,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169852+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:23:02.169852+0000","last_peered":"2024-08-29T15:23:02.169852+0000","last_clean":"2024-08-29T15:23:02.169852+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:23:02.169852+0000","last_undegraded":"2024-08-29T15:23:02.169852+0000","last_fullsized":"2024-08-29T15:23:02.169852+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":364,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.230103+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:23:18.230103+0000","last_peered":"2024-08-29T15:23:18.230103+0000","last_clean":"2024-08-29T15:23:18.230103+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:23:18.230103+0000","last_undegraded":"2024-08-29T15:23:18.230103+0000","last_fullsized":"2024-08-29T15:23:18.230103+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":364,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.227452+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:23:18.227452+0000","last_peered":"2024-08-29T15:23:18.227452+0000","last_clean":"2024-08-29T15:23:18.227452+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:23:18.227452+0000","last_undegraded":"2024-08-29T15:23:18.227452+0000","last_fullsized":"2024-08-29T15:23:18.227452+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171748+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:23:02.171748+0000","last_peered":"2024-08-29T15:23:02.171748+0000","last_clean":"2024-08-29T15:23:02.171748+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:23:02.171748+0000","last_undegraded":"2024-08-29T15:23:02.171748+0000","last_fullsized":"2024-08-29T15:23:02.171748+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":176,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214774+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:23:18.214774+0000","last_peered":"2024-08-29T15:23:18.214774+0000","last_clean":"2024-08-29T15:23:18.214774+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:23:18.214774+0000","last_undegraded":"2024-08-29T15:23:18.214774+0000","last_fullsized":"2024-08-29T15:23:18.214774+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":125,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170692+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:23:02.170692+0000","last_peered":"2024-08-29T15:23:02.170692+0000","last_clean":"2024-08-29T15:23:02.170692+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:23:02.170692+0000","last_undegraded":"2024-08-29T15:23:02.170692+0000","last_fullsized":"2024-08-29T15:23:02.170692+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"257'11039","reported_seq":19074,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:21.081850+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:23:21.081850+0000","last_peered":"2024-08-29T15:23:21.081850+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:23:21.081850+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'1000","ondisk_log_start":"175'1000","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319356928,"num_objects":4873,"num_object_clones":0,"num_object_copies":14619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11039,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":1948}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":33,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:02.172362+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:23:02.172362+0000","last_peered":"2024-08-29T15:23:02.172362+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:23:02.172362+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:23:02.172362+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":142,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169520+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:23:02.169520+0000","last_peered":"2024-08-29T15:23:02.169520+0000","last_clean":"2024-08-29T15:23:02.169520+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:23:02.169520+0000","last_undegraded":"2024-08-29T15:23:02.169520+0000","last_fullsized":"2024-08-29T15:23:02.169520+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"257'10827","reported_seq":11019,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:21.583817+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:23:21.583817+0000","last_peered":"2024-08-29T15:23:21.583817+0000","last_clean":"2024-08-29T15:23:21.583817+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:23:21.583817+0000","last_undegraded":"2024-08-29T15:23:21.583817+0000","last_fullsized":"2024-08-29T15:23:21.583817+0000","mapping_epoch":168,"log_start":"179'4128","ondisk_log_start":"179'4128","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309526528,"num_objects":4723,"num_object_clones":0,"num_object_copies":14169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10827,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":33,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:02.174880+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:23:02.174880+0000","last_peered":"2024-08-29T15:23:02.174880+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:23:02.174880+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:23:02.174880+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171590+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:23:02.171590+0000","last_peered":"2024-08-29T15:23:02.171590+0000","last_clean":"2024-08-29T15:23:02.171590+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:23:02.171590+0000","last_undegraded":"2024-08-29T15:23:02.171590+0000","last_fullsized":"2024-08-29T15:23:02.171590+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"257'10972","reported_seq":11134,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:21.486486+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:23:21.486486+0000","last_peered":"2024-08-29T15:23:21.486486+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:23:21.486486+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"179'4274","ondisk_log_start":"179'4274","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310640640,"num_objects":4740,"num_object_clones":0,"num_object_copies":14220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10972,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2263}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":181,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.228402+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:23:18.228402+0000","last_peered":"2024-08-29T15:23:18.228402+0000","last_clean":"2024-08-29T15:23:18.228402+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:23:18.228402+0000","last_undegraded":"2024-08-29T15:23:18.228402+0000","last_fullsized":"2024-08-29T15:23:18.228402+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170961+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:23:02.170961+0000","last_peered":"2024-08-29T15:23:02.170961+0000","last_clean":"2024-08-29T15:23:02.170961+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:23:02.170961+0000","last_undegraded":"2024-08-29T15:23:02.170961+0000","last_fullsized":"2024-08-29T15:23:02.170961+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"257'11026","reported_seq":16438,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:18.803163+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:23:18.803163+0000","last_peered":"2024-08-29T15:23:18.803163+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:23:18.803163+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"177'2862","ondisk_log_start":"177'2862","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8164,"log_dups_size":0,"ondisk_log_size":8164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311427072,"num_objects":4752,"num_object_clones":0,"num_object_copies":14256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11026,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4752}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":142,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170007+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:23:02.170007+0000","last_peered":"2024-08-29T15:23:02.170007+0000","last_clean":"2024-08-29T15:23:02.170007+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:23:02.170007+0000","last_undegraded":"2024-08-29T15:23:02.170007+0000","last_fullsized":"2024-08-29T15:23:02.170007+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":379,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.228175+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:23:18.228175+0000","last_peered":"2024-08-29T15:23:18.228175+0000","last_clean":"2024-08-29T15:23:18.228175+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:23:18.228175+0000","last_undegraded":"2024-08-29T15:23:18.228175+0000","last_fullsized":"2024-08-29T15:23:18.228175+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"257'10925","reported_seq":11068,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:19.673426+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:23:19.673426+0000","last_peered":"2024-08-29T15:23:19.673426+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:23:19.673426+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:23:19.673426+0000","mapping_epoch":234,"log_start":"178'2933","ondisk_log_start":"178'2933","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7992,"log_dups_size":0,"ondisk_log_size":7992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320405504,"num_objects":4889,"num_object_clones":0,"num_object_copies":14667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10925,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2412},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":373,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217130+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:23:18.217130+0000","last_peered":"2024-08-29T15:23:18.217130+0000","last_clean":"2024-08-29T15:23:18.217130+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:23:18.217130+0000","last_undegraded":"2024-08-29T15:23:18.217130+0000","last_fullsized":"2024-08-29T15:23:18.217130+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171898+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:23:02.171898+0000","last_peered":"2024-08-29T15:23:02.171898+0000","last_clean":"2024-08-29T15:23:02.171898+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:23:02.171898+0000","last_undegraded":"2024-08-29T15:23:02.171898+0000","last_fullsized":"2024-08-29T15:23:02.171898+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"257'10977","reported_seq":11186,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:21.564886+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:23:21.564886+0000","last_peered":"2024-08-29T15:23:21.564886+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:23:21.564886+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:23:21.564886+0000","mapping_epoch":234,"log_start":"179'4268","ondisk_log_start":"179'4268","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10977,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2400},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":98,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170844+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:23:02.170844+0000","last_peered":"2024-08-29T15:23:02.170844+0000","last_clean":"2024-08-29T15:23:02.170844+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:23:02.170844+0000","last_undegraded":"2024-08-29T15:23:02.170844+0000","last_fullsized":"2024-08-29T15:23:02.170844+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.172167+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:23:02.172167+0000","last_peered":"2024-08-29T15:23:02.172167+0000","last_clean":"2024-08-29T15:23:02.172167+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:23:02.172167+0000","last_undegraded":"2024-08-29T15:23:02.172167+0000","last_fullsized":"2024-08-29T15:23:02.172167+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"257'10845","reported_seq":11238,"reported_epoch":257,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:23:21.671840+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:23:21.671840+0000","last_peered":"2024-08-29T15:23:21.671840+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:23:21.671840+0000","last_undegraded":"2024-08-29T15:23:21.671840+0000","last_fullsized":"2024-08-29T15:23:21.671840+0000","mapping_epoch":202,"log_start":"179'4167","ondisk_log_start":"179'4167","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315424768,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4813,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10845,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4813}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171639+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:23:02.171639+0000","last_peered":"2024-08-29T15:23:02.171639+0000","last_clean":"2024-08-29T15:23:02.171639+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:23:02.171639+0000","last_undegraded":"2024-08-29T15:23:02.171639+0000","last_fullsized":"2024-08-29T15:23:02.171639+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":241,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.174952+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:23:02.174952+0000","last_peered":"2024-08-29T15:23:02.174952+0000","last_clean":"2024-08-29T15:23:02.174952+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:23:02.174952+0000","last_undegraded":"2024-08-29T15:23:02.174952+0000","last_fullsized":"2024-08-29T15:23:02.174952+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"257'10939","reported_seq":11076,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:20.938562+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:23:20.938562+0000","last_peered":"2024-08-29T15:23:20.938562+0000","last_clean":"2024-08-29T15:23:20.938562+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:23:20.938562+0000","last_undegraded":"2024-08-29T15:23:20.938562+0000","last_fullsized":"2024-08-29T15:23:20.938562+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9661,"log_dups_size":0,"ondisk_log_size":9661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315883544,"num_objects":4821,"num_object_clones":0,"num_object_copies":14463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4821,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10939,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":376,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216784+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:23:18.216784+0000","last_peered":"2024-08-29T15:23:18.216784+0000","last_clean":"2024-08-29T15:23:18.216784+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:23:18.216784+0000","last_undegraded":"2024-08-29T15:23:18.216784+0000","last_fullsized":"2024-08-29T15:23:18.216784+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":368,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214866+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:23:18.214866+0000","last_peered":"2024-08-29T15:23:18.214866+0000","last_clean":"2024-08-29T15:23:18.214866+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:23:18.214866+0000","last_undegraded":"2024-08-29T15:23:18.214866+0000","last_fullsized":"2024-08-29T15:23:18.214866+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"257'10584","reported_seq":10769,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:20.910046+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:23:20.910046+0000","last_peered":"2024-08-29T15:23:20.910046+0000","last_clean":"2024-08-29T15:23:20.910046+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:23:20.910046+0000","last_undegraded":"2024-08-29T15:23:20.910046+0000","last_fullsized":"2024-08-29T15:23:20.910046+0000","mapping_epoch":201,"log_start":"175'500","ondisk_log_start":"175'500","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038464,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10584,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":467,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.226740+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:23:18.226740+0000","last_peered":"2024-08-29T15:23:18.226740+0000","last_clean":"2024-08-29T15:23:18.226740+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:23:18.226740+0000","last_undegraded":"2024-08-29T15:23:18.226740+0000","last_fullsized":"2024-08-29T15:23:18.226740+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":726,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169690+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:23:02.169690+0000","last_peered":"2024-08-29T15:23:02.169690+0000","last_clean":"2024-08-29T15:23:02.169690+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:23:02.169690+0000","last_undegraded":"2024-08-29T15:23:02.169690+0000","last_fullsized":"2024-08-29T15:23:02.169690+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":234,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169715+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:23:02.169715+0000","last_peered":"2024-08-29T15:23:02.169715+0000","last_clean":"2024-08-29T15:23:02.169715+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:23:02.169715+0000","last_undegraded":"2024-08-29T15:23:02.169715+0000","last_fullsized":"2024-08-29T15:23:02.169715+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":97,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.226473+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:23:18.226473+0000","last_peered":"2024-08-29T15:23:18.226473+0000","last_clean":"2024-08-29T15:23:18.226473+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:23:18.226473+0000","last_undegraded":"2024-08-29T15:23:18.226473+0000","last_fullsized":"2024-08-29T15:23:18.226473+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"257'10950","reported_seq":11340,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:20.655884+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:23:20.655884+0000","last_peered":"2024-08-29T15:23:20.655884+0000","last_clean":"2024-08-29T15:23:20.655884+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:23:20.655884+0000","last_undegraded":"2024-08-29T15:23:20.655884+0000","last_fullsized":"2024-08-29T15:23:20.655884+0000","mapping_epoch":201,"log_start":"179'3953","ondisk_log_start":"179'3953","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310116352,"num_objects":4732,"num_object_clones":0,"num_object_copies":14196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10950,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":841,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214153+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:23:18.214153+0000","last_peered":"2024-08-29T15:23:18.214153+0000","last_clean":"2024-08-29T15:23:18.214153+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:23:18.214153+0000","last_undegraded":"2024-08-29T15:23:18.214153+0000","last_fullsized":"2024-08-29T15:23:18.214153+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":379,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.227019+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:23:18.227019+0000","last_peered":"2024-08-29T15:23:18.227019+0000","last_clean":"2024-08-29T15:23:18.227019+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:23:18.227019+0000","last_undegraded":"2024-08-29T15:23:18.227019+0000","last_fullsized":"2024-08-29T15:23:18.227019+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":387,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.227725+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:23:18.227725+0000","last_peered":"2024-08-29T15:23:18.227725+0000","last_clean":"2024-08-29T15:23:18.227725+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:23:18.227725+0000","last_undegraded":"2024-08-29T15:23:18.227725+0000","last_fullsized":"2024-08-29T15:23:18.227725+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"257'10854","reported_seq":16668,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:21.521106+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:23:21.521106+0000","last_peered":"2024-08-29T15:23:21.521106+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:23:21.521106+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"179'4130","ondisk_log_start":"179'4130","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316932096,"num_objects":4836,"num_object_clones":0,"num_object_copies":14508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10854,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":402},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":650,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217288+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:23:18.217288+0000","last_peered":"2024-08-29T15:23:18.217288+0000","last_clean":"2024-08-29T15:23:18.217288+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:23:18.217288+0000","last_undegraded":"2024-08-29T15:23:18.217288+0000","last_fullsized":"2024-08-29T15:23:18.217288+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170440+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:23:02.170440+0000","last_peered":"2024-08-29T15:23:02.170440+0000","last_clean":"2024-08-29T15:23:02.170440+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:23:02.170440+0000","last_undegraded":"2024-08-29T15:23:02.170440+0000","last_fullsized":"2024-08-29T15:23:02.170440+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":414,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214641+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:23:18.214641+0000","last_peered":"2024-08-29T15:23:18.214641+0000","last_clean":"2024-08-29T15:23:18.214641+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:23:18.214641+0000","last_undegraded":"2024-08-29T15:23:18.214641+0000","last_fullsized":"2024-08-29T15:23:18.214641+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"257'10945","reported_seq":11147,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:21.763311+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:23:21.763311+0000","last_peered":"2024-08-29T15:23:21.763311+0000","last_clean":"2024-08-29T15:23:21.763311+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:23:21.763311+0000","last_undegraded":"2024-08-29T15:23:21.763311+0000","last_fullsized":"2024-08-29T15:23:21.763311+0000","mapping_epoch":234,"log_start":"179'4197","ondisk_log_start":"179'4197","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6748,"log_dups_size":0,"ondisk_log_size":6748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312410112,"num_objects":4767,"num_object_clones":0,"num_object_copies":14301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10945,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":194,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169620+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:23:02.169620+0000","last_peered":"2024-08-29T15:23:02.169620+0000","last_clean":"2024-08-29T15:23:02.169620+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:23:02.169620+0000","last_undegraded":"2024-08-29T15:23:02.169620+0000","last_fullsized":"2024-08-29T15:23:02.169620+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171843+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:23:02.171843+0000","last_peered":"2024-08-29T15:23:02.171843+0000","last_clean":"2024-08-29T15:23:02.171843+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:23:02.171843+0000","last_undegraded":"2024-08-29T15:23:02.171843+0000","last_fullsized":"2024-08-29T15:23:02.171843+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":101,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169671+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:23:02.169671+0000","last_peered":"2024-08-29T15:23:02.169671+0000","last_clean":"2024-08-29T15:23:02.169671+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:23:02.169671+0000","last_undegraded":"2024-08-29T15:23:02.169671+0000","last_fullsized":"2024-08-29T15:23:02.169671+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"257'10947","reported_seq":11148,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:18.983475+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:23:18.983475+0000","last_peered":"2024-08-29T15:23:18.983475+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:23:18.983475+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"177'2742","ondisk_log_start":"177'2742","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8205,"log_dups_size":0,"ondisk_log_size":8205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319750144,"num_objects":4879,"num_object_clones":0,"num_object_copies":14637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10947,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2456}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170081+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:23:02.170081+0000","last_peered":"2024-08-29T15:23:02.170081+0000","last_clean":"2024-08-29T15:23:02.170081+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:23:02.170081+0000","last_undegraded":"2024-08-29T15:23:02.170081+0000","last_fullsized":"2024-08-29T15:23:02.170081+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":806,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214658+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:23:18.214658+0000","last_peered":"2024-08-29T15:23:18.214658+0000","last_clean":"2024-08-29T15:23:18.214658+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:23:18.214658+0000","last_undegraded":"2024-08-29T15:23:18.214658+0000","last_fullsized":"2024-08-29T15:23:18.214658+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":383,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214415+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:23:18.214415+0000","last_peered":"2024-08-29T15:23:18.214415+0000","last_clean":"2024-08-29T15:23:18.214415+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:23:18.214415+0000","last_undegraded":"2024-08-29T15:23:18.214415+0000","last_fullsized":"2024-08-29T15:23:18.214415+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"257'10909","reported_seq":11665,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:20.619115+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:23:20.619115+0000","last_peered":"2024-08-29T15:23:20.619115+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:23:20.619115+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"179'3880","ondisk_log_start":"179'3880","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7029,"log_dups_size":0,"ondisk_log_size":7029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311885824,"num_objects":4759,"num_object_clones":0,"num_object_copies":14277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4726,"num_objects_unfound":0,"num_objects_dirty":4759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10909,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2608}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":388,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214568+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:23:18.214568+0000","last_peered":"2024-08-29T15:23:18.214568+0000","last_clean":"2024-08-29T15:23:18.214568+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:23:18.214568+0000","last_undegraded":"2024-08-29T15:23:18.214568+0000","last_fullsized":"2024-08-29T15:23:18.214568+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":683,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:02.180683+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:23:02.180683+0000","last_peered":"2024-08-29T15:23:02.180683+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:23:02.180683+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:23:02.180683+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":379,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.228079+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:23:18.228079+0000","last_peered":"2024-08-29T15:23:18.228079+0000","last_clean":"2024-08-29T15:23:18.228079+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:23:18.228079+0000","last_undegraded":"2024-08-29T15:23:18.228079+0000","last_fullsized":"2024-08-29T15:23:18.228079+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"257'10784","reported_seq":17925,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:19.716814+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:23:19.716814+0000","last_peered":"2024-08-29T15:23:19.716814+0000","last_clean":"2024-08-29T15:23:19.716814+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:23:19.716814+0000","last_undegraded":"2024-08-29T15:23:19.716814+0000","last_fullsized":"2024-08-29T15:23:19.716814+0000","mapping_epoch":234,"log_start":"178'2799","ondisk_log_start":"178'2799","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7985,"log_dups_size":0,"ondisk_log_size":7985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304349184,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10784,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":373,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.219995+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:23:18.219995+0000","last_peered":"2024-08-29T15:23:18.219995+0000","last_clean":"2024-08-29T15:23:18.219995+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:23:18.219995+0000","last_undegraded":"2024-08-29T15:23:18.219995+0000","last_fullsized":"2024-08-29T15:23:18.219995+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":996,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:18.215366+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:23:18.215366+0000","last_peered":"2024-08-29T15:23:18.215366+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:23:18.215366+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":371,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216342+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:23:18.216342+0000","last_peered":"2024-08-29T15:23:18.216342+0000","last_clean":"2024-08-29T15:23:18.216342+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:23:18.216342+0000","last_undegraded":"2024-08-29T15:23:18.216342+0000","last_fullsized":"2024-08-29T15:23:18.216342+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"257'10837","reported_seq":14820,"reported_epoch":257,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:21.596511+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:23:21.596511+0000","last_peered":"2024-08-29T15:23:21.596511+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:23:21.596511+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'3312","ondisk_log_start":"178'3312","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7525,"log_dups_size":0,"ondisk_log_size":7525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312803328,"num_objects":4773,"num_object_clones":0,"num_object_copies":14319,"num_objects_missing_on_primary":5808,"num_objects_missing":5808,"num_objects_degraded":2313,"num_objects_misplaced":4759,"num_objects_unfound":0,"num_objects_dirty":4773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10837,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1913,"num_bytes_recovered":125239296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":2388},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":369,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.227344+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:23:18.227344+0000","last_peered":"2024-08-29T15:23:18.227344+0000","last_clean":"2024-08-29T15:23:18.227344+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:23:18.227344+0000","last_undegraded":"2024-08-29T15:23:18.227344+0000","last_fullsized":"2024-08-29T15:23:18.227344+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":107,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171292+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:23:02.171292+0000","last_peered":"2024-08-29T15:23:02.171292+0000","last_clean":"2024-08-29T15:23:02.171292+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:23:02.171292+0000","last_undegraded":"2024-08-29T15:23:02.171292+0000","last_fullsized":"2024-08-29T15:23:02.171292+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171657+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:23:02.171657+0000","last_peered":"2024-08-29T15:23:02.171657+0000","last_clean":"2024-08-29T15:23:02.171657+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:23:02.171657+0000","last_undegraded":"2024-08-29T15:23:02.171657+0000","last_fullsized":"2024-08-29T15:23:02.171657+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171915+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:23:02.171915+0000","last_peered":"2024-08-29T15:23:02.171915+0000","last_clean":"2024-08-29T15:23:02.171915+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:23:02.171915+0000","last_undegraded":"2024-08-29T15:23:02.171915+0000","last_fullsized":"2024-08-29T15:23:02.171915+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":33,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:02.172009+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:23:02.172009+0000","last_peered":"2024-08-29T15:23:02.172009+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:23:02.172009+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:23:02.172009+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":377,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214265+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:23:18.214265+0000","last_peered":"2024-08-29T15:23:18.214265+0000","last_clean":"2024-08-29T15:23:18.214265+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:23:18.214265+0000","last_undegraded":"2024-08-29T15:23:18.214265+0000","last_fullsized":"2024-08-29T15:23:18.214265+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":128,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:18.217940+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:23:18.217940+0000","last_peered":"2024-08-29T15:23:18.217940+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:23:18.217940+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:23:18.217940+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169895+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:23:02.169895+0000","last_peered":"2024-08-29T15:23:02.169895+0000","last_clean":"2024-08-29T15:23:02.169895+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:23:02.169895+0000","last_undegraded":"2024-08-29T15:23:02.169895+0000","last_fullsized":"2024-08-29T15:23:02.169895+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":337,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216318+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:23:18.216318+0000","last_peered":"2024-08-29T15:23:18.216318+0000","last_clean":"2024-08-29T15:23:18.216318+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:23:18.216318+0000","last_undegraded":"2024-08-29T15:23:18.216318+0000","last_fullsized":"2024-08-29T15:23:18.216318+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":390,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.227949+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:23:18.227949+0000","last_peered":"2024-08-29T15:23:18.227949+0000","last_clean":"2024-08-29T15:23:18.227949+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:23:18.227949+0000","last_undegraded":"2024-08-29T15:23:18.227949+0000","last_fullsized":"2024-08-29T15:23:18.227949+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":389,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216256+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:23:18.216256+0000","last_peered":"2024-08-29T15:23:18.216256+0000","last_clean":"2024-08-29T15:23:18.216256+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:23:18.216256+0000","last_undegraded":"2024-08-29T15:23:18.216256+0000","last_fullsized":"2024-08-29T15:23:18.216256+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:11.489582+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:23:11.489582+0000","last_peered":"2024-08-29T15:23:11.489582+0000","last_clean":"2024-08-29T15:23:11.489582+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:23:11.489582+0000","last_undegraded":"2024-08-29T15:23:11.489582+0000","last_fullsized":"2024-08-29T15:23:11.489582+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":245,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.226635+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:23:18.226635+0000","last_peered":"2024-08-29T15:23:18.226635+0000","last_clean":"2024-08-29T15:23:18.226635+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:23:18.226635+0000","last_undegraded":"2024-08-29T15:23:18.226635+0000","last_fullsized":"2024-08-29T15:23:18.226635+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":366,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.228446+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:23:18.228446+0000","last_peered":"2024-08-29T15:23:18.228446+0000","last_clean":"2024-08-29T15:23:18.228446+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:23:18.228446+0000","last_undegraded":"2024-08-29T15:23:18.228446+0000","last_fullsized":"2024-08-29T15:23:18.228446+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":369,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.215079+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:23:18.215079+0000","last_peered":"2024-08-29T15:23:18.215079+0000","last_clean":"2024-08-29T15:23:18.215079+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:23:18.215079+0000","last_undegraded":"2024-08-29T15:23:18.215079+0000","last_fullsized":"2024-08-29T15:23:18.215079+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":373,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217776+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:23:18.217776+0000","last_peered":"2024-08-29T15:23:18.217776+0000","last_clean":"2024-08-29T15:23:18.217776+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:23:18.217776+0000","last_undegraded":"2024-08-29T15:23:18.217776+0000","last_fullsized":"2024-08-29T15:23:18.217776+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171716+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:23:02.171716+0000","last_peered":"2024-08-29T15:23:02.171716+0000","last_clean":"2024-08-29T15:23:02.171716+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:23:02.171716+0000","last_undegraded":"2024-08-29T15:23:02.171716+0000","last_fullsized":"2024-08-29T15:23:02.171716+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":393,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214746+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:23:18.214746+0000","last_peered":"2024-08-29T15:23:18.214746+0000","last_clean":"2024-08-29T15:23:18.214746+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:23:18.214746+0000","last_undegraded":"2024-08-29T15:23:18.214746+0000","last_fullsized":"2024-08-29T15:23:18.214746+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":357,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.226968+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:23:18.226968+0000","last_peered":"2024-08-29T15:23:18.226968+0000","last_clean":"2024-08-29T15:23:18.226968+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:23:18.226968+0000","last_undegraded":"2024-08-29T15:23:18.226968+0000","last_fullsized":"2024-08-29T15:23:18.226968+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171783+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:23:02.171783+0000","last_peered":"2024-08-29T15:23:02.171783+0000","last_clean":"2024-08-29T15:23:02.171783+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:23:02.171783+0000","last_undegraded":"2024-08-29T15:23:02.171783+0000","last_fullsized":"2024-08-29T15:23:02.171783+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":148,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:02.169849+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:23:02.169849+0000","last_peered":"2024-08-29T15:23:02.169849+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:23:02.169849+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:23:02.169849+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171745+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:23:02.171745+0000","last_peered":"2024-08-29T15:23:02.171745+0000","last_clean":"2024-08-29T15:23:02.171745+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:23:02.171745+0000","last_undegraded":"2024-08-29T15:23:02.171745+0000","last_fullsized":"2024-08-29T15:23:02.171745+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":154,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169933+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:23:02.169933+0000","last_peered":"2024-08-29T15:23:02.169933+0000","last_clean":"2024-08-29T15:23:02.169933+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:23:02.169933+0000","last_undegraded":"2024-08-29T15:23:02.169933+0000","last_fullsized":"2024-08-29T15:23:02.169933+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":404,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216766+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:23:18.216766+0000","last_peered":"2024-08-29T15:23:18.216766+0000","last_clean":"2024-08-29T15:23:18.216766+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:23:18.216766+0000","last_undegraded":"2024-08-29T15:23:18.216766+0000","last_fullsized":"2024-08-29T15:23:18.216766+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":406,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.218560+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:23:18.218560+0000","last_peered":"2024-08-29T15:23:18.218560+0000","last_clean":"2024-08-29T15:23:18.218560+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:23:18.218560+0000","last_undegraded":"2024-08-29T15:23:18.218560+0000","last_fullsized":"2024-08-29T15:23:18.218560+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":35,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.171495+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:23:02.171495+0000","last_peered":"2024-08-29T15:23:02.171495+0000","last_clean":"2024-08-29T15:23:02.171495+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:23:02.171495+0000","last_undegraded":"2024-08-29T15:23:02.171495+0000","last_fullsized":"2024-08-29T15:23:02.171495+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":413,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.227852+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:23:18.227852+0000","last_peered":"2024-08-29T15:23:18.227852+0000","last_clean":"2024-08-29T15:23:18.227852+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:23:18.227852+0000","last_undegraded":"2024-08-29T15:23:18.227852+0000","last_fullsized":"2024-08-29T15:23:18.227852+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5012783128,"num_objects":76490,"num_object_clones":0,"num_object_copies":229470,"num_objects_missing_on_primary":5808,"num_objects_missing":5808,"num_objects_degraded":30073,"num_objects_misplaced":14298,"num_objects_unfound":0,"num_objects_dirty":76490,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174360,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15540,"num_bytes_recovered":898433024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14986391552,"data_stored":14986379336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123937,"ondisk_log_size":123937,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234344,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2766632,"kb_used_data":2460704,"kb_used_omap":1,"kb_used_meta":305918,"kb_avail":90978520,"statfs":{"total":95995035648,"available":93162004480,"internally_reserved":0,"allocated":2519760896,"data_stored":2518298779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":313260491},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495976,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":345508,"kb_used_data":161820,"kb_used_omap":1,"kb_used_meta":183678,"kb_avail":93399644,"statfs":{"total":95995035648,"available":95641235456,"internally_reserved":0,"allocated":165703680,"data_stored":164245363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188086727},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,1,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426562,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1596312,"kb_used_data":1533008,"kb_used_omap":8,"kb_used_meta":63287,"kb_avail":92148840,"statfs":{"total":95995035648,"available":94360412160,"internally_reserved":0,"allocated":1569800192,"data_stored":1568349341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":64806794},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019254,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1847244,"kb_used_data":1531456,"kb_used_omap":1,"kb_used_meta":315774,"kb_avail":91897908,"statfs":{"total":95995035648,"available":94103457792,"internally_reserved":0,"allocated":1568210944,"data_stored":1566775876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":323353030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435439,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3330516,"kb_used_data":3050380,"kb_used_omap":4,"kb_used_meta":280123,"kb_avail":90414636,"statfs":{"total":95995035648,"available":92584587264,"internally_reserved":0,"allocated":3123589120,"data_stored":3122142209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":286846217},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903291,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2857176,"kb_used_data":2743504,"kb_used_omap":10,"kb_used_meta":113653,"kb_avail":90887976,"statfs":{"total":95995035648,"available":93069287424,"internally_reserved":0,"allocated":2809348096,"data_stored":2807901121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":116381200},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762492,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2230100,"kb_used_data":1963788,"kb_used_omap":7,"kb_used_meta":266296,"kb_avail":91515052,"statfs":{"total":95995035648,"available":93711413248,"internally_reserved":0,"allocated":2010918912,"data_stored":2009470761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":272687458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926018,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1561184,"kb_used_data":1250452,"kb_used_omap":7,"kb_used_meta":310712,"kb_avail":92183968,"statfs":{"total":95995035648,"available":94396383232,"internally_reserved":0,"allocated":1280462848,"data_stored":1279007476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":318169231},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1269043200,"data_stored":1269039128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2002194432,"data_stored":2002190360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2802778112,"data_stored":2802778112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3116302336,"data_stored":3116302336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1563033600,"data_stored":1563033600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1563889664,"data_stored":1563885592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":160628736,"data_stored":160628736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2508521472,"data_stored":2508521472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:23:25.192 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:25.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:25.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:25.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:25.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:25.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:25.427 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:25 smithi027 ceph-mon[98970]: from='client.45629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:25.427 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:25 smithi027 ceph-mon[93921]: from='client.45629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:25.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:25 smithi171 ceph-mon[90031]: from='client.45629 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:25.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:25.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:25.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:25.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:25.811 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:25.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:25.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:25.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:25.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:25.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:26.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:26.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:26.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:26.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:26.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:26.311 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:26 smithi027 ceph-mon[93921]: pgmap v828: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 30102/229236 objects degraded (13.131%); 14298/229236 objects misplaced (6.237%); 563 KiB/s, 8 objects/s recovering 2024-08-29T15:23:26.311 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:26 smithi027 ceph-mon[98970]: pgmap v828: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 30102/229236 objects degraded (13.131%); 14298/229236 objects misplaced (6.237%); 563 KiB/s, 8 objects/s recovering 2024-08-29T15:23:26.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:26.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:26.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:26.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:26.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:26 smithi171 ceph-mon[90031]: pgmap v828: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 144 op/s; 30102/229236 objects degraded (13.131%); 14298/229236 objects misplaced (6.237%); 563 KiB/s, 8 objects/s recovering 2024-08-29T15:23:26.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:26.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:26.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:26.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:26.712 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:26.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:26.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:26.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:26.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:26.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:26.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:27.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:27.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:27.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:27.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:27.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:27.242 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:23:27.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:27.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:27.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:27.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/349552704' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/229236 objects degraded (13.131%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:27.526 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/349552704' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:27.526 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/229236 objects degraded (13.131%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:27.526 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/349552704' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:27.526 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/229236 objects degraded (13.131%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:27.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:27.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:27.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:27.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:27.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:27.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:27.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:27.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:27.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:27.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:27.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:27.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:28.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:28.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:28.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:28.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:23:28.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:28.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:28.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:28.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:28.460 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:28.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:28 smithi171 ceph-mon[90031]: pgmap v829: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 30102/228186 objects degraded (13.192%); 14242/228186 objects misplaced (6.241%); 269 KiB/s, 4 objects/s recovering 2024-08-29T15:23:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:28 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/349552704' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:28 smithi171 ceph-mon[90031]: osdmap e258: 8 total, 8 up, 8 in 2024-08-29T15:23:28.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[98970]: pgmap v829: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 30102/228186 objects degraded (13.192%); 14242/228186 objects misplaced (6.241%); 269 KiB/s, 4 objects/s recovering 2024-08-29T15:23:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/349552704' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[98970]: osdmap e258: 8 total, 8 up, 8 in 2024-08-29T15:23:28.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[93921]: pgmap v829: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 30102/228186 objects degraded (13.192%); 14242/228186 objects misplaced (6.241%); 269 KiB/s, 4 objects/s recovering 2024-08-29T15:23:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/349552704' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:28 smithi027 ceph-mon[93921]: osdmap e258: 8 total, 8 up, 8 in 2024-08-29T15:23:28.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:28.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:28.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:28.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:28.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:28.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:28.840 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:28.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:28.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:28.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:28.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:29.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:29.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:23:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:23:29] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:23:29.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:29.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:29.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:29.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:29.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:29.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:29.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:29.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:29.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:29.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:29.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:29.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:29.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:29.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:29.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:29.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:29.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:29.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:29.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:29 smithi027 ceph-mon[98970]: pgmap v831: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 30102/227271 objects degraded (13.245%); 14147/227271 objects misplaced (6.225%); 557 KiB/s, 8 objects/s recovering 2024-08-29T15:23:29.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:29.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:29.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:29 smithi027 ceph-mon[93921]: pgmap v831: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 30102/227271 objects degraded (13.245%); 14147/227271 objects misplaced (6.225%); 557 KiB/s, 8 objects/s recovering 2024-08-29T15:23:29.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:29.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:29 smithi171 ceph-mon[90031]: pgmap v831: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 30102/227271 objects degraded (13.245%); 14147/227271 objects misplaced (6.225%); 557 KiB/s, 8 objects/s recovering 2024-08-29T15:23:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:30.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:30.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:30.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:30.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:30.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:30.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:30.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:30.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:30.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:30.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:30.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:30.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:30.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:30.730 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:23:30.730 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:23:30.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:23:30.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:30.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:30.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:30.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:30.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:30.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:31.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:31.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:31.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:31.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:31.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:31.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:31.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:31.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:31.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:23:31.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:31.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:31.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:31.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:31.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:31.586 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:23:31.587 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:23:31.601 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:23:31.601 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:23:31.627 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:23:31.628 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:23:31.648 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:23:31.648 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:23:31.665 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:23:31.665 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:23:31.682 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:23:31.682 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:23:31.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:31.700 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:23:31.701 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:23:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:31.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:31.720 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:23:31.720 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:23:31.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:31.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:31.841 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:31 smithi027 ceph-mon[93921]: pgmap v832: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30102/226803 objects degraded (13.272%); 14147/226803 objects misplaced (6.238%); 557 KiB/s, 8 objects/s recovering 2024-08-29T15:23:31.841 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4194240219' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:31.841 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:31 smithi027 ceph-mon[98970]: pgmap v832: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30102/226803 objects degraded (13.272%); 14147/226803 objects misplaced (6.238%); 557 KiB/s, 8 objects/s recovering 2024-08-29T15:23:31.841 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4194240219' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:31.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:31.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:31.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:31 smithi171 ceph-mon[90031]: pgmap v832: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30102/226803 objects degraded (13.272%); 14147/226803 objects misplaced (6.238%); 557 KiB/s, 8 objects/s recovering 2024-08-29T15:23:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4194240219' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:32.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:32.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:32.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:32.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:32.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:32.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:32.337 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:32.369 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:32.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:32.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:32.475 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:23:32.476 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:23:32.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:32.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:32.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:32.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:32.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/226803 objects degraded (13.272%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/226803 objects degraded (13.272%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:32.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:32.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:32.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:32.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/226803 objects degraded (13.272%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:33.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:33.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:33.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:33.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:33.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:33.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:33.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:33.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:33.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:33.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:33.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:33.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:33.466 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":832,"stamp":"2024-08-29T15:23:30.591344+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4957476861,"num_objects":75601,"num_object_clones":0,"num_object_copies":226803,"num_objects_missing_on_primary":5771,"num_objects_missing":5771,"num_objects_degraded":30102,"num_objects_misplaced":14147,"num_objects_unfound":0,"num_objects_dirty":75601,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":176400,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15694,"num_bytes_recovered":907349197,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125236,"ondisk_log_size":125236,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16347768,"kb_used_data":14504248,"kb_used_omap":43,"kb_used_meta":1843412,"kb_avail":733613448,"statfs":{"total":767960285184,"available":751220170752,"internally_reserved":0,"allocated":14852349952,"data_stored":14840723246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1887654380},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-100073472,"num_objects":-1527,"num_object_clones":0,"num_object_copies":-4581,"num_objects_missing_on_primary":-89,"num_objects_missing":-89,"num_objects_degraded":0,"num_objects_misplaced":-292,"num_objects_unfound":0,"num_objects_dirty":-1527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1527,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":5701632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002663"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":399,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217294+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:23:18.217294+0000","last_peered":"2024-08-29T15:23:18.217294+0000","last_clean":"2024-08-29T15:23:18.217294+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:23:18.217294+0000","last_undegraded":"2024-08-29T15:23:18.217294+0000","last_fullsized":"2024-08-29T15:23:18.217294+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":35,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:18.216223+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:23:18.216223+0000","last_peered":"2024-08-29T15:23:18.216223+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:23:18.216223+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:23:18.216223+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":124,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169852+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:23:02.169852+0000","last_peered":"2024-08-29T15:23:02.169852+0000","last_clean":"2024-08-29T15:23:02.169852+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:23:02.169852+0000","last_undegraded":"2024-08-29T15:23:02.169852+0000","last_fullsized":"2024-08-29T15:23:02.169852+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":366,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244500+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:23:27.244500+0000","last_peered":"2024-08-29T15:23:27.244500+0000","last_clean":"2024-08-29T15:23:27.244500+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:23:27.244500+0000","last_undegraded":"2024-08-29T15:23:27.244500+0000","last_fullsized":"2024-08-29T15:23:27.244500+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":366,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244640+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:23:27.244640+0000","last_peered":"2024-08-29T15:23:27.244640+0000","last_clean":"2024-08-29T15:23:27.244640+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:23:27.244640+0000","last_undegraded":"2024-08-29T15:23:27.244640+0000","last_fullsized":"2024-08-29T15:23:27.244640+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.215903+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:23:18.215903+0000","last_peered":"2024-08-29T15:23:18.215903+0000","last_clean":"2024-08-29T15:23:18.215903+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:23:18.215903+0000","last_undegraded":"2024-08-29T15:23:18.215903+0000","last_fullsized":"2024-08-29T15:23:18.215903+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":176,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214774+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:23:18.214774+0000","last_peered":"2024-08-29T15:23:18.214774+0000","last_clean":"2024-08-29T15:23:18.214774+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:23:18.214774+0000","last_undegraded":"2024-08-29T15:23:18.214774+0000","last_fullsized":"2024-08-29T15:23:18.214774+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":125,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170692+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:23:02.170692+0000","last_peered":"2024-08-29T15:23:02.170692+0000","last_clean":"2024-08-29T15:23:02.170692+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:23:02.170692+0000","last_undegraded":"2024-08-29T15:23:02.170692+0000","last_fullsized":"2024-08-29T15:23:02.170692+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"257'11094","reported_seq":19129,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:26.087711+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:23:26.087711+0000","last_peered":"2024-08-29T15:23:26.087711+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:23:26.087711+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'1000","ondisk_log_start":"175'1000","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315752448,"num_objects":4818,"num_object_clones":0,"num_object_copies":14454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11094,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":1893}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":34,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:18.216893+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:23:18.216893+0000","last_peered":"2024-08-29T15:23:18.216893+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:23:18.216893+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:23:18.216893+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":142,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169520+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:23:02.169520+0000","last_peered":"2024-08-29T15:23:02.169520+0000","last_clean":"2024-08-29T15:23:02.169520+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:23:02.169520+0000","last_undegraded":"2024-08-29T15:23:02.169520+0000","last_fullsized":"2024-08-29T15:23:02.169520+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"257'10871","reported_seq":11063,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:26.720574+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:23:26.720574+0000","last_peered":"2024-08-29T15:23:26.720574+0000","last_clean":"2024-08-29T15:23:26.720574+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:23:26.720574+0000","last_undegraded":"2024-08-29T15:23:26.720574+0000","last_fullsized":"2024-08-29T15:23:26.720574+0000","mapping_epoch":168,"log_start":"179'4128","ondisk_log_start":"179'4128","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306642944,"num_objects":4679,"num_object_clones":0,"num_object_copies":14037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10871,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":34,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:18.219603+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:23:18.219603+0000","last_peered":"2024-08-29T15:23:18.219603+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:23:18.219603+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:23:18.219603+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.215679+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:23:18.215679+0000","last_peered":"2024-08-29T15:23:18.215679+0000","last_clean":"2024-08-29T15:23:18.215679+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:23:18.215679+0000","last_undegraded":"2024-08-29T15:23:18.215679+0000","last_fullsized":"2024-08-29T15:23:18.215679+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"257'11012","reported_seq":11174,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:26.717125+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:23:26.717125+0000","last_peered":"2024-08-29T15:23:26.717125+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:23:26.717125+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"179'4274","ondisk_log_start":"179'4274","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6738,"log_dups_size":0,"ondisk_log_size":6738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308019200,"num_objects":4700,"num_object_clones":0,"num_object_copies":14100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11012,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2223}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":183,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244707+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:23:27.244707+0000","last_peered":"2024-08-29T15:23:27.244707+0000","last_clean":"2024-08-29T15:23:27.244707+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:23:27.244707+0000","last_undegraded":"2024-08-29T15:23:27.244707+0000","last_fullsized":"2024-08-29T15:23:27.244707+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170961+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:23:02.170961+0000","last_peered":"2024-08-29T15:23:02.170961+0000","last_clean":"2024-08-29T15:23:02.170961+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:23:02.170961+0000","last_undegraded":"2024-08-29T15:23:02.170961+0000","last_fullsized":"2024-08-29T15:23:02.170961+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"258'11120","reported_seq":16534,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:28.977152+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:23:28.977152+0000","last_peered":"2024-08-29T15:23:28.977152+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:23:28.977152+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"178'2962","ondisk_log_start":"178'2962","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8158,"log_dups_size":0,"ondisk_log_size":8158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305266688,"num_objects":4658,"num_object_clones":0,"num_object_copies":13974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11120,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4658}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":142,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170007+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:23:02.170007+0000","last_peered":"2024-08-29T15:23:02.170007+0000","last_clean":"2024-08-29T15:23:02.170007+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:23:02.170007+0000","last_undegraded":"2024-08-29T15:23:02.170007+0000","last_fullsized":"2024-08-29T15:23:02.170007+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":381,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244860+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:23:27.244860+0000","last_peered":"2024-08-29T15:23:27.244860+0000","last_clean":"2024-08-29T15:23:27.244860+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:23:27.244860+0000","last_undegraded":"2024-08-29T15:23:27.244860+0000","last_fullsized":"2024-08-29T15:23:27.244860+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"258'11019","reported_seq":11163,"reported_epoch":258,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:29.614819+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:23:29.614819+0000","last_peered":"2024-08-29T15:23:29.614819+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:23:29.614819+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:23:29.614819+0000","mapping_epoch":234,"log_start":"178'3033","ondisk_log_start":"178'3033","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7986,"log_dups_size":0,"ondisk_log_size":7986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314245120,"num_objects":4795,"num_object_clones":0,"num_object_copies":14385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11019,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2318},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":373,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217130+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:23:18.217130+0000","last_peered":"2024-08-29T15:23:18.217130+0000","last_clean":"2024-08-29T15:23:18.217130+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:23:18.217130+0000","last_undegraded":"2024-08-29T15:23:18.217130+0000","last_fullsized":"2024-08-29T15:23:18.217130+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216517+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:23:18.216517+0000","last_peered":"2024-08-29T15:23:18.216517+0000","last_clean":"2024-08-29T15:23:18.216517+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:23:18.216517+0000","last_undegraded":"2024-08-29T15:23:18.216517+0000","last_fullsized":"2024-08-29T15:23:18.216517+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"257'11018","reported_seq":11227,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:26.644787+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:23:26.644787+0000","last_peered":"2024-08-29T15:23:26.644787+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:23:26.644787+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:23:26.644787+0000","mapping_epoch":234,"log_start":"179'4268","ondisk_log_start":"179'4268","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6750,"log_dups_size":0,"ondisk_log_size":6750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316145664,"num_objects":4824,"num_object_clones":0,"num_object_copies":14472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11018,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2359},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":98,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170844+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:23:02.170844+0000","last_peered":"2024-08-29T15:23:02.170844+0000","last_clean":"2024-08-29T15:23:02.170844+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:23:02.170844+0000","last_undegraded":"2024-08-29T15:23:02.170844+0000","last_fullsized":"2024-08-29T15:23:02.170844+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216740+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:23:18.216740+0000","last_peered":"2024-08-29T15:23:18.216740+0000","last_clean":"2024-08-29T15:23:18.216740+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:23:18.216740+0000","last_undegraded":"2024-08-29T15:23:18.216740+0000","last_fullsized":"2024-08-29T15:23:18.216740+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"257'10895","reported_seq":11288,"reported_epoch":257,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:23:26.351496+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:23:26.351496+0000","last_peered":"2024-08-29T15:23:26.351496+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:23:26.351496+0000","last_undegraded":"2024-08-29T15:23:26.351496+0000","last_fullsized":"2024-08-29T15:23:26.351496+0000","mapping_epoch":202,"log_start":"179'4167","ondisk_log_start":"179'4167","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312147968,"num_objects":4763,"num_object_clones":0,"num_object_copies":14289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4763,"num_objects_unfound":0,"num_objects_dirty":4763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10895,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4763}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.215766+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:23:18.215766+0000","last_peered":"2024-08-29T15:23:18.215766+0000","last_clean":"2024-08-29T15:23:18.215766+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:23:18.215766+0000","last_undegraded":"2024-08-29T15:23:18.215766+0000","last_fullsized":"2024-08-29T15:23:18.215766+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":242,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.219605+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:23:18.219605+0000","last_peered":"2024-08-29T15:23:18.219605+0000","last_clean":"2024-08-29T15:23:18.219605+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:23:18.219605+0000","last_undegraded":"2024-08-29T15:23:18.219605+0000","last_fullsized":"2024-08-29T15:23:18.219605+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"257'10994","reported_seq":11131,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:25.825001+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:23:25.825001+0000","last_peered":"2024-08-29T15:23:25.825001+0000","last_clean":"2024-08-29T15:23:25.825001+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:23:25.825001+0000","last_undegraded":"2024-08-29T15:23:25.825001+0000","last_fullsized":"2024-08-29T15:23:25.825001+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9716,"log_dups_size":0,"ondisk_log_size":9716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312279064,"num_objects":4766,"num_object_clones":0,"num_object_copies":14298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4766,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10994,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":376,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216784+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:23:18.216784+0000","last_peered":"2024-08-29T15:23:18.216784+0000","last_clean":"2024-08-29T15:23:18.216784+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:23:18.216784+0000","last_undegraded":"2024-08-29T15:23:18.216784+0000","last_fullsized":"2024-08-29T15:23:18.216784+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":368,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214866+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:23:18.214866+0000","last_peered":"2024-08-29T15:23:18.214866+0000","last_clean":"2024-08-29T15:23:18.214866+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:23:18.214866+0000","last_undegraded":"2024-08-29T15:23:18.214866+0000","last_fullsized":"2024-08-29T15:23:18.214866+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"257'10625","reported_seq":10810,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:25.833517+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:23:25.833517+0000","last_peered":"2024-08-29T15:23:25.833517+0000","last_clean":"2024-08-29T15:23:25.833517+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:23:25.833517+0000","last_undegraded":"2024-08-29T15:23:25.833517+0000","last_fullsized":"2024-08-29T15:23:25.833517+0000","mapping_epoch":201,"log_start":"175'600","ondisk_log_start":"175'600","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10025,"log_dups_size":0,"ondisk_log_size":10025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10625,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":469,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244466+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:23:27.244466+0000","last_peered":"2024-08-29T15:23:27.244466+0000","last_clean":"2024-08-29T15:23:27.244466+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:23:27.244466+0000","last_undegraded":"2024-08-29T15:23:27.244466+0000","last_fullsized":"2024-08-29T15:23:27.244466+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":726,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169690+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:23:02.169690+0000","last_peered":"2024-08-29T15:23:02.169690+0000","last_clean":"2024-08-29T15:23:02.169690+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:23:02.169690+0000","last_undegraded":"2024-08-29T15:23:02.169690+0000","last_fullsized":"2024-08-29T15:23:02.169690+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":234,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169715+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:23:02.169715+0000","last_peered":"2024-08-29T15:23:02.169715+0000","last_clean":"2024-08-29T15:23:02.169715+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:23:02.169715+0000","last_undegraded":"2024-08-29T15:23:02.169715+0000","last_fullsized":"2024-08-29T15:23:02.169715+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":97,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.226473+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:23:18.226473+0000","last_peered":"2024-08-29T15:23:18.226473+0000","last_clean":"2024-08-29T15:23:18.226473+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:23:18.226473+0000","last_undegraded":"2024-08-29T15:23:18.226473+0000","last_fullsized":"2024-08-29T15:23:18.226473+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"257'11008","reported_seq":11398,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:25.641489+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:23:25.641489+0000","last_peered":"2024-08-29T15:23:25.641489+0000","last_clean":"2024-08-29T15:23:25.641489+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:23:25.641489+0000","last_undegraded":"2024-08-29T15:23:25.641489+0000","last_fullsized":"2024-08-29T15:23:25.641489+0000","mapping_epoch":201,"log_start":"179'3953","ondisk_log_start":"179'3953","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306315264,"num_objects":4674,"num_object_clones":0,"num_object_copies":14022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11008,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":841,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214153+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:23:18.214153+0000","last_peered":"2024-08-29T15:23:18.214153+0000","last_clean":"2024-08-29T15:23:18.214153+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:23:18.214153+0000","last_undegraded":"2024-08-29T15:23:18.214153+0000","last_fullsized":"2024-08-29T15:23:18.214153+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":381,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244582+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:23:27.244582+0000","last_peered":"2024-08-29T15:23:27.244582+0000","last_clean":"2024-08-29T15:23:27.244582+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:23:27.244582+0000","last_undegraded":"2024-08-29T15:23:27.244582+0000","last_fullsized":"2024-08-29T15:23:27.244582+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":389,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244771+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:23:27.244771+0000","last_peered":"2024-08-29T15:23:27.244771+0000","last_clean":"2024-08-29T15:23:27.244771+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:23:27.244771+0000","last_undegraded":"2024-08-29T15:23:27.244771+0000","last_fullsized":"2024-08-29T15:23:27.244771+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"257'10902","reported_seq":16716,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:26.630420+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:23:26.630420+0000","last_peered":"2024-08-29T15:23:26.630420+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:23:26.630420+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"179'4230","ondisk_log_start":"179'4230","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313786368,"num_objects":4788,"num_object_clones":0,"num_object_copies":14364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10902,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":354},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":650,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217288+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:23:18.217288+0000","last_peered":"2024-08-29T15:23:18.217288+0000","last_clean":"2024-08-29T15:23:18.217288+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:23:18.217288+0000","last_undegraded":"2024-08-29T15:23:18.217288+0000","last_fullsized":"2024-08-29T15:23:18.217288+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170440+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:23:02.170440+0000","last_peered":"2024-08-29T15:23:02.170440+0000","last_clean":"2024-08-29T15:23:02.170440+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:23:02.170440+0000","last_undegraded":"2024-08-29T15:23:02.170440+0000","last_fullsized":"2024-08-29T15:23:02.170440+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":414,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214641+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:23:18.214641+0000","last_peered":"2024-08-29T15:23:18.214641+0000","last_clean":"2024-08-29T15:23:18.214641+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:23:18.214641+0000","last_undegraded":"2024-08-29T15:23:18.214641+0000","last_fullsized":"2024-08-29T15:23:18.214641+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"257'10982","reported_seq":11184,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:26.590247+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:23:26.590247+0000","last_peered":"2024-08-29T15:23:26.590247+0000","last_clean":"2024-08-29T15:23:26.590247+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:23:26.590247+0000","last_undegraded":"2024-08-29T15:23:26.590247+0000","last_fullsized":"2024-08-29T15:23:26.590247+0000","mapping_epoch":234,"log_start":"179'4297","ondisk_log_start":"179'4297","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309985280,"num_objects":4730,"num_object_clones":0,"num_object_copies":14190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10982,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":194,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169620+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:23:02.169620+0000","last_peered":"2024-08-29T15:23:02.169620+0000","last_clean":"2024-08-29T15:23:02.169620+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:23:02.169620+0000","last_undegraded":"2024-08-29T15:23:02.169620+0000","last_fullsized":"2024-08-29T15:23:02.169620+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216125+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:23:18.216125+0000","last_peered":"2024-08-29T15:23:18.216125+0000","last_clean":"2024-08-29T15:23:18.216125+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:23:18.216125+0000","last_undegraded":"2024-08-29T15:23:18.216125+0000","last_fullsized":"2024-08-29T15:23:18.216125+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":101,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169671+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:23:02.169671+0000","last_peered":"2024-08-29T15:23:02.169671+0000","last_clean":"2024-08-29T15:23:02.169671+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:23:02.169671+0000","last_undegraded":"2024-08-29T15:23:02.169671+0000","last_fullsized":"2024-08-29T15:23:02.169671+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"258'11043","reported_seq":11246,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:28.911011+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:23:28.911011+0000","last_peered":"2024-08-29T15:23:28.911011+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:23:28.911011+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"178'2842","ondisk_log_start":"178'2842","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8201,"log_dups_size":0,"ondisk_log_size":8201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313458688,"num_objects":4783,"num_object_clones":0,"num_object_copies":14349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11043,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2360}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.170081+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:23:02.170081+0000","last_peered":"2024-08-29T15:23:02.170081+0000","last_clean":"2024-08-29T15:23:02.170081+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:23:02.170081+0000","last_undegraded":"2024-08-29T15:23:02.170081+0000","last_fullsized":"2024-08-29T15:23:02.170081+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":806,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214658+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:23:18.214658+0000","last_peered":"2024-08-29T15:23:18.214658+0000","last_clean":"2024-08-29T15:23:18.214658+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:23:18.214658+0000","last_undegraded":"2024-08-29T15:23:18.214658+0000","last_fullsized":"2024-08-29T15:23:18.214658+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":383,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214415+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:23:18.214415+0000","last_peered":"2024-08-29T15:23:18.214415+0000","last_clean":"2024-08-29T15:23:18.214415+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:23:18.214415+0000","last_undegraded":"2024-08-29T15:23:18.214415+0000","last_fullsized":"2024-08-29T15:23:18.214415+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"257'10965","reported_seq":11721,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:25.692672+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:23:25.692672+0000","last_peered":"2024-08-29T15:23:25.692672+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:23:25.692672+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"179'3980","ondisk_log_start":"179'3980","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308215808,"num_objects":4703,"num_object_clones":0,"num_object_copies":14109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4670,"num_objects_unfound":0,"num_objects_dirty":4703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10965,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2552}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":388,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214568+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:23:18.214568+0000","last_peered":"2024-08-29T15:23:18.214568+0000","last_clean":"2024-08-29T15:23:18.214568+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:23:18.214568+0000","last_undegraded":"2024-08-29T15:23:18.214568+0000","last_fullsized":"2024-08-29T15:23:18.214568+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":684,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:18.235328+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:23:18.235328+0000","last_peered":"2024-08-29T15:23:18.235328+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:23:18.235328+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:23:18.235328+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":381,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244631+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:23:27.244631+0000","last_peered":"2024-08-29T15:23:27.244631+0000","last_clean":"2024-08-29T15:23:27.244631+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:23:27.244631+0000","last_undegraded":"2024-08-29T15:23:27.244631+0000","last_fullsized":"2024-08-29T15:23:27.244631+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"258'10847","reported_seq":17989,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:29.618982+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:23:29.618982+0000","last_peered":"2024-08-29T15:23:29.618982+0000","last_clean":"2024-08-29T15:23:29.618982+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:23:29.618982+0000","last_undegraded":"2024-08-29T15:23:29.618982+0000","last_fullsized":"2024-08-29T15:23:29.618982+0000","mapping_epoch":234,"log_start":"178'2899","ondisk_log_start":"178'2899","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7948,"log_dups_size":0,"ondisk_log_size":7948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300220416,"num_objects":4581,"num_object_clones":0,"num_object_copies":13743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10847,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":373,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.219995+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:23:18.219995+0000","last_peered":"2024-08-29T15:23:18.219995+0000","last_clean":"2024-08-29T15:23:18.219995+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:23:18.219995+0000","last_undegraded":"2024-08-29T15:23:18.219995+0000","last_fullsized":"2024-08-29T15:23:18.219995+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":996,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:18.215366+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:23:18.215366+0000","last_peered":"2024-08-29T15:23:18.215366+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:23:18.215366+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":371,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216342+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:23:18.216342+0000","last_peered":"2024-08-29T15:23:18.216342+0000","last_clean":"2024-08-29T15:23:18.216342+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:23:18.216342+0000","last_undegraded":"2024-08-29T15:23:18.216342+0000","last_fullsized":"2024-08-29T15:23:18.216342+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"257'10882","reported_seq":14955,"reported_epoch":257,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:26.673103+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:23:26.673103+0000","last_peered":"2024-08-29T15:23:26.673103+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:23:26.673103+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'3312","ondisk_log_start":"178'3312","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309854208,"num_objects":4728,"num_object_clones":0,"num_object_copies":14184,"num_objects_missing_on_primary":5763,"num_objects_missing":5763,"num_objects_degraded":2313,"num_objects_misplaced":4714,"num_objects_unfound":0,"num_objects_dirty":4728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10882,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1958,"num_bytes_recovered":128188416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":2343},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":371,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244494+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:23:27.244494+0000","last_peered":"2024-08-29T15:23:27.244494+0000","last_clean":"2024-08-29T15:23:27.244494+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:23:27.244494+0000","last_undegraded":"2024-08-29T15:23:27.244494+0000","last_fullsized":"2024-08-29T15:23:27.244494+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":108,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.226569+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:23:18.226569+0000","last_peered":"2024-08-29T15:23:18.226569+0000","last_clean":"2024-08-29T15:23:18.226569+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:23:18.226569+0000","last_undegraded":"2024-08-29T15:23:18.226569+0000","last_fullsized":"2024-08-29T15:23:18.226569+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216160+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:23:18.216160+0000","last_peered":"2024-08-29T15:23:18.216160+0000","last_clean":"2024-08-29T15:23:18.216160+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:23:18.216160+0000","last_undegraded":"2024-08-29T15:23:18.216160+0000","last_fullsized":"2024-08-29T15:23:18.216160+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216337+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:23:18.216337+0000","last_peered":"2024-08-29T15:23:18.216337+0000","last_clean":"2024-08-29T15:23:18.216337+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:23:18.216337+0000","last_undegraded":"2024-08-29T15:23:18.216337+0000","last_fullsized":"2024-08-29T15:23:18.216337+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":34,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:18.216566+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:23:18.216566+0000","last_peered":"2024-08-29T15:23:18.216566+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:23:18.216566+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:23:18.216566+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":377,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214265+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:23:18.214265+0000","last_peered":"2024-08-29T15:23:18.214265+0000","last_clean":"2024-08-29T15:23:18.214265+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:23:18.214265+0000","last_undegraded":"2024-08-29T15:23:18.214265+0000","last_fullsized":"2024-08-29T15:23:18.214265+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":128,"reported_epoch":257,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:18.217940+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:23:18.217940+0000","last_peered":"2024-08-29T15:23:18.217940+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:23:18.217940+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:23:18.217940+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169895+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:23:02.169895+0000","last_peered":"2024-08-29T15:23:02.169895+0000","last_clean":"2024-08-29T15:23:02.169895+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:23:02.169895+0000","last_undegraded":"2024-08-29T15:23:02.169895+0000","last_fullsized":"2024-08-29T15:23:02.169895+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":337,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216318+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:23:18.216318+0000","last_peered":"2024-08-29T15:23:18.216318+0000","last_clean":"2024-08-29T15:23:18.216318+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:23:18.216318+0000","last_undegraded":"2024-08-29T15:23:18.216318+0000","last_fullsized":"2024-08-29T15:23:18.216318+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":392,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244812+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:23:27.244812+0000","last_peered":"2024-08-29T15:23:27.244812+0000","last_clean":"2024-08-29T15:23:27.244812+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:23:27.244812+0000","last_undegraded":"2024-08-29T15:23:27.244812+0000","last_fullsized":"2024-08-29T15:23:27.244812+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":389,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216256+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:23:18.216256+0000","last_peered":"2024-08-29T15:23:18.216256+0000","last_clean":"2024-08-29T15:23:18.216256+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:23:18.216256+0000","last_undegraded":"2024-08-29T15:23:18.216256+0000","last_fullsized":"2024-08-29T15:23:18.216256+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:11.489582+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:23:11.489582+0000","last_peered":"2024-08-29T15:23:11.489582+0000","last_clean":"2024-08-29T15:23:11.489582+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:23:11.489582+0000","last_undegraded":"2024-08-29T15:23:11.489582+0000","last_fullsized":"2024-08-29T15:23:11.489582+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":247,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244285+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:23:27.244285+0000","last_peered":"2024-08-29T15:23:27.244285+0000","last_clean":"2024-08-29T15:23:27.244285+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:23:27.244285+0000","last_undegraded":"2024-08-29T15:23:27.244285+0000","last_fullsized":"2024-08-29T15:23:27.244285+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":368,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244734+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:23:27.244734+0000","last_peered":"2024-08-29T15:23:27.244734+0000","last_clean":"2024-08-29T15:23:27.244734+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:23:27.244734+0000","last_undegraded":"2024-08-29T15:23:27.244734+0000","last_fullsized":"2024-08-29T15:23:27.244734+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":369,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.215079+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:23:18.215079+0000","last_peered":"2024-08-29T15:23:18.215079+0000","last_clean":"2024-08-29T15:23:18.215079+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:23:18.215079+0000","last_undegraded":"2024-08-29T15:23:18.215079+0000","last_fullsized":"2024-08-29T15:23:18.215079+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":373,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.217776+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:23:18.217776+0000","last_peered":"2024-08-29T15:23:18.217776+0000","last_clean":"2024-08-29T15:23:18.217776+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:23:18.217776+0000","last_undegraded":"2024-08-29T15:23:18.217776+0000","last_fullsized":"2024-08-29T15:23:18.217776+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.215957+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:23:18.215957+0000","last_peered":"2024-08-29T15:23:18.215957+0000","last_clean":"2024-08-29T15:23:18.215957+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:23:18.215957+0000","last_undegraded":"2024-08-29T15:23:18.215957+0000","last_fullsized":"2024-08-29T15:23:18.215957+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":393,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.214746+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:23:18.214746+0000","last_peered":"2024-08-29T15:23:18.214746+0000","last_clean":"2024-08-29T15:23:18.214746+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:23:18.214746+0000","last_undegraded":"2024-08-29T15:23:18.214746+0000","last_fullsized":"2024-08-29T15:23:18.214746+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":359,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244517+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:23:27.244517+0000","last_peered":"2024-08-29T15:23:27.244517+0000","last_clean":"2024-08-29T15:23:27.244517+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:23:27.244517+0000","last_undegraded":"2024-08-29T15:23:27.244517+0000","last_fullsized":"2024-08-29T15:23:27.244517+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216053+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:23:18.216053+0000","last_peered":"2024-08-29T15:23:18.216053+0000","last_clean":"2024-08-29T15:23:18.216053+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:23:18.216053+0000","last_undegraded":"2024-08-29T15:23:18.216053+0000","last_fullsized":"2024-08-29T15:23:18.216053+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":148,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:02.169849+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:23:02.169849+0000","last_peered":"2024-08-29T15:23:02.169849+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:23:02.169849+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:23:02.169849+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216028+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:23:18.216028+0000","last_peered":"2024-08-29T15:23:18.216028+0000","last_clean":"2024-08-29T15:23:18.216028+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:23:18.216028+0000","last_undegraded":"2024-08-29T15:23:18.216028+0000","last_fullsized":"2024-08-29T15:23:18.216028+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":154,"reported_epoch":256,"state":"active+clean","last_fresh":"2024-08-29T15:23:02.169933+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:23:02.169933+0000","last_peered":"2024-08-29T15:23:02.169933+0000","last_clean":"2024-08-29T15:23:02.169933+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:23:02.169933+0000","last_undegraded":"2024-08-29T15:23:02.169933+0000","last_fullsized":"2024-08-29T15:23:02.169933+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":405,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.216811+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:23:18.216811+0000","last_peered":"2024-08-29T15:23:18.216811+0000","last_clean":"2024-08-29T15:23:18.216811+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:23:18.216811+0000","last_undegraded":"2024-08-29T15:23:18.216811+0000","last_fullsized":"2024-08-29T15:23:18.216811+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":406,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.218560+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:23:18.218560+0000","last_peered":"2024-08-29T15:23:18.218560+0000","last_clean":"2024-08-29T15:23:18.218560+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:23:18.218560+0000","last_undegraded":"2024-08-29T15:23:18.218560+0000","last_fullsized":"2024-08-29T15:23:18.218560+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":36,"reported_epoch":257,"state":"active+clean","last_fresh":"2024-08-29T15:23:18.215547+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:23:18.215547+0000","last_peered":"2024-08-29T15:23:18.215547+0000","last_clean":"2024-08-29T15:23:18.215547+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:23:18.215547+0000","last_undegraded":"2024-08-29T15:23:18.215547+0000","last_fullsized":"2024-08-29T15:23:18.215547+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":415,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244746+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:23:27.244746+0000","last_peered":"2024-08-29T15:23:27.244746+0000","last_clean":"2024-08-29T15:23:27.244746+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:23:27.244746+0000","last_undegraded":"2024-08-29T15:23:27.244746+0000","last_fullsized":"2024-08-29T15:23:27.244746+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4952686616,"num_objects":75573,"num_object_clones":0,"num_object_copies":226719,"num_objects_missing_on_primary":5763,"num_objects_missing":5763,"num_objects_degraded":30073,"num_objects_misplaced":14147,"num_objects_unfound":0,"num_objects_dirty":75573,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175277,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15585,"num_bytes_recovered":901382144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14795616256,"data_stored":14795604040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124054,"ondisk_log_size":124054,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234346,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2724216,"kb_used_data":2414192,"kb_used_omap":1,"kb_used_meta":310014,"kb_avail":91020936,"statfs":{"total":95995035648,"available":93205438464,"internally_reserved":0,"allocated":2472132608,"data_stored":2470662843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":317454795},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495977,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":342948,"kb_used_data":155164,"kb_used_omap":1,"kb_used_meta":187774,"kb_avail":93402204,"statfs":{"total":95995035648,"available":95643856896,"internally_reserved":0,"allocated":158887936,"data_stored":157429619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":192281031},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426563,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1582424,"kb_used_data":1515024,"kb_used_omap":8,"kb_used_meta":67383,"kb_avail":92162728,"statfs":{"total":95995035648,"available":94374633472,"internally_reserved":0,"allocated":1551384576,"data_stored":1549933725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":69001098},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019256,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1821016,"kb_used_data":1505232,"kb_used_omap":1,"kb_used_meta":315774,"kb_avail":91924136,"statfs":{"total":95995035648,"available":94130315264,"internally_reserved":0,"allocated":1541357568,"data_stored":1539914852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":323353030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435440,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3302548,"kb_used_data":3022412,"kb_used_omap":4,"kb_used_meta":280123,"kb_avail":90442604,"statfs":{"total":95995035648,"available":92613226496,"internally_reserved":0,"allocated":3094949888,"data_stored":3093502977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":286846217},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903292,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2837912,"kb_used_data":2720080,"kb_used_omap":10,"kb_used_meta":117813,"kb_avail":90907240,"statfs":{"total":95995035648,"available":93089013760,"internally_reserved":0,"allocated":2785361920,"data_stored":2783914945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":120641040},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762493,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2214292,"kb_used_data":1947980,"kb_used_omap":7,"kb_used_meta":266296,"kb_avail":91530860,"statfs":{"total":95995035648,"available":93727600640,"internally_reserved":0,"allocated":1994731520,"data_stored":1993283369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":272687458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926020,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1522412,"kb_used_data":1224164,"kb_used_omap":7,"kb_used_meta":298232,"kb_avail":92222740,"statfs":{"total":95995035648,"available":94436085760,"internally_reserved":0,"allocated":1253543936,"data_stored":1252080916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":305389711},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1243746304,"data_stored":1243742232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1984303104,"data_stored":1984299032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2779906048,"data_stored":2779906048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3088449536,"data_stored":3088449536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1537146880,"data_stored":1537146880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1546981376,"data_stored":1546977304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":153944064,"data_stored":153944064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2461138944,"data_stored":2461138944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:23:33.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:23:33.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:33.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:33.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:33.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:33.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:33.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:33.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:33.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:33.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:33.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:33 smithi171 ceph-mon[90031]: from='client.45647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:33 smithi171 ceph-mon[90031]: pgmap v833: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/225429 objects degraded (13.353%); 14009/225429 objects misplaced (6.214%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:34.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:33 smithi027 ceph-mon[98970]: from='client.45647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:33 smithi027 ceph-mon[98970]: pgmap v833: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/225429 objects degraded (13.353%); 14009/225429 objects misplaced (6.214%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:33 smithi027 ceph-mon[93921]: from='client.45647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:33 smithi027 ceph-mon[93921]: pgmap v833: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/225429 objects degraded (13.353%); 14009/225429 objects misplaced (6.214%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:34.080 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:34.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:34.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:34.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:34.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:34.435 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:34.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:34.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:34.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:34.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:34.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:34.807 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:34.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:34.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:34.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:34.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:35.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:35.077 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:35.179 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:35.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:35.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:35.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:35.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:35.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:35.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:35.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:35.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:35.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:35.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:35.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:35.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:35.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:35.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:35.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:35.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.823 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:35.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:35.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:35.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:35.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:35.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:35 smithi027 ceph-mon[98970]: pgmap v834: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/225252 objects degraded (13.364%); 14009/225252 objects misplaced (6.219%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:35.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:35 smithi027 ceph-mon[93921]: pgmap v834: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/225252 objects degraded (13.364%); 14009/225252 objects misplaced (6.219%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:35 smithi171 ceph-mon[90031]: pgmap v834: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/225252 objects degraded (13.364%); 14009/225252 objects misplaced (6.219%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:36.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:36.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:36.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:23:36.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:36.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:36.296 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:36.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:36.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:36.412 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:36.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:36.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:36.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:36.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:36.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:36.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:36.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:36.705 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:23:36.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:36.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/752005845' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:23:36.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/752005845' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:23:36.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:36.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:36.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/752005845' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:23:37.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:37.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:37.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:37.191 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:37.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:37.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:37.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:37.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:37.389 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:37.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:37.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:37.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:37.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:37.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:37.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:37.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:37.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:37.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[98970]: pgmap v835: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/224460 objects degraded (13.411%); 13983/224460 objects misplaced (6.230%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/752005845' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[98970]: osdmap e259: 8 total, 8 up, 8 in 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/224460 objects degraded (13.411%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[93921]: pgmap v835: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/224460 objects degraded (13.411%); 13983/224460 objects misplaced (6.230%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/752005845' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:23:37.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[93921]: osdmap e259: 8 total, 8 up, 8 in 2024-08-29T15:23:37.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/224460 objects degraded (13.411%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:37.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:37.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:37.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:37.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:37 smithi171 ceph-mon[90031]: pgmap v835: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/224460 objects degraded (13.411%); 13983/224460 objects misplaced (6.230%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:37 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:23:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:37 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/752005845' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:23:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:37 smithi171 ceph-mon[90031]: osdmap e259: 8 total, 8 up, 8 in 2024-08-29T15:23:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/224460 objects degraded (13.411%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:38.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:38.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:38.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:38.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:38.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:38.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:38.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:38.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:38.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:38.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:38.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:38.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:38.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:38.668 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:38.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:38.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:38.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:38.856 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:23:38.856 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:23:38.856 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:23:38.856 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:23:38.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:38.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:38.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:39.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:39.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:39.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:39.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:39.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:39.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:39.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:39.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:23:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:23:39] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-08-29T15:23:39.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:39.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:39.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:39.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:39.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:39.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:39.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:39.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:39 smithi027 ceph-mon[98970]: pgmap v837: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/223569 objects degraded (13.464%); 13894/223569 objects misplaced (6.215%); 601 KiB/s, 9 objects/s recovering 2024-08-29T15:23:39.703 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:39 smithi027 ceph-mon[93921]: pgmap v837: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/223569 objects degraded (13.464%); 13894/223569 objects misplaced (6.215%); 601 KiB/s, 9 objects/s recovering 2024-08-29T15:23:39.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:39.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:39.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:39.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:39.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:39.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:39 smithi171 ceph-mon[90031]: pgmap v837: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 30102/223569 objects degraded (13.464%); 13894/223569 objects misplaced (6.215%); 601 KiB/s, 9 objects/s recovering 2024-08-29T15:23:40.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:40.070 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:40.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:40.105 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:23:40.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:40.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:40.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:40.299 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:40.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:40.423 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:40.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:40.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:40.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:40.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:40.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:40.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:40.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:40.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:40.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:40.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:40.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:40.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:41.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:41.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:41.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:41.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:41.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:41.299 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:41.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:41.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:41.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:23:41.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:41.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:41.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:41.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:41.692 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:41.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:41.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:41.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:41.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:41.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:41.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:41.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:41.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:41 smithi171 ceph-mon[90031]: pgmap v838: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/223071 objects degraded (13.494%); 13894/223071 objects misplaced (6.229%); 601 KiB/s, 9 objects/s recovering 2024-08-29T15:23:42.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:41 smithi027 ceph-mon[98970]: pgmap v838: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/223071 objects degraded (13.494%); 13894/223071 objects misplaced (6.229%); 601 KiB/s, 9 objects/s recovering 2024-08-29T15:23:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:41 smithi027 ceph-mon[93921]: pgmap v838: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 30102/223071 objects degraded (13.494%); 13894/223071 objects misplaced (6.229%); 601 KiB/s, 9 objects/s recovering 2024-08-29T15:23:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:42.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:42.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:42.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:42.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:42.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:42.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:42.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:42.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:42.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:42.559 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:23:42.560 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:23:42.579 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:23:42.579 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:23:42.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:42.596 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:23:42.596 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:23:42.613 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:23:42.613 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:23:42.629 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:23:42.630 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:23:42.646 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:23:42.647 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:23:42.665 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:23:42.666 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:23:42.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:42.684 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:23:42.684 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:23:42.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:42.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:42.833 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:42.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:42.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:42.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/223071 objects degraded (13.494%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2186900471' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:43.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:43.062 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:23:43.062 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 30102/221427 objects degraded (13.595%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1204,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":259,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":73809,"data_bytes":4840036349,"bytes_used":16392531968,"bytes_avail":751567753216,"bytes_total":767960285184,"degraded_objects":30102,"degraded_total":221427,"degraded_ratio":0.13594548090341285,"misplaced_objects":13750,"misplaced_total":221427,"misplaced_ratio":0.062097214883460462,"recovering_objects_per_sec":9,"recovering_bytes_per_sec":622399,"recovering_keys_per_sec":0,"num_objects_recovered":95,"num_bytes_recovered":6225920,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":133},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:23:43.062 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/223071 objects degraded (13.494%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:43.063 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2186900471' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/223071 objects degraded (13.494%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:43.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2186900471' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:43.083 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:43.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:43.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:43.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:43.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:43.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:43.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:43.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:43.560 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:43.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:43.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:43.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:43.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:43.802 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:43.803 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:43.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:43.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:43.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:43.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:43.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:43.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:44.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:44.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:44.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:44.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:44.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:43 smithi171 ceph-mon[90031]: pgmap v839: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/221427 objects degraded (13.595%); 13750/221427 objects misplaced (6.210%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2557283671' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:23:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:43 smithi027 ceph-mon[93921]: pgmap v839: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/221427 objects degraded (13.595%); 13750/221427 objects misplaced (6.210%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2557283671' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:23:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:43 smithi027 ceph-mon[98970]: pgmap v839: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 30102/221427 objects degraded (13.595%); 13750/221427 objects misplaced (6.210%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2557283671' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:23:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:44.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:44.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:44.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:44.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:44.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:44.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:44.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:44.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:44.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:44.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:44.763 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:23:44.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:23:44.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:44.839 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:44.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:44.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:44.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:44.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:45.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:45.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:45.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:45.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:45.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:45.341 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:45.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:45.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:45.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:45.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:45.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:23:45.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:45.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:45.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:45.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:45.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:45.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:45.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:45.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:45.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:45.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:45.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:45.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:46.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:46.117 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:46.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:46.219 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:46.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:46.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:46.248 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:45 smithi027 ceph-mon[98970]: pgmap v840: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/221112 objects degraded (13.614%); 13750/221112 objects misplaced (6.219%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:46.248 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:23:46.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:45 smithi027 ceph-mon[93921]: pgmap v840: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/221112 objects degraded (13.614%); 13750/221112 objects misplaced (6.219%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:46.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:23:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:45 smithi171 ceph-mon[90031]: pgmap v840: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 30102/221112 objects degraded (13.614%); 13750/221112 objects misplaced (6.219%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:23:46.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:46.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:46.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:46.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:46.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:46.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:46.594 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:46.594 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:46.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:46.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:46.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:46.749 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:46.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:46.851 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:46.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:46.948 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:46 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/221112 objects degraded (13.614%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:46.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:46.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:46.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:47.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:47.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:47.150 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:46 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/221112 objects degraded (13.614%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:47.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:46 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/221112 objects degraded (13.614%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:47.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:47.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:47.347 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:47.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:47.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:47.496 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:47.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:47.623 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:47.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:47.748 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:47.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:47.872 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:47.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:47.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:47.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:47.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:47.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:47 smithi171 ceph-mon[90031]: pgmap v841: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 30102/220191 objects degraded (13.671%); 13710/220191 objects misplaced (6.226%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:47 smithi027 ceph-mon[93921]: pgmap v841: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 30102/220191 objects degraded (13.671%); 13710/220191 objects misplaced (6.226%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:47 smithi027 ceph-mon[98970]: pgmap v841: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 30102/220191 objects degraded (13.671%); 13710/220191 objects misplaced (6.226%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:48.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:48.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:48.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:48.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:48.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:48.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:48.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:48.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:48.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:48.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:48.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:48.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:48.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:48.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:48.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:48.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:48.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:48.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:48.871 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:48.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:48.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:48.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:48.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:49.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:49.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:49.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:49.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:49.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:49.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:49.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:49.254 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:23:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:23:49] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-08-29T15:23:49.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:49.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:49.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:49.477 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:49.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:49.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:49.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:49.660 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:49.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:49.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:49.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:49.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:49.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:49.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:49.911 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:49.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:49.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:50.012 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:50.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:50.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:50.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:49 smithi027 ceph-mon[98970]: pgmap v842: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/219408 objects degraded (13.720%); 13628/219408 objects misplaced (6.211%); 759 KiB/s, 11 objects/s recovering 2024-08-29T15:23:50.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:50.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:50.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:49 smithi027 ceph-mon[93921]: pgmap v842: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/219408 objects degraded (13.720%); 13628/219408 objects misplaced (6.211%); 759 KiB/s, 11 objects/s recovering 2024-08-29T15:23:50.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:50.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:50.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:50.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:50.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:50.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:49 smithi171 ceph-mon[90031]: pgmap v842: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 30102/219408 objects degraded (13.720%); 13628/219408 objects misplaced (6.211%); 759 KiB/s, 11 objects/s recovering 2024-08-29T15:23:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:50.285 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:50.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:50.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:50.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:50.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:50.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:50.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:50.582 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:23:50.582 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:23:50.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:50.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:50.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:50.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:50.797 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:50.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:50.856 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:23:50.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:50.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:50.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:50.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:51.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:51.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:51.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:51.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:51.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3695287245' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:51.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:51.300 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3695287245' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:51.301 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3695287245' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:23:51.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:51.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:51.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:51.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:51.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:51.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:51.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:51.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:51.673 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:51.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.799 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:51.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:51.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:51.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:51.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:51.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:52.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:52.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[98970]: pgmap v843: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/218961 objects degraded (13.748%); 13628/218961 objects misplaced (6.224%); 501 KiB/s, 7 objects/s recovering 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3695287245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[98970]: osdmap e260: 8 total, 8 up, 8 in 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/218961 objects degraded (13.748%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[93921]: pgmap v843: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/218961 objects degraded (13.748%); 13628/218961 objects misplaced (6.224%); 501 KiB/s, 7 objects/s recovering 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3695287245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:52.058 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[93921]: osdmap e260: 8 total, 8 up, 8 in 2024-08-29T15:23:52.058 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:51 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/218961 objects degraded (13.748%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:52.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:52.157 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:23:52.158 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:23:52.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:51 smithi171 ceph-mon[90031]: pgmap v843: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/218961 objects degraded (13.748%); 13628/218961 objects misplaced (6.224%); 501 KiB/s, 7 objects/s recovering 2024-08-29T15:23:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:51 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:23:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3695287245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:23:52.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:51 smithi171 ceph-mon[90031]: osdmap e260: 8 total, 8 up, 8 in 2024-08-29T15:23:52.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:51 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/218961 objects degraded (13.748%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:52.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:52.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:52.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:52.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:23:52.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:52.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:52.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:52.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:52.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:52.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:52.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:52.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:52.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:52.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:52.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:52.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:52.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:52.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:52.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:53.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:53.086 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:53.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:53.109 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:52 smithi027 ceph-mon[98970]: from='client.55468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:53.110 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:52 smithi027 ceph-mon[93921]: from='client.55468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:53.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:53.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:52 smithi171 ceph-mon[90031]: from='client.55468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:23:53.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:53.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:53.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:53.348 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":844,"stamp":"2024-08-29T15:23:50.858729+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4780464125,"num_objects":72900,"num_object_clones":0,"num_object_copies":218700,"num_objects_missing_on_primary":5584,"num_objects_missing":5584,"num_objects_degraded":30102,"num_objects_misplaced":13588,"num_objects_unfound":0,"num_objects_dirty":72900,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":179101,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15882,"num_bytes_recovered":919669965,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125237,"ondisk_log_size":125237,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15830600,"kb_used_data":14001736,"kb_used_omap":43,"kb_used_meta":1828756,"kb_avail":734130616,"statfs":{"total":767960285184,"available":751749750784,"internally_reserved":0,"allocated":14337777664,"data_stored":14326051534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1872646636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-95485952,"num_objects":-1457,"num_object_clones":0,"num_object_copies":-4371,"num_objects_missing_on_primary":-93,"num_objects_missing":-93,"num_objects_degraded":0,"num_objects_misplaced":-306,"num_objects_unfound":0,"num_objects_dirty":-1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1457,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":6160384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.264371"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":403,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.711548+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:23:36.711548+0000","last_peered":"2024-08-29T15:23:36.711548+0000","last_clean":"2024-08-29T15:23:36.711548+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:23:36.711548+0000","last_undegraded":"2024-08-29T15:23:36.711548+0000","last_fullsized":"2024-08-29T15:23:36.711548+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":36,"reported_epoch":258,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:27.244608+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:23:27.244608+0000","last_peered":"2024-08-29T15:23:27.244608+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:23:27.244608+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:23:27.244608+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":126,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244607+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:23:27.244607+0000","last_peered":"2024-08-29T15:23:27.244607+0000","last_clean":"2024-08-29T15:23:27.244607+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:23:27.244607+0000","last_undegraded":"2024-08-29T15:23:27.244607+0000","last_fullsized":"2024-08-29T15:23:27.244607+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":369,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715341+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:23:36.715341+0000","last_peered":"2024-08-29T15:23:36.715341+0000","last_clean":"2024-08-29T15:23:36.715341+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:23:36.715341+0000","last_undegraded":"2024-08-29T15:23:36.715341+0000","last_fullsized":"2024-08-29T15:23:36.715341+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":368,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715579+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:23:36.715579+0000","last_peered":"2024-08-29T15:23:36.715579+0000","last_clean":"2024-08-29T15:23:36.715579+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:23:36.715579+0000","last_undegraded":"2024-08-29T15:23:36.715579+0000","last_fullsized":"2024-08-29T15:23:36.715579+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244326+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:23:27.244326+0000","last_peered":"2024-08-29T15:23:27.244326+0000","last_clean":"2024-08-29T15:23:27.244326+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:23:27.244326+0000","last_undegraded":"2024-08-29T15:23:27.244326+0000","last_fullsized":"2024-08-29T15:23:27.244326+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":180,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714601+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:23:36.714601+0000","last_peered":"2024-08-29T15:23:36.714601+0000","last_clean":"2024-08-29T15:23:36.714601+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:23:36.714601+0000","last_undegraded":"2024-08-29T15:23:36.714601+0000","last_fullsized":"2024-08-29T15:23:36.714601+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":127,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.549783+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:23:27.549783+0000","last_peered":"2024-08-29T15:23:27.549783+0000","last_clean":"2024-08-29T15:23:27.549783+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:23:27.549783+0000","last_undegraded":"2024-08-29T15:23:27.549783+0000","last_fullsized":"2024-08-29T15:23:27.549783+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"259'11257","reported_seq":19294,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:45.834509+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:23:45.834509+0000","last_peered":"2024-08-29T15:23:45.834509+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:23:45.834509+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'1200","ondisk_log_start":"175'1200","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10057,"log_dups_size":0,"ondisk_log_size":10057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070080,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11257,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":1730}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":35,"reported_epoch":258,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:27.245261+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:23:27.245261+0000","last_peered":"2024-08-29T15:23:27.245261+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:23:27.245261+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:23:27.245261+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":144,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244377+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:23:27.244377+0000","last_peered":"2024-08-29T15:23:27.244377+0000","last_clean":"2024-08-29T15:23:27.244377+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:23:27.244377+0000","last_undegraded":"2024-08-29T15:23:27.244377+0000","last_fullsized":"2024-08-29T15:23:27.244377+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"259'11022","reported_seq":11218,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:46.818402+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:23:46.818402+0000","last_peered":"2024-08-29T15:23:46.818402+0000","last_clean":"2024-08-29T15:23:46.818402+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:23:46.818402+0000","last_undegraded":"2024-08-29T15:23:46.818402+0000","last_fullsized":"2024-08-29T15:23:46.818402+0000","mapping_epoch":168,"log_start":"180'4328","ondisk_log_start":"180'4328","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296747008,"num_objects":4528,"num_object_clones":0,"num_object_copies":13584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11022,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":35,"reported_epoch":258,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:27.245608+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:23:27.245608+0000","last_peered":"2024-08-29T15:23:27.245608+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:23:27.245608+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:23:27.245608+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244177+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:23:27.244177+0000","last_peered":"2024-08-29T15:23:27.244177+0000","last_clean":"2024-08-29T15:23:27.244177+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:23:27.244177+0000","last_undegraded":"2024-08-29T15:23:27.244177+0000","last_fullsized":"2024-08-29T15:23:27.244177+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"259'11179","reported_seq":11345,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:46.261480+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:23:46.261480+0000","last_peered":"2024-08-29T15:23:46.261480+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:23:46.261480+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"180'4474","ondisk_log_start":"180'4474","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297074688,"num_objects":4533,"num_object_clones":0,"num_object_copies":13599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11179,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2056}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":185,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715584+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:23:36.715584+0000","last_peered":"2024-08-29T15:23:36.715584+0000","last_clean":"2024-08-29T15:23:36.715584+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:23:36.715584+0000","last_undegraded":"2024-08-29T15:23:36.715584+0000","last_fullsized":"2024-08-29T15:23:36.715584+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.729270+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:23:36.729270+0000","last_peered":"2024-08-29T15:23:36.729270+0000","last_clean":"2024-08-29T15:23:36.729270+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:23:36.729270+0000","last_undegraded":"2024-08-29T15:23:36.729270+0000","last_fullsized":"2024-08-29T15:23:36.729270+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"259'11282","reported_seq":16698,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:48.836048+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:23:48.836048+0000","last_peered":"2024-08-29T15:23:48.836048+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:23:48.836048+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"178'3162","ondisk_log_start":"178'3162","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8120,"log_dups_size":0,"ondisk_log_size":8120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294649856,"num_objects":4496,"num_object_clones":0,"num_object_copies":13488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11282,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4496}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":144,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.245953+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:23:27.245953+0000","last_peered":"2024-08-29T15:23:27.245953+0000","last_clean":"2024-08-29T15:23:27.245953+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:23:27.245953+0000","last_undegraded":"2024-08-29T15:23:27.245953+0000","last_fullsized":"2024-08-29T15:23:27.245953+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":383,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714879+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:23:36.714879+0000","last_peered":"2024-08-29T15:23:36.714879+0000","last_clean":"2024-08-29T15:23:36.714879+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:23:36.714879+0000","last_undegraded":"2024-08-29T15:23:36.714879+0000","last_fullsized":"2024-08-29T15:23:36.714879+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"259'11188","reported_seq":11333,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:49.636185+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:23:49.636185+0000","last_peered":"2024-08-29T15:23:49.636185+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:23:49.636185+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:23:49.636185+0000","mapping_epoch":234,"log_start":"178'3233","ondisk_log_start":"178'3233","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7955,"log_dups_size":0,"ondisk_log_size":7955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303169536,"num_objects":4626,"num_object_clones":0,"num_object_copies":13878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11188,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2149},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":377,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.709960+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:23:36.709960+0000","last_peered":"2024-08-29T15:23:36.709960+0000","last_clean":"2024-08-29T15:23:36.709960+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:23:36.709960+0000","last_undegraded":"2024-08-29T15:23:36.709960+0000","last_fullsized":"2024-08-29T15:23:36.709960+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244844+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:23:27.244844+0000","last_peered":"2024-08-29T15:23:27.244844+0000","last_clean":"2024-08-29T15:23:27.244844+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:23:27.244844+0000","last_undegraded":"2024-08-29T15:23:27.244844+0000","last_fullsized":"2024-08-29T15:23:27.244844+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"259'11181","reported_seq":11394,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:46.154171+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:23:46.154171+0000","last_peered":"2024-08-29T15:23:46.154171+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:23:46.154171+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:23:46.154171+0000","mapping_epoch":234,"log_start":"180'4468","ondisk_log_start":"180'4468","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6713,"log_dups_size":0,"ondisk_log_size":6713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305463296,"num_objects":4661,"num_object_clones":0,"num_object_copies":13983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11181,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2196},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.729478+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:23:36.729478+0000","last_peered":"2024-08-29T15:23:36.729478+0000","last_clean":"2024-08-29T15:23:36.729478+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:23:36.729478+0000","last_undegraded":"2024-08-29T15:23:36.729478+0000","last_fullsized":"2024-08-29T15:23:36.729478+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.245098+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:23:27.245098+0000","last_peered":"2024-08-29T15:23:27.245098+0000","last_clean":"2024-08-29T15:23:27.245098+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:23:27.245098+0000","last_undegraded":"2024-08-29T15:23:27.245098+0000","last_fullsized":"2024-08-29T15:23:27.245098+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"259'11064","reported_seq":11461,"reported_epoch":259,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:23:46.676868+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:23:46.676868+0000","last_peered":"2024-08-29T15:23:46.676868+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:23:46.676868+0000","last_undegraded":"2024-08-29T15:23:46.676868+0000","last_fullsized":"2024-08-29T15:23:46.676868+0000","mapping_epoch":202,"log_start":"180'4367","ondisk_log_start":"180'4367","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301072384,"num_objects":4594,"num_object_clones":0,"num_object_copies":13782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4594,"num_objects_unfound":0,"num_objects_dirty":4594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11064,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4594}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244180+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:23:27.244180+0000","last_peered":"2024-08-29T15:23:27.244180+0000","last_clean":"2024-08-29T15:23:27.244180+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:23:27.244180+0000","last_undegraded":"2024-08-29T15:23:27.244180+0000","last_fullsized":"2024-08-29T15:23:27.244180+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.245788+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:23:27.245788+0000","last_peered":"2024-08-29T15:23:27.245788+0000","last_clean":"2024-08-29T15:23:27.245788+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:23:27.245788+0000","last_undegraded":"2024-08-29T15:23:27.245788+0000","last_fullsized":"2024-08-29T15:23:27.245788+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"259'11185","reported_seq":11324,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:45.601688+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:23:45.601688+0000","last_peered":"2024-08-29T15:23:45.601688+0000","last_clean":"2024-08-29T15:23:45.601688+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:23:45.601688+0000","last_undegraded":"2024-08-29T15:23:45.601688+0000","last_fullsized":"2024-08-29T15:23:45.601688+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":9907,"log_dups_size":0,"ondisk_log_size":9907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299761688,"num_objects":4575,"num_object_clones":0,"num_object_copies":13725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4575,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11185,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":380,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.709486+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:23:36.709486+0000","last_peered":"2024-08-29T15:23:36.709486+0000","last_clean":"2024-08-29T15:23:36.709486+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:23:36.709486+0000","last_undegraded":"2024-08-29T15:23:36.709486+0000","last_fullsized":"2024-08-29T15:23:36.709486+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":373,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714728+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:23:36.714728+0000","last_peered":"2024-08-29T15:23:36.714728+0000","last_clean":"2024-08-29T15:23:36.714728+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:23:36.714728+0000","last_undegraded":"2024-08-29T15:23:36.714728+0000","last_fullsized":"2024-08-29T15:23:36.714728+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"259'10777","reported_seq":10964,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:44.219028+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:23:44.219028+0000","last_peered":"2024-08-29T15:23:44.219028+0000","last_clean":"2024-08-29T15:23:44.219028+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:23:44.219028+0000","last_undegraded":"2024-08-29T15:23:44.219028+0000","last_fullsized":"2024-08-29T15:23:44.219028+0000","mapping_epoch":201,"log_start":"175'700","ondisk_log_start":"175'700","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10077,"log_dups_size":0,"ondisk_log_size":10077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290390016,"num_objects":4431,"num_object_clones":0,"num_object_copies":13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10777,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":471,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715263+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:23:36.715263+0000","last_peered":"2024-08-29T15:23:36.715263+0000","last_clean":"2024-08-29T15:23:36.715263+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:23:36.715263+0000","last_undegraded":"2024-08-29T15:23:36.715263+0000","last_fullsized":"2024-08-29T15:23:36.715263+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":728,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.245084+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:23:27.245084+0000","last_peered":"2024-08-29T15:23:27.245084+0000","last_clean":"2024-08-29T15:23:27.245084+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:23:27.245084+0000","last_undegraded":"2024-08-29T15:23:27.245084+0000","last_fullsized":"2024-08-29T15:23:27.245084+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":236,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.245551+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:23:27.245551+0000","last_peered":"2024-08-29T15:23:27.245551+0000","last_clean":"2024-08-29T15:23:27.245551+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:23:27.245551+0000","last_undegraded":"2024-08-29T15:23:27.245551+0000","last_fullsized":"2024-08-29T15:23:27.245551+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":99,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.551246+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:23:27.551246+0000","last_peered":"2024-08-29T15:23:27.551246+0000","last_clean":"2024-08-29T15:23:27.551246+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:23:27.551246+0000","last_undegraded":"2024-08-29T15:23:27.551246+0000","last_fullsized":"2024-08-29T15:23:27.551246+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"259'11218","reported_seq":11613,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.627324+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:23:50.627324+0000","last_peered":"2024-08-29T15:23:50.627324+0000","last_clean":"2024-08-29T15:23:50.627324+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:23:50.627324+0000","last_undegraded":"2024-08-29T15:23:50.627324+0000","last_fullsized":"2024-08-29T15:23:50.627324+0000","mapping_epoch":201,"log_start":"179'4153","ondisk_log_start":"179'4153","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11218,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":845,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714392+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:23:36.714392+0000","last_peered":"2024-08-29T15:23:36.714392+0000","last_clean":"2024-08-29T15:23:36.714392+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:23:36.714392+0000","last_undegraded":"2024-08-29T15:23:36.714392+0000","last_fullsized":"2024-08-29T15:23:36.714392+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":383,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.716721+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:23:36.716721+0000","last_peered":"2024-08-29T15:23:36.716721+0000","last_clean":"2024-08-29T15:23:36.716721+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:23:36.716721+0000","last_undegraded":"2024-08-29T15:23:36.716721+0000","last_fullsized":"2024-08-29T15:23:36.716721+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":391,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715816+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:23:36.715816+0000","last_peered":"2024-08-29T15:23:36.715816+0000","last_clean":"2024-08-29T15:23:36.715816+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:23:36.715816+0000","last_undegraded":"2024-08-29T15:23:36.715816+0000","last_fullsized":"2024-08-29T15:23:36.715816+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"259'11044","reported_seq":16862,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:46.543955+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:23:46.543955+0000","last_peered":"2024-08-29T15:23:46.543955+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:23:46.543955+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"179'4330","ondisk_log_start":"179'4330","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6714,"log_dups_size":0,"ondisk_log_size":6714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304480256,"num_objects":4646,"num_object_clones":0,"num_object_copies":13938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11044,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":212},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":654,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.709278+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:23:36.709278+0000","last_peered":"2024-08-29T15:23:36.709278+0000","last_clean":"2024-08-29T15:23:36.709278+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:23:36.709278+0000","last_undegraded":"2024-08-29T15:23:36.709278+0000","last_fullsized":"2024-08-29T15:23:36.709278+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.549546+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:23:27.549546+0000","last_peered":"2024-08-29T15:23:27.549546+0000","last_clean":"2024-08-29T15:23:27.549546+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:23:27.549546+0000","last_undegraded":"2024-08-29T15:23:27.549546+0000","last_fullsized":"2024-08-29T15:23:27.549546+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":418,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714450+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:23:36.714450+0000","last_peered":"2024-08-29T15:23:36.714450+0000","last_clean":"2024-08-29T15:23:36.714450+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:23:36.714450+0000","last_undegraded":"2024-08-29T15:23:36.714450+0000","last_fullsized":"2024-08-29T15:23:36.714450+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"259'11121","reported_seq":11327,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:46.537654+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:23:46.537654+0000","last_peered":"2024-08-29T15:23:46.537654+0000","last_clean":"2024-08-29T15:23:46.537654+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:23:46.537654+0000","last_undegraded":"2024-08-29T15:23:46.537654+0000","last_fullsized":"2024-08-29T15:23:46.537654+0000","mapping_epoch":234,"log_start":"179'4397","ondisk_log_start":"179'4397","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300875776,"num_objects":4591,"num_object_clones":0,"num_object_copies":13773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11121,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":196,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244431+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:23:27.244431+0000","last_peered":"2024-08-29T15:23:27.244431+0000","last_clean":"2024-08-29T15:23:27.244431+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:23:27.244431+0000","last_undegraded":"2024-08-29T15:23:27.244431+0000","last_fullsized":"2024-08-29T15:23:27.244431+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244500+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:23:27.244500+0000","last_peered":"2024-08-29T15:23:27.244500+0000","last_clean":"2024-08-29T15:23:27.244500+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:23:27.244500+0000","last_undegraded":"2024-08-29T15:23:27.244500+0000","last_fullsized":"2024-08-29T15:23:27.244500+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":103,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.245544+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:23:27.245544+0000","last_peered":"2024-08-29T15:23:27.245544+0000","last_clean":"2024-08-29T15:23:27.245544+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:23:27.245544+0000","last_undegraded":"2024-08-29T15:23:27.245544+0000","last_fullsized":"2024-08-29T15:23:27.245544+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"259'11192","reported_seq":11397,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:48.501409+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:23:48.501409+0000","last_peered":"2024-08-29T15:23:48.501409+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:23:48.501409+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"178'3042","ondisk_log_start":"178'3042","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8150,"log_dups_size":0,"ondisk_log_size":8150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303693824,"num_objects":4634,"num_object_clones":0,"num_object_copies":13902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11192,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2211}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":128,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244539+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:23:27.244539+0000","last_peered":"2024-08-29T15:23:27.244539+0000","last_clean":"2024-08-29T15:23:27.244539+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:23:27.244539+0000","last_undegraded":"2024-08-29T15:23:27.244539+0000","last_fullsized":"2024-08-29T15:23:27.244539+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":810,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715789+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:23:36.715789+0000","last_peered":"2024-08-29T15:23:36.715789+0000","last_clean":"2024-08-29T15:23:36.715789+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:23:36.715789+0000","last_undegraded":"2024-08-29T15:23:36.715789+0000","last_fullsized":"2024-08-29T15:23:36.715789+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":387,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714933+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:23:36.714933+0000","last_peered":"2024-08-29T15:23:36.714933+0000","last_clean":"2024-08-29T15:23:36.714933+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:23:36.714933+0000","last_undegraded":"2024-08-29T15:23:36.714933+0000","last_fullsized":"2024-08-29T15:23:36.714933+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"259'11172","reported_seq":11933,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:50.551892+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:23:50.551892+0000","last_peered":"2024-08-29T15:23:50.551892+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:23:50.551892+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"179'4180","ondisk_log_start":"179'4180","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294649856,"num_objects":4496,"num_object_clones":0,"num_object_copies":13488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4467,"num_objects_unfound":0,"num_objects_dirty":4496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11172,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2345}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":392,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715624+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:23:36.715624+0000","last_peered":"2024-08-29T15:23:36.715624+0000","last_clean":"2024-08-29T15:23:36.715624+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:23:36.715624+0000","last_undegraded":"2024-08-29T15:23:36.715624+0000","last_fullsized":"2024-08-29T15:23:36.715624+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":686,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:36.746971+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:23:36.746971+0000","last_peered":"2024-08-29T15:23:36.746971+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:23:36.746971+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:23:36.746971+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":383,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714783+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:23:36.714783+0000","last_peered":"2024-08-29T15:23:36.714783+0000","last_clean":"2024-08-29T15:23:36.714783+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:23:36.714783+0000","last_undegraded":"2024-08-29T15:23:36.714783+0000","last_fullsized":"2024-08-29T15:23:36.714783+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"259'11027","reported_seq":18170,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:49.692575+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:23:49.692575+0000","last_peered":"2024-08-29T15:23:49.692575+0000","last_clean":"2024-08-29T15:23:49.692575+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:23:49.692575+0000","last_undegraded":"2024-08-29T15:23:49.692575+0000","last_fullsized":"2024-08-29T15:23:49.692575+0000","mapping_epoch":234,"log_start":"178'3099","ondisk_log_start":"178'3099","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7928,"log_dups_size":0,"ondisk_log_size":7928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288423936,"num_objects":4401,"num_object_clones":0,"num_object_copies":13203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11027,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":377,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.713316+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:23:36.713316+0000","last_peered":"2024-08-29T15:23:36.713316+0000","last_clean":"2024-08-29T15:23:36.713316+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:23:36.713316+0000","last_undegraded":"2024-08-29T15:23:36.713316+0000","last_fullsized":"2024-08-29T15:23:36.713316+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1000,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:36.714074+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:23:36.714074+0000","last_peered":"2024-08-29T15:23:36.714074+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:23:36.714074+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":375,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.709103+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:23:36.709103+0000","last_peered":"2024-08-29T15:23:36.709103+0000","last_clean":"2024-08-29T15:23:36.709103+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:23:36.709103+0000","last_undegraded":"2024-08-29T15:23:36.709103+0000","last_fullsized":"2024-08-29T15:23:36.709103+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"259'11069","reported_seq":15519,"reported_epoch":259,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:46.997257+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:23:46.997257+0000","last_peered":"2024-08-29T15:23:46.997257+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:23:46.997257+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"178'3512","ondisk_log_start":"178'3512","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297598976,"num_objects":4541,"num_object_clones":0,"num_object_copies":13623,"num_objects_missing_on_primary":5576,"num_objects_missing":5576,"num_objects_degraded":2313,"num_objects_misplaced":4527,"num_objects_unfound":0,"num_objects_dirty":4541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11069,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2146,"num_bytes_recovered":140509184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":2156},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":373,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714503+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:23:36.714503+0000","last_peered":"2024-08-29T15:23:36.714503+0000","last_clean":"2024-08-29T15:23:36.714503+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:23:36.714503+0000","last_undegraded":"2024-08-29T15:23:36.714503+0000","last_fullsized":"2024-08-29T15:23:36.714503+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":110,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.551004+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:23:27.551004+0000","last_peered":"2024-08-29T15:23:27.551004+0000","last_clean":"2024-08-29T15:23:27.551004+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:23:27.551004+0000","last_undegraded":"2024-08-29T15:23:27.551004+0000","last_fullsized":"2024-08-29T15:23:27.551004+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244251+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:23:27.244251+0000","last_peered":"2024-08-29T15:23:27.244251+0000","last_clean":"2024-08-29T15:23:27.244251+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:23:27.244251+0000","last_undegraded":"2024-08-29T15:23:27.244251+0000","last_fullsized":"2024-08-29T15:23:27.244251+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244618+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:23:27.244618+0000","last_peered":"2024-08-29T15:23:27.244618+0000","last_clean":"2024-08-29T15:23:27.244618+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:23:27.244618+0000","last_undegraded":"2024-08-29T15:23:27.244618+0000","last_fullsized":"2024-08-29T15:23:27.244618+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":35,"reported_epoch":258,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:27.244903+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:23:27.244903+0000","last_peered":"2024-08-29T15:23:27.244903+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:23:27.244903+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:23:27.244903+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":381,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714766+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:23:36.714766+0000","last_peered":"2024-08-29T15:23:36.714766+0000","last_clean":"2024-08-29T15:23:36.714766+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:23:36.714766+0000","last_undegraded":"2024-08-29T15:23:36.714766+0000","last_fullsized":"2024-08-29T15:23:36.714766+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":132,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:36.710242+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:23:36.710242+0000","last_peered":"2024-08-29T15:23:36.710242+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:23:36.710242+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:23:36.710242+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":126,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244932+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:23:27.244932+0000","last_peered":"2024-08-29T15:23:27.244932+0000","last_clean":"2024-08-29T15:23:27.244932+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:23:27.244932+0000","last_undegraded":"2024-08-29T15:23:27.244932+0000","last_fullsized":"2024-08-29T15:23:27.244932+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":341,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.709010+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:23:36.709010+0000","last_peered":"2024-08-29T15:23:36.709010+0000","last_clean":"2024-08-29T15:23:36.709010+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:23:36.709010+0000","last_undegraded":"2024-08-29T15:23:36.709010+0000","last_fullsized":"2024-08-29T15:23:36.709010+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":394,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715310+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:23:36.715310+0000","last_peered":"2024-08-29T15:23:36.715310+0000","last_clean":"2024-08-29T15:23:36.715310+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:23:36.715310+0000","last_undegraded":"2024-08-29T15:23:36.715310+0000","last_fullsized":"2024-08-29T15:23:36.715310+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":393,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.709055+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:23:36.709055+0000","last_peered":"2024-08-29T15:23:36.709055+0000","last_clean":"2024-08-29T15:23:36.709055+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:23:36.709055+0000","last_undegraded":"2024-08-29T15:23:36.709055+0000","last_fullsized":"2024-08-29T15:23:36.709055+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.551041+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:23:27.551041+0000","last_peered":"2024-08-29T15:23:27.551041+0000","last_clean":"2024-08-29T15:23:27.551041+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:23:27.551041+0000","last_undegraded":"2024-08-29T15:23:27.551041+0000","last_fullsized":"2024-08-29T15:23:27.551041+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":250,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715213+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:23:36.715213+0000","last_peered":"2024-08-29T15:23:36.715213+0000","last_clean":"2024-08-29T15:23:36.715213+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:23:36.715213+0000","last_undegraded":"2024-08-29T15:23:36.715213+0000","last_fullsized":"2024-08-29T15:23:36.715213+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":370,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715734+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:23:36.715734+0000","last_peered":"2024-08-29T15:23:36.715734+0000","last_clean":"2024-08-29T15:23:36.715734+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:23:36.715734+0000","last_undegraded":"2024-08-29T15:23:36.715734+0000","last_fullsized":"2024-08-29T15:23:36.715734+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":373,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.717080+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:23:36.717080+0000","last_peered":"2024-08-29T15:23:36.717080+0000","last_clean":"2024-08-29T15:23:36.717080+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:23:36.717080+0000","last_undegraded":"2024-08-29T15:23:36.717080+0000","last_fullsized":"2024-08-29T15:23:36.717080+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":377,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.710122+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:23:36.710122+0000","last_peered":"2024-08-29T15:23:36.710122+0000","last_clean":"2024-08-29T15:23:36.710122+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:23:36.710122+0000","last_undegraded":"2024-08-29T15:23:36.710122+0000","last_fullsized":"2024-08-29T15:23:36.710122+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244377+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:23:27.244377+0000","last_peered":"2024-08-29T15:23:27.244377+0000","last_clean":"2024-08-29T15:23:27.244377+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:23:27.244377+0000","last_undegraded":"2024-08-29T15:23:27.244377+0000","last_fullsized":"2024-08-29T15:23:27.244377+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":397,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715912+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:23:36.715912+0000","last_peered":"2024-08-29T15:23:36.715912+0000","last_clean":"2024-08-29T15:23:36.715912+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:23:36.715912+0000","last_undegraded":"2024-08-29T15:23:36.715912+0000","last_fullsized":"2024-08-29T15:23:36.715912+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":361,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.714636+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:23:36.714636+0000","last_peered":"2024-08-29T15:23:36.714636+0000","last_clean":"2024-08-29T15:23:36.714636+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:23:36.714636+0000","last_undegraded":"2024-08-29T15:23:36.714636+0000","last_fullsized":"2024-08-29T15:23:36.714636+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244428+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:23:27.244428+0000","last_peered":"2024-08-29T15:23:27.244428+0000","last_clean":"2024-08-29T15:23:27.244428+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:23:27.244428+0000","last_undegraded":"2024-08-29T15:23:27.244428+0000","last_fullsized":"2024-08-29T15:23:27.244428+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":150,"reported_epoch":258,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:27.245711+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:23:27.245711+0000","last_peered":"2024-08-29T15:23:27.245711+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:23:27.245711+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:23:27.245711+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244447+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:23:27.244447+0000","last_peered":"2024-08-29T15:23:27.244447+0000","last_clean":"2024-08-29T15:23:27.244447+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:23:27.244447+0000","last_undegraded":"2024-08-29T15:23:27.244447+0000","last_fullsized":"2024-08-29T15:23:27.244447+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":156,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244683+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:23:27.244683+0000","last_peered":"2024-08-29T15:23:27.244683+0000","last_clean":"2024-08-29T15:23:27.244683+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:23:27.244683+0000","last_undegraded":"2024-08-29T15:23:27.244683+0000","last_fullsized":"2024-08-29T15:23:27.244683+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":409,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.709403+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:23:36.709403+0000","last_peered":"2024-08-29T15:23:36.709403+0000","last_clean":"2024-08-29T15:23:36.709403+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:23:36.709403+0000","last_undegraded":"2024-08-29T15:23:36.709403+0000","last_fullsized":"2024-08-29T15:23:36.709403+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":410,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.711110+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:23:36.711110+0000","last_peered":"2024-08-29T15:23:36.711110+0000","last_clean":"2024-08-29T15:23:36.711110+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:23:36.711110+0000","last_undegraded":"2024-08-29T15:23:36.711110+0000","last_fullsized":"2024-08-29T15:23:36.711110+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":37,"reported_epoch":258,"state":"active+clean","last_fresh":"2024-08-29T15:23:27.244068+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:23:27.244068+0000","last_peered":"2024-08-29T15:23:27.244068+0000","last_clean":"2024-08-29T15:23:27.244068+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:23:27.244068+0000","last_undegraded":"2024-08-29T15:23:27.244068+0000","last_fullsized":"2024-08-29T15:23:27.244068+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":417,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.715152+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:23:36.715152+0000","last_peered":"2024-08-29T15:23:36.715152+0000","last_clean":"2024-08-29T15:23:36.715152+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:23:36.715152+0000","last_undegraded":"2024-08-29T15:23:36.715152+0000","last_fullsized":"2024-08-29T15:23:36.715152+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4775673880,"num_objects":72872,"num_object_clones":0,"num_object_copies":218616,"num_objects_missing_on_primary":5576,"num_objects_missing":5576,"num_objects_degraded":30073,"num_objects_misplaced":13588,"num_objects_unfound":0,"num_objects_dirty":72872,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":177978,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15773,"num_bytes_recovered":913702912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14278602752,"data_stored":14278590536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124055,"ondisk_log_size":124055,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234350,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2631880,"kb_used_data":2334144,"kb_used_omap":1,"kb_used_meta":297726,"kb_avail":91113272,"statfs":{"total":95995035648,"available":93299990528,"internally_reserved":0,"allocated":2390163456,"data_stored":2388686043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":304871883},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495981,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":309892,"kb_used_data":134396,"kb_used_omap":1,"kb_used_meta":175486,"kb_avail":93435260,"statfs":{"total":95995035648,"available":95677706240,"internally_reserved":0,"allocated":137621504,"data_stored":136147891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":179698119},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426567,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1511992,"kb_used_data":1460976,"kb_used_omap":8,"kb_used_meta":50999,"kb_avail":92233160,"statfs":{"total":95995035648,"available":94446755840,"internally_reserved":0,"allocated":1496039424,"data_stored":1494573277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":52223882},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019260,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1774440,"kb_used_data":1454560,"kb_used_omap":1,"kb_used_meta":319870,"kb_avail":91970712,"statfs":{"total":95995035648,"available":94178009088,"internally_reserved":0,"allocated":1489469440,"data_stored":1488019076,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":327547334},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435444,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3209908,"kb_used_data":2921452,"kb_used_omap":4,"kb_used_meta":288443,"kb_avail":90535244,"statfs":{"total":95995035648,"available":92708089856,"internally_reserved":0,"allocated":2991566848,"data_stored":2990104641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":295365897},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903296,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2740408,"kb_used_data":2635056,"kb_used_omap":10,"kb_used_meta":105333,"kb_avail":91004744,"statfs":{"total":95995035648,"available":93188857856,"internally_reserved":0,"allocated":2698297344,"data_stored":2696835073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":107861520},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762498,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2150132,"kb_used_data":1879660,"kb_used_omap":7,"kb_used_meta":270456,"kb_avail":91595020,"statfs":{"total":95995035648,"available":93793300480,"internally_reserved":0,"allocated":1924771840,"data_stored":1923308393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":276947298},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926024,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1501948,"kb_used_data":1181492,"kb_used_omap":7,"kb_used_meta":320440,"kb_avail":92243204,"statfs":{"total":95995035648,"available":94457040896,"internally_reserved":0,"allocated":1209847808,"data_stored":1208377140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":328130703},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1200885760,"data_stored":1200881688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1916014592,"data_stored":1916010520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2693464064,"data_stored":2693464064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2985885696,"data_stored":2985885696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1482489856,"data_stored":1482489856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1492389888,"data_stored":1492385816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":133300224,"data_stored":133300224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2374172672,"data_stored":2374172672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:23:53.417 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:23:53.418 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:23:53.418 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:23:53.418 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:23:53.418 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:23:53.418 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:23:53.418 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:23:53.418 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:23:53.419 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:23:53.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:53.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:53.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:53.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:53.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:53.671 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:53.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:53.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:53.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:23:53.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:53.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:53.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:53.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:53.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:53.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:54.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:53 smithi027 ceph-mon[93921]: pgmap v845: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/217467 objects degraded (13.842%); 13511/217467 objects misplaced (6.213%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/864540297' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:53 smithi027 ceph-mon[98970]: pgmap v845: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/217467 objects degraded (13.842%); 13511/217467 objects misplaced (6.213%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:54.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/864540297' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:54.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:54.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:54.203 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:54.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:53 smithi171 ceph-mon[90031]: pgmap v845: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 30102/217467 objects degraded (13.842%); 13511/217467 objects misplaced (6.213%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/864540297' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:23:54.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:54.329 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:54.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:54.431 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:54.454 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:54.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:54.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:54.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:54.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:54.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:54.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.833 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:54.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:54.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:54.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:54.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:54.974 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:55.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:55.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:55.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:55.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:55.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:55.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:55.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:55.435 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:55.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:55.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:55.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:55.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:55.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:55.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:55.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:55.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:55.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:55.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:55.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:55.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:55.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:55.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:56.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:55 smithi027 ceph-mon[98970]: pgmap v846: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/217236 objects degraded (13.857%); 13511/217236 objects misplaced (6.220%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:55 smithi027 ceph-mon[93921]: pgmap v846: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/217236 objects degraded (13.857%); 13511/217236 objects misplaced (6.220%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:56.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:56.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:56.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:56.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:23:56.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:55 smithi171 ceph-mon[90031]: pgmap v846: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/217236 objects degraded (13.857%); 13511/217236 objects misplaced (6.220%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:56.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:56.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:56.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:23:56.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:56.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:56.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:56.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:56.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:56.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:56.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:56.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:56.812 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:56.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:56.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:56.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:56.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:56.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:57.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:57.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:56 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/217236 objects degraded (13.857%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:57.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:56 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/217236 objects degraded (13.857%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:57.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:57.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:57.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:57.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:57.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:57.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:56 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/217236 objects degraded (13.857%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:23:57.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:57.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:57.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:57.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:57.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:57.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:57.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239396 5.3 5:c28abb26:::benchmark_data_smithi027_135490_object105644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 591.58990708900001, 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4671054730000002, 2024-08-29T15:23:57.484 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239396 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742976+0000", 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742978+0000", 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.674e-06 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742984+0000", 2024-08-29T15:23:57.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7530000000000004e-06 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.742995+0000", 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0967e-05 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.743005+0000", 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4690000000000003e-06 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.751912+0000", 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008907491 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.751924+0000", 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1246999999999999e-05 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692438+0000", 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94051403300000003 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692450+0000", 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.269e-05 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.176340+0000", 2024-08-29T15:23:57.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.483889434 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.176472+0000", 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013255499999999999 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.177116+0000", 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064412200000000001 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.201804+0000", 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024687298999999999 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.201859+0000", 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5201000000000002e-05 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.206234+0000", 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043748629999999997 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.209955+0000", 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037214819999999999 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.209993+0000", 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7898999999999997e-05 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210081+0000", 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8294000000000005e-05 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239400 5.3 5:c91c85c5:::benchmark_data_smithi027_135490_object105648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 591.57453662700004, 2024-08-29T15:23:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4617042850000002, 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239400 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:23:57.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758346+0000", 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758348+0000", 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4130000000000001e-06 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758354+0000", 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0510000000000002e-06 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758382+0000", 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7864999999999998e-05 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758391+0000", 2024-08-29T15:23:57.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2390000000000008e-06 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758468+0000", 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7152999999999998e-05 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:05.758481+0000", 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2927e-05 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692477+0000", 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93399591900000001 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:06.692485+0000", 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6340000000000001e-06 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210333+0000", 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5178479149999999 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210442+0000", 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108992 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.210625+0000", 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018322499999999999 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219736+0000", 2024-08-29T15:23:57.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0091110099999999992 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219805+0000", 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8861000000000004e-05 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219816+0000", 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1392999999999999e-05 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219858+0000", 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2416000000000002e-05 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.219940+0000", 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1883000000000002e-05 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220051+0000", 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110389 2024-08-29T15:23:57.501 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239422 5.3 5:cc7358ac:::benchmark_data_smithi027_135490_object105670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 589.31311194299997, 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21943114, 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.502 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239422 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019771+0000", 2024-08-29T15:23:57.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019773+0000", 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2479999999999999e-06 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019782+0000", 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0650000000000005e-06 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019802+0000", 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9653999999999999e-05 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019811+0000", 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4369999999999996e-06 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019893+0000", 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1891999999999995e-05 2024-08-29T15:23:57.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.019903+0000", 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0003999999999999e-05 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220071+0000", 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20016803999999999 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220151+0000", 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9785999999999998e-05 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.220220+0000", 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8520000000000001e-05 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228770+0000", 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085506139999999998 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228784+0000", 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3207000000000001e-05 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228816+0000", 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2806999999999999e-05 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239075+0000", 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010258751 2024-08-29T15:23:57.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239111+0000", 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5917999999999997e-05 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239202+0000", 2024-08-29T15:23:57.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1197000000000001e-05 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239427 5.3 5:cac43ec6:::benchmark_data_smithi027_135490_object105675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 589.29015789499999, 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.197921239, 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.510 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239427 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042725+0000", 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042726+0000", 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.215e-06 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042730+0000", 2024-08-29T15:23:57.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9469999999999996e-06 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042741+0000", 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0995999999999999e-05 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042748+0000", 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.652e-06 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042842+0000", 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4467000000000002e-05 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.042852+0000", 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9080000000000007e-06 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.223936+0000", 2024-08-29T15:23:57.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18108415 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.224043+0000", 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106591 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.224167+0000", 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012385599999999999 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228861+0000", 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046937619999999998 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228899+0000", 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8325000000000003e-05 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.239219+0000", 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010320424999999999 2024-08-29T15:23:57.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240479+0000", 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001259825 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240537+0000", 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7386000000000002e-05 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240646+0000", 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109734 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44573.0:239430 5.3 5:c7a58ecd:::benchmark_data_smithi027_135490_object105678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 589.27979085699997, 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18776626399999999, 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.518 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44573", 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/660234170", 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 239430 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053092+0000", 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053094+0000", 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4100000000000001e-06 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053100+0000", 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2979999999999997e-06 2024-08-29T15:23:57.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053111+0000", 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0664e-05 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053117+0000", 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7479999999999996e-06 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053200+0000", 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2795999999999998e-05 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.053210+0000", 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0399e-05 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.228935+0000", 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.175724134 2024-08-29T15:23:57.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.229054+0000", 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119704 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.229207+0000", 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015269400000000001 2024-08-29T15:23:57.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240172+0000", 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010965088 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240433+0000", 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026095699999999998 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240446+0000", 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3414000000000001e-05 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240699+0000", 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025234500000000001 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240748+0000", 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9118e-05 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:14:08.240858+0000", 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011049500000000001 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140284 6.a 6.98e9132a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 396.15931774299997, 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59515916599999996, 2024-08-29T15:23:57.527 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140284 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173567+0000", 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4729999999999999e-06 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173572+0000", 2024-08-29T15:23:57.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7270000000000004e-06 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173590+0000", 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7442999999999999e-05 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173598+0000", 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2830000000000001e-06 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173700+0000", 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101889 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173709+0000", 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4749999999999993e-06 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768713+0000", 2024-08-29T15:23:57.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59500392899999999 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768724+0000", 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1946999999999999e-05 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140312 6.a 6.b6852f8a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 396.14779944399999, 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58367630699999995, 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140312 2024-08-29T15:23:57.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5400000000000002e-07 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185087+0000", 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1209999999999998e-06 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185095+0000", 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0530000000000007e-06 2024-08-29T15:23:57.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185101+0000", 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0349999999999998e-06 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185153+0000", 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1468999999999998e-05 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185161+0000", 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9549999999999992e-06 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768751+0000", 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58359058799999997 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768760+0000", 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6319999999999997e-06 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.537 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140319 6.a 6.5ae9c7ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 396.13541129100003, 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57131815699999999, 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.538 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140319 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197473+0000", 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2270000000000001e-06 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197513+0000", 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9731000000000003e-05 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197529+0000", 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6337e-05 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197537+0000", 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1890000000000007e-06 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197704+0000", 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016718599999999999 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197716+0000", 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1282e-05 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768782+0000", 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57106648599999998 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768790+0000", 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7190000000000001e-06 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140323 6.a 6.def6e63a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 396.13429041099999, 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57023369499999998, 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.543 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140323 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7599999999999996e-07 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198597+0000", 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9940000000000002e-06 2024-08-29T15:23:57.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198611+0000", 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3522e-05 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198617+0000", 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4880000000000004e-06 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198665+0000", 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8040000000000001e-05 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198673+0000", 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1400000000000002e-06 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768811+0000", 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57013830200000004 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768826+0000", 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5432999999999999e-05 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140331 6.a 6.ecf214ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 396.11938166599998, 2024-08-29T15:23:57.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56336548399999997, 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140331 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213503+0000", 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.091e-06 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213510+0000", 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.861e-06 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213531+0000", 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0811000000000001e-05 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213538+0000", 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2869999999999999e-06 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213591+0000", 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2735999999999998e-05 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213598+0000", 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4330000000000001e-06 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776857+0000", 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56325978499999996 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776867+0000", 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4800000000000007e-06 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140333 6.a 6.7707feaa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:23:57.553 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 396.11750779300002, 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56152404199999995, 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140333 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215377+0000", 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7320000000000001e-06 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215385+0000", 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6839999999999996e-06 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215402+0000", 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7489000000000001e-05 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215411+0000", 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7450000000000001e-06 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215492+0000", 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0819e-05 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215501+0000", 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6369999999999994e-06 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776893+0000", 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56139153100000005 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776899+0000", 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4049999999999997e-06 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.558 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140336 6.a 6.ee7969ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 396.11444317799999, 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.55869965600000004, 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140336 2024-08-29T15:23:57.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218442+0000", 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8250000000000001e-06 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218450+0000", 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0690000000000002e-06 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218472+0000", 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2178e-05 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218482+0000", 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0071999999999999e-05 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218555+0000", 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3465000000000002e-05 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218568+0000", 2024-08-29T15:23:57.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2772000000000001e-05 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.777129+0000", 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.55856027900000005 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.777140+0000", 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0995999999999999e-05 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:159450 6.9 6:9b2ff3c4:::benchmark_data_smithi027_169445_object34023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:21:28.266945+0000", 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 149.06593756699999, 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014894456, 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 159450 2024-08-29T15:23:57.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.266945+0000", 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.266945+0000", 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.266947+0000", 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6780000000000001e-06 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.266954+0000", 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1919999999999999e-06 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.266973+0000", 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8236999999999998e-05 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.266980+0000", 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.323e-06 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.267051+0000", 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1524000000000002e-05 2024-08-29T15:23:57.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.267185+0000", 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013407 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.267309+0000", 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000123674 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.271151+0000", 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003841541 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.271741+0000", 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058994500000000003 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.281580+0000", 2024-08-29T15:23:57.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098397599999999995 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.281636+0000", 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5772000000000001e-05 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.281840+0000", 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020374 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:162790 6.9 6:99b07a3c:::benchmark_data_smithi027_169445_object37363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e249)", 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:21:54.420441+0000", 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.912442218, 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015921615, 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 162790 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420441+0000", 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420441+0000", 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420443+0000", 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.821e-06 2024-08-29T15:23:57.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420450+0000", 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2470000000000002e-06 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420467+0000", 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7377999999999999e-05 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420475+0000", 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4619999999999999e-06 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420544+0000", 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8861000000000004e-05 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420705+0000", 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161044 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.420887+0000", 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018264400000000001 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.424815+0000", 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039281369999999999 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.424943+0000", 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012718699999999999 2024-08-29T15:23:57.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.436158+0000", 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011215397 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.436194+0000", 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5849000000000002e-05 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.436362+0000", 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016858800000000001 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:164618 6.0 6:0329f014:::benchmark_data_smithi027_169445_object39191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:22:09.352084+0000", 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 107.980798605, 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014398136000000001, 2024-08-29T15:23:57.577 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 164618 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352084+0000", 2024-08-29T15:23:57.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352084+0000", 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352086+0000", 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3680000000000001e-06 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352092+0000", 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6100000000000002e-06 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352107+0000", 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4729000000000001e-05 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352116+0000", 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9709999999999993e-06 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352178+0000", 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1499000000000002e-05 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352333+0000", 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000155253 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.352502+0000", 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016922899999999999 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.359347+0000", 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068451689999999999 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.361155+0000", 2024-08-29T15:23:57.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001807819 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.365012+0000", 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003856707 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.365058+0000", 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6113999999999999e-05 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:09.366483+0000", 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001424668 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:165228 6.0 6:04c926f6:::benchmark_data_smithi027_169445_object39801:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:22:14.278791+0000", 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 103.05409162799999, 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017361237000000002, 2024-08-29T15:23:57.584 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 165228 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.278791+0000", 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.278791+0000", 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.278792+0000", 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.879999999999999e-07 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.278797+0000", 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3590000000000001e-06 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.278809+0000", 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1858000000000001e-05 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.278814+0000", 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4519999999999996e-06 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.281649+0000", 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028354019999999999 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.281767+0000", 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011765300000000001 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.281864+0000", 2024-08-29T15:23:57.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7079999999999999e-05 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.286537+0000", 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046728150000000003 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.286577+0000", 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0204999999999999e-05 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.294458+0000", 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0078809250000000004 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.294520+0000", 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1679000000000004e-05 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:14.296153+0000", 2024-08-29T15:23:57.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001632821 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:165469 6.0 6:065aba91:::benchmark_data_smithi027_169445_object40042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:22:15.742868+0000", 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 101.590014876, 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014050082, 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.591 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 165469 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.742868+0000", 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.742868+0000", 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.742869+0000", 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.263e-06 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.742874+0000", 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6179999999999997e-06 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.742885+0000", 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.113e-05 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.742890+0000", 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1340000000000003e-06 2024-08-29T15:23:57.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.743960+0000", 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001069422 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.744095+0000", 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135764 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:23:57.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.744233+0000", 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137317 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.751977+0000", 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077438580000000002 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.752694+0000", 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00071744400000000002 2024-08-29T15:23:57.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.755537+0000", 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028431749999999999 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.755578+0000", 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0429999999999997e-05 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:15.756918+0000", 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001340527 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:168429 6.0 6:0f6c2352:::benchmark_data_smithi027_169445_object43002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e253)", 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:22:38.672187+0000", 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 78.660695622000006, 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015849767000000001, 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.598 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 168429 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672187+0000", 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672187+0000", 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672189+0000", 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9649999999999998e-06 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672196+0000", 2024-08-29T15:23:57.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7460000000000003e-06 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672212+0000", 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6353000000000001e-05 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672222+0000", 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8120000000000002e-06 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672294+0000", 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1447999999999998e-05 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672399+0000", 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105697 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.672511+0000", 2024-08-29T15:23:57.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111362 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.676742+0000", 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004230892 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.678940+0000", 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021979149999999999 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.686427+0000", 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074872280000000003 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.686503+0000", 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5842999999999999e-05 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:22:38.688037+0000", 2024-08-29T15:23:57.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0015345059999999999 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:173514 6.0 6:02027ba4:::benchmark_data_smithi027_169445_object48087:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:23:15.182971+0000", 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 42.149912245000003, 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015345442000000001, 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.605 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 173514 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.182971+0000", 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.182971+0000", 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.182972+0000", 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7700000000000004e-07 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.182976+0000", 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2180000000000001e-06 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.182994+0000", 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8022000000000001e-05 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.183002+0000", 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4530000000000002e-06 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.186600+0000", 2024-08-29T15:23:57.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035981059999999998 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.186769+0000", 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016879899999999999 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.186956+0000", 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000186758 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.194851+0000", 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0078951639999999997 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.194913+0000", 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2086000000000002e-05 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.197072+0000", 2024-08-29T15:23:57.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021592400000000002 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.197120+0000", 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7799000000000001e-05 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.198316+0000", 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011960199999999999 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:173515 6.0 6:0ef5ad2f:::benchmark_data_smithi027_169445_object48088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:23:15.183261+0000", 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 42.149621801000002, 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01600975, 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 173515 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.612 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.183261+0000", 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.183261+0000", 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.183262+0000", 2024-08-29T15:23:57.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2500000000000005e-07 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.183266+0000", 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9489999999999998e-06 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.183277+0000", 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.112e-05 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.183284+0000", 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8750000000000002e-06 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.190117+0000", 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068333140000000001 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.190267+0000", 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000149736 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.190363+0000", 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6156000000000006e-05 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.194971+0000", 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046081580000000002 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.194985+0000", 2024-08-29T15:23:57.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3668e-05 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.198371+0000", 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033858360000000001 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.198428+0000", 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7652999999999998e-05 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:23:57.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:23:15.199271+0000", 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00084256000000000001 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:23:57.618 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:23:57.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:57.645 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:57.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:57.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:57.769 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:57.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:57.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:57.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:57.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:57.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:57.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:58.018 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:58.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:58.120 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:58.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.142 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:58.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:58.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:58.266 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:57 smithi171 ceph-mon[90031]: pgmap v847: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 30102/215802 objects degraded (13.949%); 13432/215802 objects misplaced (6.224%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:57 smithi027 ceph-mon[98970]: pgmap v847: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 30102/215802 objects degraded (13.949%); 13432/215802 objects misplaced (6.224%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:57 smithi027 ceph-mon[93921]: pgmap v847: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 30102/215802 objects degraded (13.949%); 13432/215802 objects misplaced (6.224%); 589 KiB/s, 9 objects/s recovering 2024-08-29T15:23:58.368 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:23:58.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:58.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:23:58.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:23:58.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:58.514 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:58.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:58.637 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:58.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:58.739 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:23:58.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:58.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:23:58.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:58.886 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:58.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:58.988 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:23:59.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:23:59.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:59.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:59.240 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:23:59.264 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:23:59.267 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:23:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:23:59] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-08-29T15:23:59.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:23:59.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:59.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:23:59.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:59.520 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:59.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:59.548 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:59.559 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:59.606 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:23:59.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:23:59.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:23:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:23:59.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:23:59.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:59.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:23:59.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:23:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:23:59.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:23:59.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:23:59.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:59 smithi027 ceph-mon[93921]: pgmap v848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/215655 objects degraded (13.958%); 13382/215655 objects misplaced (6.205%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:59.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:59.970 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:23:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:23:59.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:59 smithi027 ceph-mon[98970]: pgmap v848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/215655 objects degraded (13.958%); 13382/215655 objects misplaced (6.205%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:23:59.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:23:59.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:23:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:00.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:59 smithi171 ceph-mon[90031]: pgmap v848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 30102/215655 objects degraded (13.958%); 13382/215655 objects misplaced (6.205%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:24:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:23:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:00.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:00.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:00.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:00.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:00.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:00.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:00.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:00.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:00.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:00.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:00.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:00.629 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:00.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:00.706 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:24:00.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:00.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:00.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:00.902 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:00.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:00.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:01.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2267521493' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:00 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:01.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2267521493' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:00 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2267521493' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:00 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:01.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:01.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:01.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:01.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:01.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:01.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.400 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:01.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:01.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.524 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:01.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:01.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:01.651 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:01.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:01.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.778 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:01.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[98970]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/215133 objects degraded (13.992%); 13382/215133 objects misplaced (6.220%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:24:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:24:01.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[98970]: osdmap e261: 8 total, 8 up, 8 in 2024-08-29T15:24:01.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[93921]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/215133 objects degraded (13.992%); 13382/215133 objects misplaced (6.220%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:24:01.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:01.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:24:01.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:01 smithi027 ceph-mon[93921]: osdmap e261: 8 total, 8 up, 8 in 2024-08-29T15:24:01.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:01.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:01.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:02.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:01 smithi171 ceph-mon[90031]: pgmap v849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 127 op/s; 30102/215133 objects degraded (13.992%); 13382/215133 objects misplaced (6.220%); 608 KiB/s, 9 objects/s recovering 2024-08-29T15:24:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:01 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:01 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:24:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:01 smithi171 ceph-mon[90031]: osdmap e261: 8 total, 8 up, 8 in 2024-08-29T15:24:02.037 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:02.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:02.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:02.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:02.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:02.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:02.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:02.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:02.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:02.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:02.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:02.640 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:02.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:02.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:02.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:02.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:02.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:02.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:02.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:03.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30102/215133 objects degraded (13.992%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:03.023 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30102/215133 objects degraded (13.992%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:03.023 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30102/215133 objects degraded (13.992%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:03.040 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:03.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:03.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:03.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:03.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:03.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:03.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:03.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:03.415 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:03.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:03.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:24:03.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:03.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:03.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:03.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:03.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:03.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:03.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:03.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:03.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:03.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:03.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:03 smithi171 ceph-mon[90031]: pgmap v851: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/213582 objects degraded (14.094%); 13246/213582 objects misplaced (6.202%); 646 KiB/s, 10 objects/s recovering 2024-08-29T15:24:04.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:04.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:04.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:04.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:03 smithi027 ceph-mon[98970]: pgmap v851: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/213582 objects degraded (14.094%); 13246/213582 objects misplaced (6.202%); 646 KiB/s, 10 objects/s recovering 2024-08-29T15:24:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:03 smithi027 ceph-mon[93921]: pgmap v851: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 30102/213582 objects degraded (14.094%); 13246/213582 objects misplaced (6.202%); 646 KiB/s, 10 objects/s recovering 2024-08-29T15:24:04.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:04.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:04.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:04.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:04.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:04.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:04.429 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:04.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:04.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:04.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:04.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:04.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:04.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:04.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:04.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:04.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:04.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:04.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:05.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:05.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:05.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:05.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:05.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:05.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:05.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:05.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:05.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:05.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:05.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:05.576 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:24:05.577 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:24:05.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:05.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:05.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:05.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:05.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:05 smithi027 ceph-mon[98970]: pgmap v852: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/213267 objects degraded (14.115%); 13246/213267 objects misplaced (6.211%); 646 KiB/s, 10 objects/s recovering 2024-08-29T15:24:05.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:05 smithi027 ceph-mon[93921]: pgmap v852: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/213267 objects degraded (14.115%); 13246/213267 objects misplaced (6.211%); 646 KiB/s, 10 objects/s recovering 2024-08-29T15:24:05.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:05.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:05.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:05.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:05.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:05 smithi171 ceph-mon[90031]: pgmap v852: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 30102/213267 objects degraded (14.115%); 13246/213267 objects misplaced (6.211%); 646 KiB/s, 10 objects/s recovering 2024-08-29T15:24:06.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:06.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:06.080 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:24:06.080 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:24:06.080 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:24:06.080 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:24:06.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:06.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:06.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:06.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:06.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:06.423 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:06.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:06.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:06.535 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:24:06.535 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:24:06.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:06.567 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:24:06.567 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:24:06.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:06.584 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:24:06.584 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:24:06.600 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:24:06.601 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:24:06.617 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:24:06.617 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:24:06.633 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:24:06.633 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:24:06.649 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:24:06.650 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:24:06.668 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:24:06.669 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:24:06.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:06.696 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:06.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:06.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:06.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:06.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:06.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:06.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:06 smithi171 ceph-mon[90031]: from='client.45695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1478964293' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:07.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:07.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:06 smithi027 ceph-mon[98970]: from='client.45695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:07.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1478964293' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:07.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:06 smithi027 ceph-mon[93921]: from='client.45695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:07.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1478964293' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:07.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:07.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:07.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:24:07.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:07.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.203 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:07.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:07.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:07.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:07.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:07.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:07.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:07.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:07.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:07.702 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:07.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:07.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:07.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:07.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:07.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:07.948 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:07.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:07.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:07 smithi171 ceph-mon[90031]: pgmap v853: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30082/211221 objects degraded (14.242%); 13135/211221 objects misplaced (6.219%); 774 KiB/s, 12 objects/s recovering 2024-08-29T15:24:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30082/211221 objects degraded (14.242%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:08.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:07 smithi027 ceph-mon[98970]: pgmap v853: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30082/211221 objects degraded (14.242%); 13135/211221 objects misplaced (6.219%); 774 KiB/s, 12 objects/s recovering 2024-08-29T15:24:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30082/211221 objects degraded (14.242%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:08.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:07 smithi027 ceph-mon[93921]: pgmap v853: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30082/211221 objects degraded (14.242%); 13135/211221 objects misplaced (6.219%); 774 KiB/s, 12 objects/s recovering 2024-08-29T15:24:08.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30082/211221 objects degraded (14.242%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:08.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:08.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:08.200 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:08.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:08.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:08.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:08.427 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:08.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:08.518 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":852,"stamp":"2024-08-29T15:24:04.601369+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4661778429,"num_objects":71089,"num_object_clones":0,"num_object_copies":213267,"num_objects_missing_on_primary":5439,"num_objects_missing":5439,"num_objects_degraded":30102,"num_objects_misplaced":13246,"num_objects_unfound":0,"num_objects_dirty":71089,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":180912,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16029,"num_bytes_recovered":929303757,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125548,"ondisk_log_size":125548,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15543692,"kb_used_data":13672008,"kb_used_omap":43,"kb_used_meta":1871572,"kb_avail":734417524,"statfs":{"total":767960285184,"available":752043544576,"internally_reserved":0,"allocated":14000136192,"data_stored":13988318286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1916490220},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,2,1,1,2,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-86704128,"num_objects":-1323,"num_object_clones":0,"num_object_copies":-3969,"num_objects_missing_on_primary":-99,"num_objects_missing":-99,"num_objects_degraded":0,"num_objects_misplaced":-265,"num_objects_unfound":0,"num_objects_dirty":-1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1323,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":6619136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002959"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":407,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.743385+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:24:00.743385+0000","last_peered":"2024-08-29T15:24:00.743385+0000","last_clean":"2024-08-29T15:24:00.743385+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:24:00.743385+0000","last_undegraded":"2024-08-29T15:24:00.743385+0000","last_fullsized":"2024-08-29T15:24:00.743385+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":37,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:36.712962+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:23:36.712962+0000","last_peered":"2024-08-29T15:23:36.712962+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:23:36.712962+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:23:36.712962+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":128,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.858473+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:23:50.858473+0000","last_peered":"2024-08-29T15:23:50.858473+0000","last_clean":"2024-08-29T15:23:50.858473+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:23:50.858473+0000","last_undegraded":"2024-08-29T15:23:50.858473+0000","last_fullsized":"2024-08-29T15:23:50.858473+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":373,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704778+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:24:00.704778+0000","last_peered":"2024-08-29T15:24:00.704778+0000","last_clean":"2024-08-29T15:24:00.704778+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:24:00.704778+0000","last_undegraded":"2024-08-29T15:24:00.704778+0000","last_fullsized":"2024-08-29T15:24:00.704778+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":372,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704863+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:24:00.704863+0000","last_peered":"2024-08-29T15:24:00.704863+0000","last_clean":"2024-08-29T15:24:00.704863+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:24:00.704863+0000","last_undegraded":"2024-08-29T15:24:00.704863+0000","last_fullsized":"2024-08-29T15:24:00.704863+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.712812+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:23:36.712812+0000","last_peered":"2024-08-29T15:23:36.712812+0000","last_clean":"2024-08-29T15:23:36.712812+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:23:36.712812+0000","last_undegraded":"2024-08-29T15:23:36.712812+0000","last_fullsized":"2024-08-29T15:23:36.712812+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":182,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013695+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:23:51.013695+0000","last_peered":"2024-08-29T15:23:51.013695+0000","last_clean":"2024-08-29T15:23:51.013695+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:23:51.013695+0000","last_undegraded":"2024-08-29T15:23:51.013695+0000","last_fullsized":"2024-08-29T15:23:51.013695+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":130,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:57.240219+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:23:57.240219+0000","last_peered":"2024-08-29T15:23:57.240219+0000","last_clean":"2024-08-29T15:23:57.240219+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:23:57.240219+0000","last_undegraded":"2024-08-29T15:23:57.240219+0000","last_fullsized":"2024-08-29T15:23:57.240219+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"261'11377","reported_seq":19416,"reported_epoch":261,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:01.082400+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:24:01.082400+0000","last_peered":"2024-08-29T15:24:01.082400+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:24:01.082400+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"175'1300","ondisk_log_start":"175'1300","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10077,"log_dups_size":0,"ondisk_log_size":10077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297205760,"num_objects":4535,"num_object_clones":0,"num_object_copies":13605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11377,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":1610}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":36,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:36.713437+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:23:36.713437+0000","last_peered":"2024-08-29T15:23:36.713437+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:23:36.713437+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:23:36.713437+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":146,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.858300+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:23:50.858300+0000","last_peered":"2024-08-29T15:23:50.858300+0000","last_clean":"2024-08-29T15:23:50.858300+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:23:50.858300+0000","last_undegraded":"2024-08-29T15:23:50.858300+0000","last_fullsized":"2024-08-29T15:23:50.858300+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"261'11140","reported_seq":11340,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:01.086458+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:24:01.086458+0000","last_peered":"2024-08-29T15:24:01.086458+0000","last_clean":"2024-08-29T15:24:01.086458+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:24:01.086458+0000","last_undegraded":"2024-08-29T15:24:01.086458+0000","last_fullsized":"2024-08-29T15:24:01.086458+0000","mapping_epoch":168,"log_start":"180'4428","ondisk_log_start":"180'4428","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289013760,"num_objects":4410,"num_object_clones":0,"num_object_copies":13230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11140,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":36,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:36.716654+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:23:36.716654+0000","last_peered":"2024-08-29T15:23:36.716654+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:23:36.716654+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:23:36.716654+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.712689+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:23:36.712689+0000","last_peered":"2024-08-29T15:23:36.712689+0000","last_clean":"2024-08-29T15:23:36.712689+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:23:36.712689+0000","last_undegraded":"2024-08-29T15:23:36.712689+0000","last_fullsized":"2024-08-29T15:23:36.712689+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"261'11306","reported_seq":11476,"reported_epoch":261,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:01.309466+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:24:01.309466+0000","last_peered":"2024-08-29T15:24:01.309466+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:24:01.309466+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"180'4574","ondisk_log_start":"180'4574","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6732,"log_dups_size":0,"ondisk_log_size":6732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288751616,"num_objects":4406,"num_object_clones":0,"num_object_copies":13218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11306,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1929}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":189,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.705104+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:24:00.705104+0000","last_peered":"2024-08-29T15:24:00.705104+0000","last_clean":"2024-08-29T15:24:00.705104+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:24:00.705104+0000","last_undegraded":"2024-08-29T15:24:00.705104+0000","last_fullsized":"2024-08-29T15:24:00.705104+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":85,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.060578+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:23:51.060578+0000","last_peered":"2024-08-29T15:23:51.060578+0000","last_clean":"2024-08-29T15:23:51.060578+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:23:51.060578+0000","last_undegraded":"2024-08-29T15:23:51.060578+0000","last_fullsized":"2024-08-29T15:23:51.060578+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"261'11417","reported_seq":16838,"reported_epoch":261,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:03.913020+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:24:03.913020+0000","last_peered":"2024-08-29T15:24:03.913020+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:24:03.913020+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"178'3262","ondisk_log_start":"178'3262","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8155,"log_dups_size":0,"ondisk_log_size":8155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285802496,"num_objects":4361,"num_object_clones":0,"num_object_copies":13083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11417,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4361}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.859488+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:23:50.859488+0000","last_peered":"2024-08-29T15:23:50.859488+0000","last_clean":"2024-08-29T15:23:50.859488+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:23:50.859488+0000","last_undegraded":"2024-08-29T15:23:50.859488+0000","last_fullsized":"2024-08-29T15:23:50.859488+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":387,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704519+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:24:00.704519+0000","last_peered":"2024-08-29T15:24:00.704519+0000","last_clean":"2024-08-29T15:24:00.704519+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:24:00.704519+0000","last_undegraded":"2024-08-29T15:24:00.704519+0000","last_fullsized":"2024-08-29T15:24:00.704519+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"260'11271","reported_seq":11417,"reported_epoch":260,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:59.523646+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:23:59.523646+0000","last_peered":"2024-08-29T15:23:59.523646+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:23:59.523646+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:23:59.523646+0000","mapping_epoch":234,"log_start":"178'3333","ondisk_log_start":"178'3333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297730048,"num_objects":4543,"num_object_clones":0,"num_object_copies":13629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11271,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":2066},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":381,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.743152+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:24:00.743152+0000","last_peered":"2024-08-29T15:24:00.743152+0000","last_clean":"2024-08-29T15:24:00.743152+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:24:00.743152+0000","last_undegraded":"2024-08-29T15:24:00.743152+0000","last_fullsized":"2024-08-29T15:24:00.743152+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.713043+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:23:36.713043+0000","last_peered":"2024-08-29T15:23:36.713043+0000","last_clean":"2024-08-29T15:23:36.713043+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:23:36.713043+0000","last_undegraded":"2024-08-29T15:23:36.713043+0000","last_fullsized":"2024-08-29T15:23:36.713043+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"261'11317","reported_seq":11534,"reported_epoch":261,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:01.336324+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:24:01.336324+0000","last_peered":"2024-08-29T15:24:01.336324+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:24:01.336324+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:24:01.336324+0000","mapping_epoch":234,"log_start":"180'4568","ondisk_log_start":"180'4568","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296550400,"num_objects":4525,"num_object_clones":0,"num_object_copies":13575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11317,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":2060},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.060514+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:23:51.060514+0000","last_peered":"2024-08-29T15:23:51.060514+0000","last_clean":"2024-08-29T15:23:51.060514+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:23:51.060514+0000","last_undegraded":"2024-08-29T15:23:51.060514+0000","last_fullsized":"2024-08-29T15:23:51.060514+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.713378+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:23:36.713378+0000","last_peered":"2024-08-29T15:23:36.713378+0000","last_clean":"2024-08-29T15:23:36.713378+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:23:36.713378+0000","last_undegraded":"2024-08-29T15:23:36.713378+0000","last_fullsized":"2024-08-29T15:23:36.713378+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"261'11182","reported_seq":11583,"reported_epoch":261,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:24:01.325039+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:24:01.325039+0000","last_peered":"2024-08-29T15:24:01.325039+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:24:01.325039+0000","last_undegraded":"2024-08-29T15:24:01.325039+0000","last_fullsized":"2024-08-29T15:24:01.325039+0000","mapping_epoch":202,"log_start":"180'4467","ondisk_log_start":"180'4467","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293339136,"num_objects":4476,"num_object_clones":0,"num_object_copies":13428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4476,"num_objects_unfound":0,"num_objects_dirty":4476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11182,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4476}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.712726+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:23:36.712726+0000","last_peered":"2024-08-29T15:23:36.712726+0000","last_clean":"2024-08-29T15:23:36.712726+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:23:36.712726+0000","last_undegraded":"2024-08-29T15:23:36.712726+0000","last_fullsized":"2024-08-29T15:23:36.712726+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":244,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.716718+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:23:36.716718+0000","last_peered":"2024-08-29T15:23:36.716718+0000","last_clean":"2024-08-29T15:23:36.716718+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:23:36.716718+0000","last_undegraded":"2024-08-29T15:23:36.716718+0000","last_fullsized":"2024-08-29T15:23:36.716718+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"261'11311","reported_seq":11452,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:01.095421+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:24:01.095421+0000","last_peered":"2024-08-29T15:24:01.095421+0000","last_clean":"2024-08-29T15:24:01.095421+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:24:01.095421+0000","last_undegraded":"2024-08-29T15:24:01.095421+0000","last_fullsized":"2024-08-29T15:24:01.095421+0000","mapping_epoch":218,"log_start":"175'1278","ondisk_log_start":"175'1278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10033,"log_dups_size":0,"ondisk_log_size":10033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291504152,"num_objects":4449,"num_object_clones":0,"num_object_copies":13347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4449,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11311,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":384,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.742096+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:24:00.742096+0000","last_peered":"2024-08-29T15:24:00.742096+0000","last_clean":"2024-08-29T15:24:00.742096+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:24:00.742096+0000","last_undegraded":"2024-08-29T15:24:00.742096+0000","last_fullsized":"2024-08-29T15:24:00.742096+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":375,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013641+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:23:51.013641+0000","last_peered":"2024-08-29T15:23:51.013641+0000","last_clean":"2024-08-29T15:23:51.013641+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:23:51.013641+0000","last_undegraded":"2024-08-29T15:23:51.013641+0000","last_fullsized":"2024-08-29T15:23:51.013641+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"261'10892","reported_seq":11081,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.898982+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:24:00.898982+0000","last_peered":"2024-08-29T15:24:00.898982+0000","last_clean":"2024-08-29T15:24:00.898982+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:24:00.898982+0000","last_undegraded":"2024-08-29T15:24:00.898982+0000","last_fullsized":"2024-08-29T15:24:00.898982+0000","mapping_epoch":201,"log_start":"175'800","ondisk_log_start":"175'800","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282853376,"num_objects":4316,"num_object_clones":0,"num_object_copies":12948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10892,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":475,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704232+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:24:00.704232+0000","last_peered":"2024-08-29T15:24:00.704232+0000","last_clean":"2024-08-29T15:24:00.704232+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:24:00.704232+0000","last_undegraded":"2024-08-29T15:24:00.704232+0000","last_fullsized":"2024-08-29T15:24:00.704232+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":731,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:56.175052+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:23:56.175052+0000","last_peered":"2024-08-29T15:23:56.175052+0000","last_clean":"2024-08-29T15:23:56.175052+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:23:56.175052+0000","last_undegraded":"2024-08-29T15:23:56.175052+0000","last_fullsized":"2024-08-29T15:23:56.175052+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":239,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.173286+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:23:51.173286+0000","last_peered":"2024-08-29T15:23:51.173286+0000","last_clean":"2024-08-29T15:23:51.173286+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:23:51.173286+0000","last_undegraded":"2024-08-29T15:23:51.173286+0000","last_fullsized":"2024-08-29T15:23:51.173286+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":101,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.064455+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:23:51.064455+0000","last_peered":"2024-08-29T15:23:51.064455+0000","last_clean":"2024-08-29T15:23:51.064455+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:23:51.064455+0000","last_undegraded":"2024-08-29T15:23:51.064455+0000","last_fullsized":"2024-08-29T15:23:51.064455+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"260'11289","reported_seq":11686,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.229288+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:24:00.229288+0000","last_peered":"2024-08-29T15:24:00.229288+0000","last_clean":"2024-08-29T15:24:00.229288+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:24:00.229288+0000","last_undegraded":"2024-08-29T15:24:00.229288+0000","last_fullsized":"2024-08-29T15:24:00.229288+0000","mapping_epoch":201,"log_start":"179'4253","ondisk_log_start":"179'4253","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11289,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":847,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013161+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:23:51.013161+0000","last_peered":"2024-08-29T15:23:51.013161+0000","last_clean":"2024-08-29T15:23:51.013161+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:23:51.013161+0000","last_undegraded":"2024-08-29T15:23:51.013161+0000","last_fullsized":"2024-08-29T15:23:51.013161+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":388,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704166+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:24:00.704166+0000","last_peered":"2024-08-29T15:24:00.704166+0000","last_clean":"2024-08-29T15:24:00.704166+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:24:00.704166+0000","last_undegraded":"2024-08-29T15:24:00.704166+0000","last_fullsized":"2024-08-29T15:24:00.704166+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":395,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.705023+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:24:00.705023+0000","last_peered":"2024-08-29T15:24:00.705023+0000","last_clean":"2024-08-29T15:24:00.705023+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:24:00.705023+0000","last_undegraded":"2024-08-29T15:24:00.705023+0000","last_fullsized":"2024-08-29T15:24:00.705023+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"261'11151","reported_seq":16973,"reported_epoch":261,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:01.353047+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:24:01.353047+0000","last_peered":"2024-08-29T15:24:01.353047+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:24:01.353047+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"180'4430","ondisk_log_start":"180'4430","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4965,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11151,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2050},{"shards":"3,6","objects":105},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":658,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.742581+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:24:00.742581+0000","last_peered":"2024-08-29T15:24:00.742581+0000","last_clean":"2024-08-29T15:24:00.742581+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:24:00.742581+0000","last_undegraded":"2024-08-29T15:24:00.742581+0000","last_fullsized":"2024-08-29T15:24:00.742581+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":91,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.060007+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:23:51.060007+0000","last_peered":"2024-08-29T15:23:51.060007+0000","last_clean":"2024-08-29T15:23:51.060007+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:23:51.060007+0000","last_undegraded":"2024-08-29T15:23:51.060007+0000","last_fullsized":"2024-08-29T15:23:51.060007+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":420,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013521+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:23:51.013521+0000","last_peered":"2024-08-29T15:23:51.013521+0000","last_clean":"2024-08-29T15:23:51.013521+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:23:51.013521+0000","last_undegraded":"2024-08-29T15:23:51.013521+0000","last_fullsized":"2024-08-29T15:23:51.013521+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"260'11239","reported_seq":11450,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.743863+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:24:00.743863+0000","last_peered":"2024-08-29T15:24:00.743863+0000","last_clean":"2024-08-29T15:24:00.743863+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:24:00.743863+0000","last_undegraded":"2024-08-29T15:24:00.743863+0000","last_fullsized":"2024-08-29T15:24:00.743863+0000","mapping_epoch":234,"log_start":"180'4497","ondisk_log_start":"180'4497","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293142528,"num_objects":4473,"num_object_clones":0,"num_object_copies":13419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11239,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":199,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.173247+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:23:51.173247+0000","last_peered":"2024-08-29T15:23:51.173247+0000","last_clean":"2024-08-29T15:23:51.173247+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:23:51.173247+0000","last_undegraded":"2024-08-29T15:23:51.173247+0000","last_fullsized":"2024-08-29T15:23:51.173247+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":39,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.859967+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:23:50.859967+0000","last_peered":"2024-08-29T15:23:50.859967+0000","last_clean":"2024-08-29T15:23:50.859967+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:23:50.859967+0000","last_undegraded":"2024-08-29T15:23:50.859967+0000","last_fullsized":"2024-08-29T15:23:50.859967+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":106,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:56.175085+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:23:56.175085+0000","last_peered":"2024-08-29T15:23:56.175085+0000","last_clean":"2024-08-29T15:23:56.175085+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:23:56.175085+0000","last_undegraded":"2024-08-29T15:23:56.175085+0000","last_fullsized":"2024-08-29T15:23:56.175085+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"261'11325","reported_seq":11534,"reported_epoch":261,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:03.778990+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:24:03.778990+0000","last_peered":"2024-08-29T15:24:03.778990+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:24:03.778990+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"178'3142","ondisk_log_start":"178'3142","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8183,"log_dups_size":0,"ondisk_log_size":8183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294977536,"num_objects":4501,"num_object_clones":0,"num_object_copies":13503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11325,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":2078}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":131,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:56.175123+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:23:56.175123+0000","last_peered":"2024-08-29T15:23:56.175123+0000","last_clean":"2024-08-29T15:23:56.175123+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:23:56.175123+0000","last_undegraded":"2024-08-29T15:23:56.175123+0000","last_fullsized":"2024-08-29T15:23:56.175123+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":812,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013733+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:23:51.013733+0000","last_peered":"2024-08-29T15:23:51.013733+0000","last_clean":"2024-08-29T15:23:51.013733+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:23:51.013733+0000","last_undegraded":"2024-08-29T15:23:51.013733+0000","last_fullsized":"2024-08-29T15:23:51.013733+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":389,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013358+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:23:51.013358+0000","last_peered":"2024-08-29T15:23:51.013358+0000","last_clean":"2024-08-29T15:23:51.013358+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:23:51.013358+0000","last_undegraded":"2024-08-29T15:23:51.013358+0000","last_fullsized":"2024-08-29T15:23:51.013358+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"260'11252","reported_seq":12015,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:00.327015+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:24:00.327015+0000","last_peered":"2024-08-29T15:24:00.327015+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:24:00.327015+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"179'4180","ondisk_log_start":"179'4180","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4388,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11252,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2265}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":394,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013616+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:23:51.013616+0000","last_peered":"2024-08-29T15:23:51.013616+0000","last_clean":"2024-08-29T15:23:51.013616+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:23:51.013616+0000","last_undegraded":"2024-08-29T15:23:51.013616+0000","last_fullsized":"2024-08-29T15:23:51.013616+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":687,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:39.961719+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:23:39.961719+0000","last_peered":"2024-08-29T15:23:39.961719+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:23:39.961719+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:23:39.961719+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":388,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704448+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:24:00.704448+0000","last_peered":"2024-08-29T15:24:00.704448+0000","last_clean":"2024-08-29T15:24:00.704448+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:24:00.704448+0000","last_undegraded":"2024-08-29T15:24:00.704448+0000","last_fullsized":"2024-08-29T15:24:00.704448+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"260'11103","reported_seq":18248,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:59.713519+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:23:59.713519+0000","last_peered":"2024-08-29T15:23:59.713519+0000","last_clean":"2024-08-29T15:23:59.713519+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:23:59.713519+0000","last_undegraded":"2024-08-29T15:23:59.713519+0000","last_fullsized":"2024-08-29T15:23:59.713519+0000","mapping_epoch":234,"log_start":"178'3199","ondisk_log_start":"178'3199","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7904,"log_dups_size":0,"ondisk_log_size":7904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283443200,"num_objects":4325,"num_object_clones":0,"num_object_copies":12975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11103,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":381,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.744438+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:24:00.744438+0000","last_peered":"2024-08-29T15:24:00.744438+0000","last_clean":"2024-08-29T15:24:00.744438+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:24:00.744438+0000","last_undegraded":"2024-08-29T15:24:00.744438+0000","last_fullsized":"2024-08-29T15:24:00.744438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1002,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:23:51.013697+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:23:51.013697+0000","last_peered":"2024-08-29T15:23:51.013697+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:23:51.013697+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":379,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.741814+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:24:00.741814+0000","last_peered":"2024-08-29T15:24:00.741814+0000","last_clean":"2024-08-29T15:24:00.741814+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:24:00.741814+0000","last_undegraded":"2024-08-29T15:24:00.741814+0000","last_fullsized":"2024-08-29T15:24:00.741814+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"261'11217","reported_seq":15959,"reported_epoch":261,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:02.558359+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:24:02.558359+0000","last_peered":"2024-08-29T15:24:02.558359+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:24:02.558359+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"179'3712","ondisk_log_start":"179'3712","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":5431,"num_objects_missing":5431,"num_objects_degraded":2313,"num_objects_misplaced":4382,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11217,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2293,"num_bytes_recovered":150142976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":11},{"shards":"1,2,7","objects":2011},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":377,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704727+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:24:00.704727+0000","last_peered":"2024-08-29T15:24:00.704727+0000","last_clean":"2024-08-29T15:24:00.704727+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:24:00.704727+0000","last_undegraded":"2024-08-29T15:24:00.704727+0000","last_fullsized":"2024-08-29T15:24:00.704727+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":112,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.060876+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:23:51.060876+0000","last_peered":"2024-08-29T15:23:51.060876+0000","last_clean":"2024-08-29T15:23:51.060876+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:23:51.060876+0000","last_undegraded":"2024-08-29T15:23:51.060876+0000","last_fullsized":"2024-08-29T15:23:51.060876+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":39,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.860056+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:23:50.860056+0000","last_peered":"2024-08-29T15:23:50.860056+0000","last_clean":"2024-08-29T15:23:50.860056+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:23:50.860056+0000","last_undegraded":"2024-08-29T15:23:50.860056+0000","last_fullsized":"2024-08-29T15:23:50.860056+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.713088+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:23:36.713088+0000","last_peered":"2024-08-29T15:23:36.713088+0000","last_clean":"2024-08-29T15:23:36.713088+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:23:36.713088+0000","last_undegraded":"2024-08-29T15:23:36.713088+0000","last_fullsized":"2024-08-29T15:23:36.713088+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":36,"reported_epoch":259,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:36.713140+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:23:36.713140+0000","last_peered":"2024-08-29T15:23:36.713140+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:23:36.713140+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:23:36.713140+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":383,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013296+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:23:51.013296+0000","last_peered":"2024-08-29T15:23:51.013296+0000","last_clean":"2024-08-29T15:23:51.013296+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:23:51.013296+0000","last_undegraded":"2024-08-29T15:23:51.013296+0000","last_fullsized":"2024-08-29T15:23:51.013296+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":136,"reported_epoch":261,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:00.743937+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:24:00.743937+0000","last_peered":"2024-08-29T15:24:00.743937+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:24:00.743937+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:24:00.743937+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":128,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.858613+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:23:50.858613+0000","last_peered":"2024-08-29T15:23:50.858613+0000","last_clean":"2024-08-29T15:23:50.858613+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:23:50.858613+0000","last_undegraded":"2024-08-29T15:23:50.858613+0000","last_fullsized":"2024-08-29T15:23:50.858613+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":345,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.741547+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:24:00.741547+0000","last_peered":"2024-08-29T15:24:00.741547+0000","last_clean":"2024-08-29T15:24:00.741547+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:24:00.741547+0000","last_undegraded":"2024-08-29T15:24:00.741547+0000","last_fullsized":"2024-08-29T15:24:00.741547+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":398,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704729+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:24:00.704729+0000","last_peered":"2024-08-29T15:24:00.704729+0000","last_clean":"2024-08-29T15:24:00.704729+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:24:00.704729+0000","last_undegraded":"2024-08-29T15:24:00.704729+0000","last_fullsized":"2024-08-29T15:24:00.704729+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":397,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.741733+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:24:00.741733+0000","last_peered":"2024-08-29T15:24:00.741733+0000","last_clean":"2024-08-29T15:24:00.741733+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:24:00.741733+0000","last_undegraded":"2024-08-29T15:24:00.741733+0000","last_fullsized":"2024-08-29T15:24:00.741733+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.060894+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:23:51.060894+0000","last_peered":"2024-08-29T15:23:51.060894+0000","last_clean":"2024-08-29T15:23:51.060894+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:23:51.060894+0000","last_undegraded":"2024-08-29T15:23:51.060894+0000","last_fullsized":"2024-08-29T15:23:51.060894+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":254,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704127+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:24:00.704127+0000","last_peered":"2024-08-29T15:24:00.704127+0000","last_clean":"2024-08-29T15:24:00.704127+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:24:00.704127+0000","last_undegraded":"2024-08-29T15:24:00.704127+0000","last_fullsized":"2024-08-29T15:24:00.704127+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":374,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704965+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:24:00.704965+0000","last_peered":"2024-08-29T15:24:00.704965+0000","last_clean":"2024-08-29T15:24:00.704965+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:24:00.704965+0000","last_undegraded":"2024-08-29T15:24:00.704965+0000","last_fullsized":"2024-08-29T15:24:00.704965+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":375,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013302+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:23:51.013302+0000","last_peered":"2024-08-29T15:23:51.013302+0000","last_clean":"2024-08-29T15:23:51.013302+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:23:51.013302+0000","last_undegraded":"2024-08-29T15:23:51.013302+0000","last_fullsized":"2024-08-29T15:23:51.013302+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":382,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.743774+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:24:00.743774+0000","last_peered":"2024-08-29T15:24:00.743774+0000","last_clean":"2024-08-29T15:24:00.743774+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:24:00.743774+0000","last_undegraded":"2024-08-29T15:24:00.743774+0000","last_fullsized":"2024-08-29T15:24:00.743774+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.712812+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:23:36.712812+0000","last_peered":"2024-08-29T15:23:36.712812+0000","last_clean":"2024-08-29T15:23:36.712812+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:23:36.712812+0000","last_undegraded":"2024-08-29T15:23:36.712812+0000","last_fullsized":"2024-08-29T15:23:36.712812+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":400,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:51.013779+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:23:51.013779+0000","last_peered":"2024-08-29T15:23:51.013779+0000","last_clean":"2024-08-29T15:23:51.013779+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:23:51.013779+0000","last_undegraded":"2024-08-29T15:23:51.013779+0000","last_fullsized":"2024-08-29T15:23:51.013779+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":365,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704367+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:24:00.704367+0000","last_peered":"2024-08-29T15:24:00.704367+0000","last_clean":"2024-08-29T15:24:00.704367+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:24:00.704367+0000","last_undegraded":"2024-08-29T15:24:00.704367+0000","last_fullsized":"2024-08-29T15:24:00.704367+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.712800+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:23:36.712800+0000","last_peered":"2024-08-29T15:23:36.712800+0000","last_clean":"2024-08-29T15:23:36.712800+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:23:36.712800+0000","last_undegraded":"2024-08-29T15:23:36.712800+0000","last_fullsized":"2024-08-29T15:23:36.712800+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":152,"reported_epoch":260,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:23:50.859204+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:23:50.859204+0000","last_peered":"2024-08-29T15:23:50.859204+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:23:50.859204+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:23:50.859204+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.712858+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:23:36.712858+0000","last_peered":"2024-08-29T15:23:36.712858+0000","last_clean":"2024-08-29T15:23:36.712858+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:23:36.712858+0000","last_undegraded":"2024-08-29T15:23:36.712858+0000","last_fullsized":"2024-08-29T15:23:36.712858+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":158,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-08-29T15:23:50.858670+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:23:50.858670+0000","last_peered":"2024-08-29T15:23:50.858670+0000","last_clean":"2024-08-29T15:23:50.858670+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:23:50.858670+0000","last_undegraded":"2024-08-29T15:23:50.858670+0000","last_fullsized":"2024-08-29T15:23:50.858670+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":413,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.744542+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:24:00.744542+0000","last_peered":"2024-08-29T15:24:00.744542+0000","last_clean":"2024-08-29T15:24:00.744542+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:24:00.744542+0000","last_undegraded":"2024-08-29T15:24:00.744542+0000","last_fullsized":"2024-08-29T15:24:00.744542+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":414,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.744019+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:24:00.744019+0000","last_peered":"2024-08-29T15:24:00.744019+0000","last_clean":"2024-08-29T15:24:00.744019+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:24:00.744019+0000","last_undegraded":"2024-08-29T15:24:00.744019+0000","last_fullsized":"2024-08-29T15:24:00.744019+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":259,"state":"active+clean","last_fresh":"2024-08-29T15:23:36.712611+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:23:36.712611+0000","last_peered":"2024-08-29T15:23:36.712611+0000","last_clean":"2024-08-29T15:23:36.712611+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:23:36.712611+0000","last_undegraded":"2024-08-29T15:23:36.712611+0000","last_fullsized":"2024-08-29T15:23:36.712611+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":422,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.704726+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:24:00.704726+0000","last_peered":"2024-08-29T15:24:00.704726+0000","last_clean":"2024-08-29T15:24:00.704726+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:24:00.704726+0000","last_undegraded":"2024-08-29T15:24:00.704726+0000","last_fullsized":"2024-08-29T15:24:00.704726+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4656988184,"num_objects":71061,"num_object_clones":0,"num_object_copies":213183,"num_objects_missing_on_primary":5431,"num_objects_missing":5431,"num_objects_degraded":30073,"num_objects_misplaced":13246,"num_objects_unfound":0,"num_objects_dirty":71061,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179789,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15920,"num_bytes_recovered":923336704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13945090048,"data_stored":13945077832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124366,"ondisk_log_size":124366,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234352,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2588120,"kb_used_data":2286288,"kb_used_omap":1,"kb_used_meta":301822,"kb_avail":91157032,"statfs":{"total":95995035648,"available":93344800768,"internally_reserved":0,"allocated":2341158912,"data_stored":2339673851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":309066187},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495984,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":294692,"kb_used_data":119196,"kb_used_omap":1,"kb_used_meta":175486,"kb_avail":93450460,"statfs":{"total":95995035648,"available":95693271040,"internally_reserved":0,"allocated":122056704,"data_stored":120567795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":179698119},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,1,1,2,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426570,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1478984,"kb_used_data":1423872,"kb_used_omap":8,"kb_used_meta":55095,"kb_avail":92266168,"statfs":{"total":95995035648,"available":94480556032,"internally_reserved":0,"allocated":1458044928,"data_stored":1456571133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":56418186},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019262,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1751480,"kb_used_data":1427504,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":91993672,"statfs":{"total":95995035648,"available":94201520128,"internally_reserved":0,"allocated":1461764096,"data_stored":1460306084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435447,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3136344,"kb_used_data":2843724,"kb_used_omap":4,"kb_used_meta":292603,"kb_avail":90608808,"statfs":{"total":95995035648,"available":92783419392,"internally_reserved":0,"allocated":2911973376,"data_stored":2910495873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":299625737},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903299,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2697176,"kb_used_data":2569616,"kb_used_omap":10,"kb_used_meta":127541,"kb_avail":91047976,"statfs":{"total":95995035648,"available":93233127424,"internally_reserved":0,"allocated":2631286784,"data_stored":2629809217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":130602512},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762500,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2128964,"kb_used_data":1854332,"kb_used_omap":7,"kb_used_meta":274616,"kb_avail":91616188,"statfs":{"total":95995035648,"available":93814976512,"internally_reserved":0,"allocated":1898835968,"data_stored":1897364873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":281207138},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926027,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1467932,"kb_used_data":1147476,"kb_used_omap":7,"kb_used_meta":320440,"kb_avail":92277220,"statfs":{"total":95995035648,"available":94491873280,"internally_reserved":0,"allocated":1175015424,"data_stored":1173529460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":328130703},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1165627392,"data_stored":1165623320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1889865728,"data_stored":1889861656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2623275008,"data_stored":2623275008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2907701248,"data_stored":2907701248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1455554560,"data_stored":1455554560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1452937216,"data_stored":1452933144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":117243904,"data_stored":117243904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2332884992,"data_stored":2332884992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:24:08.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:24:08.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:08.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:08.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:08.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:08.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:08.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:08.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:08.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:08.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:08.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:09.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:09.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:09.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:09.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:09.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:09.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:09.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:09.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:24:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:24:09] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.51.0" 2024-08-29T15:24:09.329 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:09.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:09.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:09.454 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:09.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:09.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:09.581 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:09.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:09.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:09.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:09.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:09.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:09.807 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:09.833 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:09.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:09.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:10.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:24:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:09 smithi171 ceph-mon[90031]: pgmap v854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30082/211086 objects degraded (14.251%); 13087/211086 objects misplaced (6.200%); 749 KiB/s, 11 objects/s recovering 2024-08-29T15:24:10.023 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:10.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:09 smithi027 ceph-mon[98970]: pgmap v854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30082/211086 objects degraded (14.251%); 13087/211086 objects misplaced (6.200%); 749 KiB/s, 11 objects/s recovering 2024-08-29T15:24:10.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:09 smithi027 ceph-mon[93921]: pgmap v854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 30082/211086 objects degraded (14.251%); 13087/211086 objects misplaced (6.200%); 749 KiB/s, 11 objects/s recovering 2024-08-29T15:24:10.042 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:10.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:10.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:10.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:10.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:10.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:10.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:10.415 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:10.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:10.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:10.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:10.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:10.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:10.779 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:10.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:10.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:10.911 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:10.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:10.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:11.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:11.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:11.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:11.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:11.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:11.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:11.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:11.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:11.388 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:11.411 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:11.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:11.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:11.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:11.658 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:11.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:11.784 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:11.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:11 smithi027 ceph-mon[93921]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 30082/210498 objects degraded (14.291%); 13087/210498 objects misplaced (6.217%); 749 KiB/s, 11 objects/s recovering 2024-08-29T15:24:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:11 smithi027 ceph-mon[98970]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 30082/210498 objects degraded (14.291%); 13087/210498 objects misplaced (6.217%); 749 KiB/s, 11 objects/s recovering 2024-08-29T15:24:11.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:11.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:11.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:12.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:11 smithi171 ceph-mon[90031]: pgmap v855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 30082/210498 objects degraded (14.291%); 13087/210498 objects misplaced (6.217%); 749 KiB/s, 11 objects/s recovering 2024-08-29T15:24:12.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:12.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:12.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:12.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:12.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:12.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:12.286 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:12.388 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:12.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:12.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:12.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:12.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:12.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:12.659 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:12.761 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:12.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:12.851 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 30082/210498 objects degraded (14.291%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:12.851 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 30082/210498 objects degraded (14.291%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:12.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:12.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:12.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:12.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:13.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 30082/210498 objects degraded (14.291%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:13.033 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:13.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:13.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:13.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:13.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:13.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:13.288 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:13.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:13.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:13.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:13.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:13.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:13.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:13.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:13.735 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:24:13.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:13.792 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:13.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:13.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:13.917 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:13.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:13.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:13 smithi171 ceph-mon[90031]: pgmap v856: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 30023/208794 objects degraded (14.379%); 12988/208794 objects misplaced (6.220%); 947 KiB/s, 14 objects/s recovering 2024-08-29T15:24:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2848861191' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:14.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:14.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[98970]: pgmap v856: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 30023/208794 objects degraded (14.379%); 12988/208794 objects misplaced (6.220%); 947 KiB/s, 14 objects/s recovering 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2848861191' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[93921]: pgmap v856: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 30023/208794 objects degraded (14.379%); 12988/208794 objects misplaced (6.220%); 947 KiB/s, 14 objects/s recovering 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2848861191' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:14.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:14.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:14.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:14.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:14.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:14.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:14.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:14.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:14.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:14.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:14.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:14.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:14.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:14.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:14.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:14.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:14.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[93921]: osdmap e262: 8 total, 8 up, 8 in 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:14.885 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[98970]: osdmap e262: 8 total, 8 up, 8 in 2024-08-29T15:24:14.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:14.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:14.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:14.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:14.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:14.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:14 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:14 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:14 smithi171 ceph-mon[90031]: osdmap e262: 8 total, 8 up, 8 in 2024-08-29T15:24:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:15.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:15.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:15.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:15.126 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:15.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:15.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:15.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:15.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:15.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:15.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:15.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:15.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:15.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:15.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:15.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:15.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:15.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:15.778 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:15.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:15.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:15.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:15.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:15 smithi171 ceph-mon[90031]: pgmap v858: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 30023/208353 objects degraded (14.410%); 12939/208353 objects misplaced (6.210%); 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:24:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:15 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:24:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:15 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:24:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:15 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:16.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:16.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:16.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[98970]: pgmap v858: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 30023/208353 objects degraded (14.410%); 12939/208353 objects misplaced (6.210%); 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:24:16.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:24:16.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:24:16.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:16.067 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[93921]: pgmap v858: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 30023/208353 objects degraded (14.410%); 12939/208353 objects misplaced (6.210%); 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:24:16.067 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:24:16.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:24:16.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:15 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:24:16.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:16.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:16.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:16.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:16.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:16.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:16.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:16.536 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:16.558 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:16.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:16.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:16.685 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:16.722 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:24:16.722 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:24:16.787 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:16.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:16.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:16.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:16.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:16.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:16.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:17.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:17.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:17.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:17.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:17.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:17.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:17.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:17.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:17.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:17.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:17.435 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:17.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:17.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:24:17.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:17.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:17.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:17.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:24:17.685 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:17.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:17.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:17.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:17.936 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:17.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:17 smithi171 ceph-mon[90031]: pgmap v859: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 29990/206712 objects degraded (14.508%); 12854/206712 objects misplaced (6.218%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29990/206712 objects degraded (14.508%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:18.034 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:17 smithi027 ceph-mon[98970]: pgmap v859: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 29990/206712 objects degraded (14.508%); 12854/206712 objects misplaced (6.218%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:18.034 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29990/206712 objects degraded (14.508%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:18.035 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:17 smithi027 ceph-mon[93921]: pgmap v859: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 29990/206712 objects degraded (14.508%); 12854/206712 objects misplaced (6.218%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:18.035 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29990/206712 objects degraded (14.508%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:18.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:18.060 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:18.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:18.185 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:18.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:18.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:18.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:18.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:18.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:18.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:18.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:18.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:18.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:18.764 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:24:18.765 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 29990/206565 objects degraded (14.518%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1240,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":262,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":68855,"data_bytes":4515371005,"bytes_used":15496802304,"bytes_avail":752463482880,"bytes_total":767960285184,"degraded_objects":29990,"degraded_total":206565,"degraded_ratio":0.14518432454675284,"misplaced_objects":12803,"misplaced_total":206565,"misplaced_ratio":0.061980490402536732,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1251398,"recovering_keys_per_sec":0,"num_objects_recovered":191,"num_bytes_recovered":12517376,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":150},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:24:18.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:18.819 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:18.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:18.929 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:24:18.930 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:24:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:18.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:18.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:18.952 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:24:18.952 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:24:18.970 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:24:18.970 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:24:18.988 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:24:18.988 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:24:19.007 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:24:19.007 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:24:19.028 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:24:19.028 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:24:19.045 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:24:19.046 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:24:19.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:19.065 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:24:19.066 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:24:19.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:19.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.202 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:19.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:19.289 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:24:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:24:19] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-08-29T15:24:19.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:19.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:19.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:19.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:19.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:19.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:19.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:19.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:19.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:19.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:19.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:19.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:19.819 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:24:19.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:24:19.824 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:19.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:19.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:19.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:19.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:19.999 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:19 smithi171 ceph-mon[90031]: pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 29990/206565 objects degraded (14.518%); 12803/206565 objects misplaced (6.198%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1975731020' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:24:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4223333010' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:20.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:19 smithi027 ceph-mon[93921]: pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 29990/206565 objects degraded (14.518%); 12803/206565 objects misplaced (6.198%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1975731020' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:24:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4223333010' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:19 smithi027 ceph-mon[98970]: pgmap v860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 29990/206565 objects degraded (14.518%); 12803/206565 objects misplaced (6.198%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1975731020' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:24:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4223333010' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:20.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:20.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:20.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:20.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:20.321 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:20.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:20.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:20.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:20.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:20.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:20.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:20.575 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:20.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:20.696 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:20.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:20.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:20.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:20.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:20.946 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:20.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:21.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:21.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:21.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:21.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:21.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:21.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:21.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:21.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:21.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:21.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:21.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:21.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:21.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:21.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:21.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:21.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:21.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:21 smithi027 ceph-mon[98970]: pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 29990/205935 objects degraded (14.563%); 12803/205935 objects misplaced (6.217%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:21.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:21 smithi027 ceph-mon[93921]: pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 29990/205935 objects degraded (14.563%); 12803/205935 objects misplaced (6.217%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:21.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:21.859 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:21.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:21.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:21.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:21 smithi171 ceph-mon[90031]: pgmap v861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 29990/205935 objects degraded (14.563%); 12803/205935 objects misplaced (6.217%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:22.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:22.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:22.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:22.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:22.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:22.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:22.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:22.427 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:22.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:22.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:22.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:22.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:24:22.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:22.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:22.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:22.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29990/205935 objects degraded (14.563%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:22.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:22.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:22.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:22.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:22.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:22.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:22.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29990/205935 objects degraded (14.563%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:23.063 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29990/205935 objects degraded (14.563%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:23.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:23.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:23.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:23.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:23.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:23.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:23.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:23.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:23.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:23.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:23.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:23.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:23.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:23.711 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:23.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:23.742 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:24:23.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:23 smithi027 ceph-mon[98970]: pgmap v862: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 29939/204354 objects degraded (14.651%); 12717/204354 objects misplaced (6.223%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:24:23.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1669776378' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:24:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:23 smithi027 ceph-mon[93921]: pgmap v862: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 29939/204354 objects degraded (14.651%); 12717/204354 objects misplaced (6.223%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:24:23.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1669776378' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:24:23.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:23.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:23.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:23.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:23.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:23.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:23 smithi171 ceph-mon[90031]: pgmap v862: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 29939/204354 objects degraded (14.651%); 12717/204354 objects misplaced (6.223%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:24:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1669776378' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:24:24.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:24.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:24.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:24.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:24.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:24.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:24.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:24.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:24.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:24.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:24.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:24.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:24.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:24.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:24.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:24.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:24.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:24.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:24.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:24.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:24.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:24.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:24 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1669776378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:24:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:24 smithi171 ceph-mon[90031]: osdmap e263: 8 total, 8 up, 8 in 2024-08-29T15:24:25.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:24 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1669776378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:24:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:24 smithi027 ceph-mon[98970]: osdmap e263: 8 total, 8 up, 8 in 2024-08-29T15:24:25.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:24 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:25.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1669776378' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:24:25.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:24 smithi027 ceph-mon[93921]: osdmap e263: 8 total, 8 up, 8 in 2024-08-29T15:24:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:25.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:25.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:25.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:25.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:25.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:25.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:25.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:25.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:25.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:25.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:25.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:25.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:25.815 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:25.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:25.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:25.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:25.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:25.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:25 smithi171 ceph-mon[90031]: pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 29939/203952 objects degraded (14.679%); 12669/203952 objects misplaced (6.212%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:26.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:25 smithi027 ceph-mon[93921]: pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 29939/203952 objects degraded (14.679%); 12669/203952 objects misplaced (6.212%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:26.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:25 smithi027 ceph-mon[98970]: pgmap v864: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 29939/203952 objects degraded (14.679%); 12669/203952 objects misplaced (6.212%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:26.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:26.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:26.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:26.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:26.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:26.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:26.342 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:26.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:26.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:26.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:26.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:26.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:26.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:24:26.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:26.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:26.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:26.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:26.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:26.842 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:24:26.843 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:24:26.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:26.965 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:26.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:27.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:27.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:27.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:27.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:27.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:27.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:27.352 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:27.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:27.478 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:27.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:27.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:27.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.604 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:27.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:27.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:27.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:27.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:27.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:27.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:27.945 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":865,"stamp":"2024-08-29T15:24:26.607538+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4420737021,"num_objects":67411,"num_object_clones":0,"num_object_copies":202233,"num_objects_missing_on_primary":5243,"num_objects_missing":5243,"num_objects_degraded":29893,"num_objects_misplaced":12579,"num_objects_unfound":0,"num_objects_dirty":67411,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":184590,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16435,"num_bytes_recovered":955911373,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125226,"ondisk_log_size":125226,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14933692,"kb_used_data":13013736,"kb_used_omap":43,"kb_used_meta":1919828,"kb_avail":735027524,"statfs":{"total":767960285184,"available":752668184576,"internally_reserved":0,"allocated":13326065664,"data_stored":13314110094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1965904364},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,0,0,0,0,0,1,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-97845248,"num_objects":-1493,"num_object_clones":0,"num_object_copies":-4479,"num_objects_missing_on_primary":-99,"num_objects_missing":-99,"num_objects_degraded":-97,"num_objects_misplaced":-275,"num_objects_unfound":0,"num_objects_dirty":-1493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1493,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":198,"num_bytes_recovered":12976128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002859"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":411,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745036+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:24:23.745036+0000","last_peered":"2024-08-29T15:24:23.745036+0000","last_clean":"2024-08-29T15:24:23.745036+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:24:23.745036+0000","last_undegraded":"2024-08-29T15:24:23.745036+0000","last_fullsized":"2024-08-29T15:24:23.745036+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":40,"reported_epoch":262,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:13.736313+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:24:13.736313+0000","last_peered":"2024-08-29T15:24:13.736313+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:24:13.736313+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:24:13.736313+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":130,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733968+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:24:13.733968+0000","last_peered":"2024-08-29T15:24:13.733968+0000","last_clean":"2024-08-29T15:24:13.733968+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:24:13.733968+0000","last_undegraded":"2024-08-29T15:24:13.733968+0000","last_fullsized":"2024-08-29T15:24:13.733968+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":377,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.792939+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:24:23.792939+0000","last_peered":"2024-08-29T15:24:23.792939+0000","last_clean":"2024-08-29T15:24:23.792939+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:24:23.792939+0000","last_undegraded":"2024-08-29T15:24:23.792939+0000","last_fullsized":"2024-08-29T15:24:23.792939+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":377,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793362+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:24:23.793362+0000","last_peered":"2024-08-29T15:24:23.793362+0000","last_clean":"2024-08-29T15:24:23.793362+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:24:23.793362+0000","last_undegraded":"2024-08-29T15:24:23.793362+0000","last_fullsized":"2024-08-29T15:24:23.793362+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":41,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735830+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:24:13.735830+0000","last_peered":"2024-08-29T15:24:13.735830+0000","last_clean":"2024-08-29T15:24:13.735830+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:24:13.735830+0000","last_undegraded":"2024-08-29T15:24:13.735830+0000","last_fullsized":"2024-08-29T15:24:13.735830+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":188,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.748188+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:24:23.748188+0000","last_peered":"2024-08-29T15:24:23.748188+0000","last_clean":"2024-08-29T15:24:23.748188+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:24:23.748188+0000","last_undegraded":"2024-08-29T15:24:23.748188+0000","last_fullsized":"2024-08-29T15:24:23.748188+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":131,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.741956+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:24:00.741956+0000","last_peered":"2024-08-29T15:24:00.741956+0000","last_clean":"2024-08-29T15:24:00.741956+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:24:00.741956+0000","last_undegraded":"2024-08-29T15:24:00.741956+0000","last_fullsized":"2024-08-29T15:24:00.741956+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"263'11603","reported_seq":19645,"reported_epoch":263,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:25.903245+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:24:25.903245+0000","last_peered":"2024-08-29T15:24:25.903245+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:24:25.903245+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"176'1600","ondisk_log_start":"176'1600","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10003,"log_dups_size":0,"ondisk_log_size":10003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282394624,"num_objects":4309,"num_object_clones":0,"num_object_copies":12927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11603,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":1384}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":40,"reported_epoch":262,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:19.816945+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:24:19.816945+0000","last_peered":"2024-08-29T15:24:19.816945+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:24:19.816945+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:24:19.816945+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":149,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733108+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:24:13.733108+0000","last_peered":"2024-08-29T15:24:13.733108+0000","last_clean":"2024-08-29T15:24:13.733108+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:24:13.733108+0000","last_undegraded":"2024-08-29T15:24:13.733108+0000","last_fullsized":"2024-08-29T15:24:13.733108+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"263'11359","reported_seq":11564,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:26.360940+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:24:26.360940+0000","last_peered":"2024-08-29T15:24:26.360940+0000","last_clean":"2024-08-29T15:24:26.360940+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:24:26.360940+0000","last_undegraded":"2024-08-29T15:24:26.360940+0000","last_fullsized":"2024-08-29T15:24:26.360940+0000","mapping_epoch":168,"log_start":"180'4628","ondisk_log_start":"180'4628","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274661376,"num_objects":4191,"num_object_clones":0,"num_object_copies":12573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11359,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":40,"reported_epoch":262,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:19.816841+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:24:19.816841+0000","last_peered":"2024-08-29T15:24:19.816841+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:24:19.816841+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:24:19.816841+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":42,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735734+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:24:13.735734+0000","last_peered":"2024-08-29T15:24:13.735734+0000","last_clean":"2024-08-29T15:24:13.735734+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:24:13.735734+0000","last_undegraded":"2024-08-29T15:24:13.735734+0000","last_fullsized":"2024-08-29T15:24:13.735734+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"263'11555","reported_seq":11729,"reported_epoch":263,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:26.353708+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:24:26.353708+0000","last_peered":"2024-08-29T15:24:26.353708+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:24:26.353708+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"180'4874","ondisk_log_start":"180'4874","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272433152,"num_objects":4157,"num_object_clones":0,"num_object_copies":12471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11555,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1680}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":193,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793364+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:24:23.793364+0000","last_peered":"2024-08-29T15:24:23.793364+0000","last_clean":"2024-08-29T15:24:23.793364+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:24:23.793364+0000","last_undegraded":"2024-08-29T15:24:23.793364+0000","last_fullsized":"2024-08-29T15:24:23.793364+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":86,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.742554+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:24:00.742554+0000","last_peered":"2024-08-29T15:24:00.742554+0000","last_clean":"2024-08-29T15:24:00.742554+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:24:00.742554+0000","last_undegraded":"2024-08-29T15:24:00.742554+0000","last_fullsized":"2024-08-29T15:24:00.742554+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"263'11592","reported_seq":17017,"reported_epoch":263,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:23.931680+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:24:23.931680+0000","last_peered":"2024-08-29T15:24:23.931680+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:24:23.931680+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"178'3462","ondisk_log_start":"178'3462","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274333696,"num_objects":4186,"num_object_clones":0,"num_object_copies":12558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11592,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4186}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":149,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:21.179246+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:24:21.179246+0000","last_peered":"2024-08-29T15:24:21.179246+0000","last_clean":"2024-08-29T15:24:21.179246+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:24:21.179246+0000","last_undegraded":"2024-08-29T15:24:21.179246+0000","last_fullsized":"2024-08-29T15:24:21.179246+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":391,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793184+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:24:23.793184+0000","last_peered":"2024-08-29T15:24:23.793184+0000","last_clean":"2024-08-29T15:24:23.793184+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:24:23.793184+0000","last_undegraded":"2024-08-29T15:24:23.793184+0000","last_fullsized":"2024-08-29T15:24:23.793184+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"263'11499","reported_seq":11649,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:24.507389+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:24:24.507389+0000","last_peered":"2024-08-29T15:24:24.507389+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:24:24.507389+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:24:24.507389+0000","mapping_epoch":234,"log_start":"178'3533","ondisk_log_start":"178'3533","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7966,"log_dups_size":0,"ondisk_log_size":7966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282787840,"num_objects":4315,"num_object_clones":0,"num_object_copies":12945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11499,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":1838},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":385,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745438+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:24:23.745438+0000","last_peered":"2024-08-29T15:24:23.745438+0000","last_clean":"2024-08-29T15:24:23.745438+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:24:23.745438+0000","last_undegraded":"2024-08-29T15:24:23.745438+0000","last_fullsized":"2024-08-29T15:24:23.745438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":42,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:19.814853+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:24:19.814853+0000","last_peered":"2024-08-29T15:24:19.814853+0000","last_clean":"2024-08-29T15:24:19.814853+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:24:19.814853+0000","last_undegraded":"2024-08-29T15:24:19.814853+0000","last_fullsized":"2024-08-29T15:24:19.814853+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"263'11554","reported_seq":11775,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:26.381166+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:24:26.381166+0000","last_peered":"2024-08-29T15:24:26.381166+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:24:26.381166+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:24:26.381166+0000","mapping_epoch":234,"log_start":"180'4868","ondisk_log_start":"180'4868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281018368,"num_objects":4288,"num_object_clones":0,"num_object_copies":12864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11554,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":1823},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":104,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.742132+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:24:00.742132+0000","last_peered":"2024-08-29T15:24:00.742132+0000","last_clean":"2024-08-29T15:24:00.742132+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:24:00.742132+0000","last_undegraded":"2024-08-29T15:24:00.742132+0000","last_fullsized":"2024-08-29T15:24:00.742132+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":42,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:19.814887+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:24:19.814887+0000","last_peered":"2024-08-29T15:24:19.814887+0000","last_clean":"2024-08-29T15:24:19.814887+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:24:19.814887+0000","last_undegraded":"2024-08-29T15:24:19.814887+0000","last_fullsized":"2024-08-29T15:24:19.814887+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"263'11407","reported_seq":11813,"reported_epoch":263,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:24:25.831026+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:24:25.831026+0000","last_peered":"2024-08-29T15:24:25.831026+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:24:25.831026+0000","last_undegraded":"2024-08-29T15:24:25.831026+0000","last_fullsized":"2024-08-29T15:24:25.831026+0000","mapping_epoch":202,"log_start":"180'4667","ondisk_log_start":"180'4667","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278593536,"num_objects":4251,"num_object_clones":0,"num_object_copies":12753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4251,"num_objects_unfound":0,"num_objects_dirty":4251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11407,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4251}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":42,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735752+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:24:13.735752+0000","last_peered":"2024-08-29T15:24:13.735752+0000","last_clean":"2024-08-29T15:24:13.735752+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:24:13.735752+0000","last_undegraded":"2024-08-29T15:24:13.735752+0000","last_fullsized":"2024-08-29T15:24:13.735752+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":247,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.739573+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:24:13.739573+0000","last_peered":"2024-08-29T15:24:13.739573+0000","last_clean":"2024-08-29T15:24:13.739573+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:24:13.739573+0000","last_undegraded":"2024-08-29T15:24:13.739573+0000","last_fullsized":"2024-08-29T15:24:13.739573+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"263'11567","reported_seq":11711,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:25.729076+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:24:25.729076+0000","last_peered":"2024-08-29T15:24:25.729076+0000","last_clean":"2024-08-29T15:24:25.729076+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:24:25.729076+0000","last_undegraded":"2024-08-29T15:24:25.729076+0000","last_fullsized":"2024-08-29T15:24:25.729076+0000","mapping_epoch":218,"log_start":"176'1478","ondisk_log_start":"176'1478","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10089,"log_dups_size":0,"ondisk_log_size":10089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274726936,"num_objects":4193,"num_object_clones":0,"num_object_copies":12579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4193,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11567,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":389,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.746265+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:24:23.746265+0000","last_peered":"2024-08-29T15:24:23.746265+0000","last_clean":"2024-08-29T15:24:23.746265+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:24:23.746265+0000","last_undegraded":"2024-08-29T15:24:23.746265+0000","last_fullsized":"2024-08-29T15:24:23.746265+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":381,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.748288+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:24:23.748288+0000","last_peered":"2024-08-29T15:24:23.748288+0000","last_clean":"2024-08-29T15:24:23.748288+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:24:23.748288+0000","last_undegraded":"2024-08-29T15:24:23.748288+0000","last_fullsized":"2024-08-29T15:24:23.748288+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"263'11105","reported_seq":11297,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:25.761627+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:24:25.761627+0000","last_peered":"2024-08-29T15:24:25.761627+0000","last_clean":"2024-08-29T15:24:25.761627+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:24:25.761627+0000","last_undegraded":"2024-08-29T15:24:25.761627+0000","last_fullsized":"2024-08-29T15:24:25.761627+0000","mapping_epoch":201,"log_start":"175'1100","ondisk_log_start":"175'1100","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10005,"log_dups_size":0,"ondisk_log_size":10005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268894208,"num_objects":4103,"num_object_clones":0,"num_object_copies":12309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11105,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":480,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.792903+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:24:23.792903+0000","last_peered":"2024-08-29T15:24:23.792903+0000","last_clean":"2024-08-29T15:24:23.792903+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:24:23.792903+0000","last_undegraded":"2024-08-29T15:24:23.792903+0000","last_fullsized":"2024-08-29T15:24:23.792903+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":733,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733489+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:24:13.733489+0000","last_peered":"2024-08-29T15:24:13.733489+0000","last_clean":"2024-08-29T15:24:13.733489+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:24:13.733489+0000","last_undegraded":"2024-08-29T15:24:13.733489+0000","last_fullsized":"2024-08-29T15:24:13.733489+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":241,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733368+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:24:13.733368+0000","last_peered":"2024-08-29T15:24:13.733368+0000","last_clean":"2024-08-29T15:24:13.733368+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:24:13.733368+0000","last_undegraded":"2024-08-29T15:24:13.733368+0000","last_fullsized":"2024-08-29T15:24:13.733368+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.742715+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:24:00.742715+0000","last_peered":"2024-08-29T15:24:00.742715+0000","last_clean":"2024-08-29T15:24:00.742715+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:24:00.742715+0000","last_undegraded":"2024-08-29T15:24:00.742715+0000","last_fullsized":"2024-08-29T15:24:00.742715+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"263'11531","reported_seq":11934,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:25.245585+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:24:25.245585+0000","last_peered":"2024-08-29T15:24:25.245585+0000","last_clean":"2024-08-29T15:24:25.245585+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:24:25.245585+0000","last_undegraded":"2024-08-29T15:24:25.245585+0000","last_fullsized":"2024-08-29T15:24:25.245585+0000","mapping_epoch":201,"log_start":"180'4553","ondisk_log_start":"180'4553","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272039936,"num_objects":4151,"num_object_clones":0,"num_object_copies":12453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11531,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":854,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.746770+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:24:23.746770+0000","last_peered":"2024-08-29T15:24:23.746770+0000","last_clean":"2024-08-29T15:24:23.746770+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:24:23.746770+0000","last_undegraded":"2024-08-29T15:24:23.746770+0000","last_fullsized":"2024-08-29T15:24:23.746770+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":392,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.792850+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:24:23.792850+0000","last_peered":"2024-08-29T15:24:23.792850+0000","last_clean":"2024-08-29T15:24:23.792850+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:24:23.792850+0000","last_undegraded":"2024-08-29T15:24:23.792850+0000","last_fullsized":"2024-08-29T15:24:23.792850+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":399,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793534+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:24:23.793534+0000","last_peered":"2024-08-29T15:24:23.793534+0000","last_clean":"2024-08-29T15:24:23.793534+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:24:23.793534+0000","last_undegraded":"2024-08-29T15:24:23.793534+0000","last_fullsized":"2024-08-29T15:24:23.793534+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"263'11411","reported_seq":17655,"reported_epoch":263,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:26.340122+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:24:26.340122+0000","last_peered":"2024-08-29T15:24:26.340122+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:24:26.340122+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"180'4730","ondisk_log_start":"180'4730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280428544,"num_objects":4279,"num_object_clones":0,"num_object_copies":12837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4756,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11411,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3035,"num_bytes_recovered":198901760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1841},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":662,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744797+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:24:23.744797+0000","last_peered":"2024-08-29T15:24:23.744797+0000","last_clean":"2024-08-29T15:24:23.744797+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:24:23.744797+0000","last_undegraded":"2024-08-29T15:24:23.744797+0000","last_fullsized":"2024-08-29T15:24:23.744797+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":93,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.744667+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:24:13.744667+0000","last_peered":"2024-08-29T15:24:13.744667+0000","last_clean":"2024-08-29T15:24:13.744667+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:24:13.744667+0000","last_undegraded":"2024-08-29T15:24:13.744667+0000","last_fullsized":"2024-08-29T15:24:13.744667+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":426,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.747667+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:24:23.747667+0000","last_peered":"2024-08-29T15:24:23.747667+0000","last_clean":"2024-08-29T15:24:23.747667+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:24:23.747667+0000","last_undegraded":"2024-08-29T15:24:23.747667+0000","last_fullsized":"2024-08-29T15:24:23.747667+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"263'11483","reported_seq":11698,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:26.313984+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:24:26.313984+0000","last_peered":"2024-08-29T15:24:26.313984+0000","last_clean":"2024-08-29T15:24:26.313984+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:24:26.313984+0000","last_undegraded":"2024-08-29T15:24:26.313984+0000","last_fullsized":"2024-08-29T15:24:26.313984+0000","mapping_epoch":234,"log_start":"180'4797","ondisk_log_start":"180'4797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277151744,"num_objects":4229,"num_object_clones":0,"num_object_copies":12687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11483,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":201,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733225+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:24:13.733225+0000","last_peered":"2024-08-29T15:24:13.733225+0000","last_clean":"2024-08-29T15:24:13.733225+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:24:13.733225+0000","last_undegraded":"2024-08-29T15:24:13.733225+0000","last_fullsized":"2024-08-29T15:24:13.733225+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":42,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735984+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:24:13.735984+0000","last_peered":"2024-08-29T15:24:13.735984+0000","last_clean":"2024-08-29T15:24:13.735984+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:24:13.735984+0000","last_undegraded":"2024-08-29T15:24:13.735984+0000","last_fullsized":"2024-08-29T15:24:13.735984+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":108,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733279+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:24:13.733279+0000","last_peered":"2024-08-29T15:24:13.733279+0000","last_clean":"2024-08-29T15:24:13.733279+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:24:13.733279+0000","last_undegraded":"2024-08-29T15:24:13.733279+0000","last_fullsized":"2024-08-29T15:24:13.733279+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"263'11524","reported_seq":11737,"reported_epoch":263,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:23.794697+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:24:23.794697+0000","last_peered":"2024-08-29T15:24:23.794697+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:24:23.794697+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"178'3342","ondisk_log_start":"178'3342","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8182,"log_dups_size":0,"ondisk_log_size":8182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281935872,"num_objects":4302,"num_object_clones":0,"num_object_copies":12906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11524,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1879}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":133,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733897+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:24:13.733897+0000","last_peered":"2024-08-29T15:24:13.733897+0000","last_clean":"2024-08-29T15:24:13.733897+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:24:13.733897+0000","last_undegraded":"2024-08-29T15:24:13.733897+0000","last_fullsized":"2024-08-29T15:24:13.733897+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":818,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.748842+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:24:23.748842+0000","last_peered":"2024-08-29T15:24:23.748842+0000","last_clean":"2024-08-29T15:24:23.748842+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:24:23.748842+0000","last_undegraded":"2024-08-29T15:24:23.748842+0000","last_fullsized":"2024-08-29T15:24:23.748842+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":395,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.747141+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:24:23.747141+0000","last_peered":"2024-08-29T15:24:23.747141+0000","last_clean":"2024-08-29T15:24:23.747141+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:24:23.747141+0000","last_undegraded":"2024-08-29T15:24:23.747141+0000","last_fullsized":"2024-08-29T15:24:23.747141+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"263'11498","reported_seq":12267,"reported_epoch":263,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:25.663202+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:24:25.663202+0000","last_peered":"2024-08-29T15:24:25.663202+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:24:25.663202+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"180'4480","ondisk_log_start":"180'4480","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273285120,"num_objects":4170,"num_object_clones":0,"num_object_copies":12510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4142,"num_objects_unfound":0,"num_objects_dirty":4170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11498,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":2019}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":400,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.748740+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:24:23.748740+0000","last_peered":"2024-08-29T15:24:23.748740+0000","last_clean":"2024-08-29T15:24:23.748740+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:24:23.748740+0000","last_undegraded":"2024-08-29T15:24:23.748740+0000","last_fullsized":"2024-08-29T15:24:23.748740+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":690,"reported_epoch":262,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:13.751350+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:24:13.751350+0000","last_peered":"2024-08-29T15:24:13.751350+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:24:13.751350+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:24:13.751350+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":392,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793115+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:24:23.793115+0000","last_peered":"2024-08-29T15:24:23.793115+0000","last_clean":"2024-08-29T15:24:23.793115+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:24:23.793115+0000","last_undegraded":"2024-08-29T15:24:23.793115+0000","last_fullsized":"2024-08-29T15:24:23.793115+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"263'11365","reported_seq":18513,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:24.454865+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:24:24.454865+0000","last_peered":"2024-08-29T15:24:24.454865+0000","last_clean":"2024-08-29T15:24:24.454865+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:24:24.454865+0000","last_undegraded":"2024-08-29T15:24:24.454865+0000","last_fullsized":"2024-08-29T15:24:24.454865+0000","mapping_epoch":234,"log_start":"178'3399","ondisk_log_start":"178'3399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7966,"log_dups_size":0,"ondisk_log_size":7966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266272768,"num_objects":4063,"num_object_clones":0,"num_object_copies":12189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11365,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":386,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.746010+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:24:23.746010+0000","last_peered":"2024-08-29T15:24:23.746010+0000","last_clean":"2024-08-29T15:24:23.746010+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:24:23.746010+0000","last_undegraded":"2024-08-29T15:24:23.746010+0000","last_fullsized":"2024-08-29T15:24:23.746010+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1008,"reported_epoch":263,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:23.747399+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:24:23.747399+0000","last_peered":"2024-08-29T15:24:23.747399+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:24:23.747399+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":383,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744514+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:24:23.744514+0000","last_peered":"2024-08-29T15:24:23.744514+0000","last_clean":"2024-08-29T15:24:23.744514+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:24:23.744514+0000","last_undegraded":"2024-08-29T15:24:23.744514+0000","last_fullsized":"2024-08-29T15:24:23.744514+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"262'11414","reported_seq":16551,"reported_epoch":262,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:22.791018+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:24:22.791018+0000","last_peered":"2024-08-29T15:24:22.791018+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:24:22.791018+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"179'3912","ondisk_log_start":"179'3912","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7502,"log_dups_size":0,"ondisk_log_size":7502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274989056,"num_objects":4196,"num_object_clones":0,"num_object_copies":12588,"num_objects_missing_on_primary":5235,"num_objects_missing":5235,"num_objects_degraded":2313,"num_objects_misplaced":4186,"num_objects_unfound":0,"num_objects_dirty":4196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11414,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2490,"num_bytes_recovered":163053568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":10},{"shards":"1,2,7","objects":1815},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":381,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793349+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:24:23.793349+0000","last_peered":"2024-08-29T15:24:23.793349+0000","last_clean":"2024-08-29T15:24:23.793349+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:24:23.793349+0000","last_undegraded":"2024-08-29T15:24:23.793349+0000","last_fullsized":"2024-08-29T15:24:23.793349+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":113,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.742659+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:24:00.742659+0000","last_peered":"2024-08-29T15:24:00.742659+0000","last_clean":"2024-08-29T15:24:00.742659+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:24:00.742659+0000","last_undegraded":"2024-08-29T15:24:00.742659+0000","last_fullsized":"2024-08-29T15:24:00.742659+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":42,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735798+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:24:13.735798+0000","last_peered":"2024-08-29T15:24:13.735798+0000","last_clean":"2024-08-29T15:24:13.735798+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:24:13.735798+0000","last_undegraded":"2024-08-29T15:24:13.735798+0000","last_fullsized":"2024-08-29T15:24:13.735798+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":41,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.736055+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:24:13.736055+0000","last_peered":"2024-08-29T15:24:13.736055+0000","last_clean":"2024-08-29T15:24:13.736055+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:24:13.736055+0000","last_undegraded":"2024-08-29T15:24:13.736055+0000","last_fullsized":"2024-08-29T15:24:13.736055+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":39,"reported_epoch":262,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:13.736127+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:24:13.736127+0000","last_peered":"2024-08-29T15:24:13.736127+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:24:13.736127+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:24:13.736127+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":390,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.747001+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:24:23.747001+0000","last_peered":"2024-08-29T15:24:23.747001+0000","last_clean":"2024-08-29T15:24:23.747001+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:24:23.747001+0000","last_undegraded":"2024-08-29T15:24:23.747001+0000","last_fullsized":"2024-08-29T15:24:23.747001+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":140,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:23.745502+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:24:23.745502+0000","last_peered":"2024-08-29T15:24:23.745502+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:24:23.745502+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:24:23.745502+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.733797+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:24:13.733797+0000","last_peered":"2024-08-29T15:24:13.733797+0000","last_clean":"2024-08-29T15:24:13.733797+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:24:13.733797+0000","last_undegraded":"2024-08-29T15:24:13.733797+0000","last_fullsized":"2024-08-29T15:24:13.733797+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":349,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744453+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:24:23.744453+0000","last_peered":"2024-08-29T15:24:23.744453+0000","last_clean":"2024-08-29T15:24:23.744453+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:24:23.744453+0000","last_undegraded":"2024-08-29T15:24:23.744453+0000","last_fullsized":"2024-08-29T15:24:23.744453+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":403,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793426+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:24:23.793426+0000","last_peered":"2024-08-29T15:24:23.793426+0000","last_clean":"2024-08-29T15:24:23.793426+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:24:23.793426+0000","last_undegraded":"2024-08-29T15:24:23.793426+0000","last_fullsized":"2024-08-29T15:24:23.793426+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":401,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.746118+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:24:23.746118+0000","last_peered":"2024-08-29T15:24:23.746118+0000","last_clean":"2024-08-29T15:24:23.746118+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:24:23.746118+0000","last_undegraded":"2024-08-29T15:24:23.746118+0000","last_fullsized":"2024-08-29T15:24:23.746118+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-08-29T15:24:00.742739+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:24:00.742739+0000","last_peered":"2024-08-29T15:24:00.742739+0000","last_clean":"2024-08-29T15:24:00.742739+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:24:00.742739+0000","last_undegraded":"2024-08-29T15:24:00.742739+0000","last_fullsized":"2024-08-29T15:24:00.742739+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.792784+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:24:23.792784+0000","last_peered":"2024-08-29T15:24:23.792784+0000","last_clean":"2024-08-29T15:24:23.792784+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:24:23.792784+0000","last_undegraded":"2024-08-29T15:24:23.792784+0000","last_fullsized":"2024-08-29T15:24:23.792784+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":378,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793469+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:24:23.793469+0000","last_peered":"2024-08-29T15:24:23.793469+0000","last_clean":"2024-08-29T15:24:23.793469+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:24:23.793469+0000","last_undegraded":"2024-08-29T15:24:23.793469+0000","last_fullsized":"2024-08-29T15:24:23.793469+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":381,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.749357+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:24:23.749357+0000","last_peered":"2024-08-29T15:24:23.749357+0000","last_clean":"2024-08-29T15:24:23.749357+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:24:23.749357+0000","last_undegraded":"2024-08-29T15:24:23.749357+0000","last_fullsized":"2024-08-29T15:24:23.749357+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":386,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745356+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:24:23.745356+0000","last_peered":"2024-08-29T15:24:23.745356+0000","last_clean":"2024-08-29T15:24:23.745356+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:24:23.745356+0000","last_undegraded":"2024-08-29T15:24:23.745356+0000","last_fullsized":"2024-08-29T15:24:23.745356+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":41,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735993+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:24:13.735993+0000","last_peered":"2024-08-29T15:24:13.735993+0000","last_clean":"2024-08-29T15:24:13.735993+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:24:13.735993+0000","last_undegraded":"2024-08-29T15:24:13.735993+0000","last_fullsized":"2024-08-29T15:24:13.735993+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":406,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.748921+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:24:23.748921+0000","last_peered":"2024-08-29T15:24:23.748921+0000","last_clean":"2024-08-29T15:24:23.748921+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:24:23.748921+0000","last_undegraded":"2024-08-29T15:24:23.748921+0000","last_fullsized":"2024-08-29T15:24:23.748921+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":369,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793029+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:24:23.793029+0000","last_peered":"2024-08-29T15:24:23.793029+0000","last_clean":"2024-08-29T15:24:23.793029+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:24:23.793029+0000","last_undegraded":"2024-08-29T15:24:23.793029+0000","last_fullsized":"2024-08-29T15:24:23.793029+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":41,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735933+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:24:13.735933+0000","last_peered":"2024-08-29T15:24:13.735933+0000","last_clean":"2024-08-29T15:24:13.735933+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:24:13.735933+0000","last_undegraded":"2024-08-29T15:24:13.735933+0000","last_fullsized":"2024-08-29T15:24:13.735933+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":154,"reported_epoch":262,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:13.733649+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:24:13.733649+0000","last_peered":"2024-08-29T15:24:13.733649+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:24:13.733649+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:24:13.733649+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":41,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.736064+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:24:13.736064+0000","last_peered":"2024-08-29T15:24:13.736064+0000","last_clean":"2024-08-29T15:24:13.736064+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:24:13.736064+0000","last_undegraded":"2024-08-29T15:24:13.736064+0000","last_fullsized":"2024-08-29T15:24:13.736064+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":160,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.734043+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:24:13.734043+0000","last_peered":"2024-08-29T15:24:13.734043+0000","last_clean":"2024-08-29T15:24:13.734043+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:24:13.734043+0000","last_undegraded":"2024-08-29T15:24:13.734043+0000","last_fullsized":"2024-08-29T15:24:13.734043+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":417,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744923+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:24:23.744923+0000","last_peered":"2024-08-29T15:24:23.744923+0000","last_clean":"2024-08-29T15:24:23.744923+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:24:23.744923+0000","last_undegraded":"2024-08-29T15:24:23.744923+0000","last_fullsized":"2024-08-29T15:24:23.744923+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":418,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745089+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:24:23.745089+0000","last_peered":"2024-08-29T15:24:23.745089+0000","last_clean":"2024-08-29T15:24:23.745089+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:24:23.745089+0000","last_undegraded":"2024-08-29T15:24:23.745089+0000","last_fullsized":"2024-08-29T15:24:23.745089+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":41,"reported_epoch":262,"state":"active+clean","last_fresh":"2024-08-29T15:24:13.735545+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:24:13.735545+0000","last_peered":"2024-08-29T15:24:13.735545+0000","last_clean":"2024-08-29T15:24:13.735545+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:24:13.735545+0000","last_undegraded":"2024-08-29T15:24:13.735545+0000","last_fullsized":"2024-08-29T15:24:13.735545+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":426,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.793401+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:24:23.793401+0000","last_peered":"2024-08-29T15:24:23.793401+0000","last_clean":"2024-08-29T15:24:23.793401+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:24:23.793401+0000","last_undegraded":"2024-08-29T15:24:23.793401+0000","last_fullsized":"2024-08-29T15:24:23.793401+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4415946776,"num_objects":67383,"num_object_clones":0,"num_object_copies":202149,"num_objects_missing_on_primary":5235,"num_objects_missing":5235,"num_objects_degraded":29864,"num_objects_misplaced":12579,"num_objects_unfound":0,"num_objects_dirty":67383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":183467,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16326,"num_bytes_recovered":949944320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13267447808,"data_stored":13267435592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124044,"ondisk_log_size":124044,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234357,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2493452,"kb_used_data":2169472,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":91251700,"statfs":{"total":95995035648,"available":93441740800,"internally_reserved":0,"allocated":2221539328,"data_stored":2220031323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":331741643},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495988,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":289332,"kb_used_data":105644,"kb_used_omap":1,"kb_used_meta":183678,"kb_avail":93455820,"statfs":{"total":95995035648,"available":95698759680,"internally_reserved":0,"allocated":108179456,"data_stored":106682899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188086727},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,0,0,0,0,1,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426575,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1426748,"kb_used_data":1349488,"kb_used_omap":8,"kb_used_meta":77239,"kb_avail":92318404,"statfs":{"total":95995035648,"available":94534045696,"internally_reserved":0,"allocated":1381875712,"data_stored":1380378973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":79093642},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019267,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680300,"kb_used_data":1352224,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":92064852,"statfs":{"total":95995035648,"available":94274408448,"internally_reserved":0,"allocated":1384677376,"data_stored":1383196420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":335935942},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435452,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2975924,"kb_used_data":2691628,"kb_used_omap":4,"kb_used_meta":284283,"kb_avail":90769228,"statfs":{"total":95995035648,"available":92947689472,"internally_reserved":0,"allocated":2756227072,"data_stored":2754734273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":291106057},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903303,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2605672,"kb_used_data":2469792,"kb_used_omap":10,"kb_used_meta":135861,"kb_avail":91139480,"statfs":{"total":95995035648,"available":93326827520,"internally_reserved":0,"allocated":2529067008,"data_stored":2527581793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":139122192},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762505,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2039288,"kb_used_data":1777132,"kb_used_omap":7,"kb_used_meta":262136,"kb_avail":91705864,"statfs":{"total":95995035648,"available":93906804736,"internally_reserved":0,"allocated":1819783168,"data_stored":1818289129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":268427618},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926031,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1422976,"kb_used_data":1098356,"kb_used_omap":7,"kb_used_meta":324600,"kb_avail":92322176,"statfs":{"total":95995035648,"available":94537908224,"internally_reserved":0,"allocated":1124716544,"data_stored":1123215284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":332390543},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1116213248,"data_stored":1116209176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1811353600,"data_stored":1811349528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2523004928,"data_stored":2523004928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2750349312,"data_stored":2750349312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1378680832,"data_stored":1378680832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1376391168,"data_stored":1376387096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":103022592,"data_stored":103022592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2208432128,"data_stored":2208432128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:24:27.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:27.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:27.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:27 smithi171 ceph-mon[90031]: pgmap v865: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 29893/202233 objects degraded (14.781%); 12579/202233 objects misplaced (6.220%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:27 smithi171 ceph-mon[90031]: from='client.45731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29893/202233 objects degraded (14.781%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:27 smithi027 ceph-mon[98970]: pgmap v865: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 29893/202233 objects degraded (14.781%); 12579/202233 objects misplaced (6.220%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:27 smithi027 ceph-mon[98970]: from='client.45731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29893/202233 objects degraded (14.781%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:27 smithi027 ceph-mon[93921]: pgmap v865: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 29893/202233 objects degraded (14.781%); 12579/202233 objects misplaced (6.220%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:27 smithi027 ceph-mon[93921]: from='client.45731 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29893/202233 objects degraded (14.781%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:28.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:28.102 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:28.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:28.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:28.196 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:24:28.196 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:24:28.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:68822 6.1 6:86e313fb:::benchmark_data_smithi027_169445_object68821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e179)", 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 495.65866708300001, 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.107834163, 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.197 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 68822 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366408+0000", 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366410+0000", 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8899999999999999e-06 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366438+0000", 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8589e-05 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366453+0000", 2024-08-29T15:24:28.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5085e-05 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366463+0000", 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1179999999999998e-06 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366622+0000", 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000159746 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366765+0000", 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000142237 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.366907+0000", 2024-08-29T15:24:28.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014195399999999999 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.369648+0000", 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002740957 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.370870+0000", 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001222835 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474094+0000", 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10322338 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474180+0000", 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6434999999999995e-05 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:11.474242+0000", 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1937e-05 2024-08-29T15:24:28.203 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124391 6.1 6:80fa3010:::benchmark_data_smithi027_169445_object124390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 449.01388025, 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1538895120000001, 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.204 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124391 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011195+0000", 2024-08-29T15:24:28.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011196+0000", 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3300000000000001e-07 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011216+0000", 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0068000000000002e-05 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011228+0000", 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.243e-05 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.011234+0000", 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2890000000000003e-06 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.018714+0000", 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007479388 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.020057+0000", 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001342768 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.020202+0000", 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014581300000000001 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.035616+0000", 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015413736000000001 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.035626+0000", 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0301e-05 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025526+0000", 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98989923499999999 2024-08-29T15:24:28.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025540+0000", 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4188e-05 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.164991+0000", 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139451225 2024-08-29T15:24:28.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165084+0000", 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3238000000000004e-05 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124405 6.1 6:81409361:::benchmark_data_smithi027_169445_object124404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 448.99728689199998, 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1459181940000001, 2024-08-29T15:24:28.211 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124405 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027788+0000", 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027789+0000", 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.24e-06 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027810+0000", 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0891999999999999e-05 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027833+0000", 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3217000000000001e-05 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027843+0000", 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1419999999999999e-06 2024-08-29T15:24:28.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.027929+0000", 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6318999999999998e-05 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.028098+0000", 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016911500000000001 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 7", 2024-08-29T15:24:28.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.028250+0000", 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015169000000000001 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.036231+0000", 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079816360000000003 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.036241+0000", 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9119999999999993e-06 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025573+0000", 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98933212100000001 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025582+0000", 2024-08-29T15:24:28.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2900000000000002e-06 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165270+0000", 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1396881759999999 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165781+0000", 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00051096900000000001 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173600+0000", 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0078194070000000004 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173706+0000", 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106068 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:124438 6.1 6:8a3680bc:::benchmark_data_smithi027_169445_object124437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:24:28.219 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 448.95537321400002, 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.104159272, 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124438 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069702+0000", 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069704+0000", 2024-08-29T15:24:28.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.719e-06 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069728+0000", 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4142999999999999e-05 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069747+0000", 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9177e-05 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069755+0000", 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4859999999999997e-06 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069844+0000", 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8729999999999999e-05 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:58.069853+0000", 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0489999999999993e-06 2024-08-29T15:24:28.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025603+0000", 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.955749607 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:16:59.025609+0000", 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5390000000000004e-06 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165847+0000", 2024-08-29T15:24:28.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1402372869999999 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.165953+0000", 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106016 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.166064+0000", 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111908 2024-08-29T15:24:28.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.168561+0000", 2024-08-29T15:24:28.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024969829999999999 2024-08-29T15:24:28.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.169970+0000", 2024-08-29T15:24:28.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014081149999999999 2024-08-29T15:24:28.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173730+0000", 2024-08-29T15:24:28.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037604190000000001 2024-08-29T15:24:28.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173779+0000", 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9508000000000001e-05 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:00.173861+0000", 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1586000000000001e-05 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.229 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140304 6.4 6:2a4ae72d:::benchmark_data_smithi027_169445_object14877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 425.845373268, 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63546416900000002, 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140304 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179702+0000", 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179704+0000", 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0020000000000001e-06 2024-08-29T15:24:28.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179709+0000", 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6659999999999996e-06 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179727+0000", 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7693000000000002e-05 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179735+0000", 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7819999999999994e-06 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179797+0000", 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1838e-05 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.179805+0000", 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3289999999999994e-06 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810640+0000", 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63083506700000003 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810759+0000", 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011892699999999999 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3", 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810883+0000", 2024-08-29T15:24:28.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124098 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814964+0000", 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040809540000000004 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815029+0000", 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4528000000000003e-05 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815081+0000", 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2151999999999998e-05 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815166+0000", 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5133e-05 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140324 6.d 6:b4e34602:::benchmark_data_smithi027_169445_object14897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:24:28.236 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 425.82406260400001, 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.61402909999999999, 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140324 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201012+0000", 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201014+0000", 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6649999999999999e-06 2024-08-29T15:24:28.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201019+0000", 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0340000000000004e-06 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201033+0000", 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4331000000000001e-05 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201041+0000", 2024-08-29T15:24:28.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2200000000000003e-06 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201098+0000", 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7484000000000001e-05 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.201106+0000", 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8949999999999998e-06 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.786966+0000", 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58585960400000003 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.790090+0000", 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003124258 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:24:28.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.790556+0000", 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046574499999999999 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806640+0000", 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016084317000000001 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814905+0000", 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082646869999999997 2024-08-29T15:24:28.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814955+0000", 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9997999999999998e-05 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815042+0000", 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6861999999999996e-05 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140335 6.d 6:b3ea1342:::benchmark_data_smithi027_169445_object14908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 425.80751372200001, 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59763891800000002, 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140335 2024-08-29T15:24:28.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217561+0000", 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217563+0000", 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.88e-06 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217570+0000", 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0049999999999998e-06 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217589+0000", 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8686e-05 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217598+0000", 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6419999999999996e-06 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217665+0000", 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7447000000000007e-05 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.217675+0000", 2024-08-29T15:24:28.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7799999999999995e-06 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802631+0000", 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58495612900000005 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802735+0000", 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104397 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.802810+0000", 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5044999999999997e-05 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.814892+0000", 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01208188 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815054+0000", 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016216000000000001 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815094+0000", 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.015e-05 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815200+0000", 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105717 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140337 6.d 6:bf19c6d4:::benchmark_data_smithi027_169445_object14910:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 425.804188039, 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59452165300000004, 2024-08-29T15:24:28.251 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140337 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220887+0000", 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220889+0000", 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.708e-06 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220896+0000", 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8249999999999999e-06 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220914+0000", 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8476000000000001e-05 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.220923+0000", 2024-08-29T15:24:28.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1209999999999996e-06 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.221258+0000", 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033503 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.221267+0000", 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7930000000000003e-06 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806696+0000", 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58542917100000003 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806843+0000", 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000146623 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.806962+0000", 2024-08-29T15:24:28.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011940200000000001 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815264+0000", 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083018880000000003 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815277+0000", 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2835e-05 2024-08-29T15:24:28.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815315+0000", 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7991000000000003e-05 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815409+0000", 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3789999999999995e-05 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.258 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140341 6.d 6:ba4d184c:::benchmark_data_smithi027_169445_object14914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 425.790010327, 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58419336799999999, 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.259 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140341 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235065+0000", 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235067+0000", 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.984e-06 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235074+0000", 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0840000000000003e-06 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235095+0000", 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1307000000000001e-05 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235106+0000", 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0665999999999999e-05 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235173+0000", 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7220999999999999e-05 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.235185+0000", 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2189e-05 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.810910+0000", 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57572489900000001 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.811020+0000", 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110179 2024-08-29T15:24:28.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.811153+0000", 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013316200000000001 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.815416+0000", 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042625670000000001 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819099+0000", 2024-08-29T15:24:28.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003683151 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819162+0000", 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2558000000000007e-05 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.819258+0000", 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6401000000000001e-05 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.265 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:159235 6.4 6:2b791eb5:::benchmark_data_smithi027_169445_object33808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:21:26.250557+0000", 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 180.774517672, 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018851414, 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.266 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 159235 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250557+0000", 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250557+0000", 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250559+0000", 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0590000000000001e-06 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250568+0000", 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0999999999999993e-06 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250589+0000", 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0792e-05 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250598+0000", 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7460000000000006e-06 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250664+0000", 2024-08-29T15:24:28.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5852000000000006e-05 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250771+0000", 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107025 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5", 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.250877+0000", 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010566299999999999 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.260961+0000", 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010084305 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.269145+0000", 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0081843179999999995 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.269343+0000", 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019761099999999999 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.269378+0000", 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5601e-05 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:26.269409+0000", 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0341999999999999e-05 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.272 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:159348 6.4 6:2c01b7b0:::benchmark_data_smithi027_169445_object33921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:21:28.033726+0000", 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 178.99134866099999, 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019258416, 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 159348 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.273 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033726+0000", 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033726+0000", 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033728+0000", 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9709999999999998e-06 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033735+0000", 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5830000000000003e-06 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033752+0000", 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7414999999999998e-05 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033763+0000", 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0638999999999999e-05 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033858+0000", 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4829999999999998e-05 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.033988+0000", 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013019099999999999 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5", 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.034144+0000", 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000155804 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.037396+0000", 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003252301 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.037412+0000", 2024-08-29T15:24:28.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5789000000000001e-05 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.052914+0000", 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015501832 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.052951+0000", 2024-08-29T15:24:28.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7248e-05 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:28.052985+0000", 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3813000000000003e-05 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:162792 6.4 6:2928dff7:::benchmark_data_smithi027_169445_object37365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e249)", 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:21:54.440620+0000", 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 152.58445467300001, 2024-08-29T15:24:28.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014898987000000001, 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 162792 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440620+0000", 2024-08-29T15:24:28.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440620+0000", 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440621+0000", 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9299999999999996e-07 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440625+0000", 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.591e-06 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440634+0000", 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0769999999999997e-06 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440638+0000", 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3769999999999998e-06 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440688+0000", 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0124000000000003e-05 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440801+0000", 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112364 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5", 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.440915+0000", 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011387000000000001 2024-08-29T15:24:28.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.443846+0000", 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029313160000000002 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.444396+0000", 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00054962700000000001 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.455424+0000", 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011028288000000001 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.455477+0000", 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2719999999999997e-05 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:21:54.455519+0000", 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2740000000000001e-05 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.285 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:180370 6.1 6:82f741a0:::benchmark_data_smithi027_169445_object54943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:04.693297+0000", 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 22.331777856999999, 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011075473000000001, 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.286 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 180370 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693297+0000", 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693297+0000", 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693299+0000", 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7689999999999999e-06 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693304+0000", 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5509999999999999e-06 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693321+0000", 2024-08-29T15:24:28.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6775000000000001e-05 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693330+0000", 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0019999999999995e-06 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693400+0000", 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9720000000000003e-05 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693571+0000", 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000171356 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.693760+0000", 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018845400000000001 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.697154+0000", 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033939119999999998 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.697202+0000", 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8742999999999997e-05 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.697254+0000", 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1198e-05 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.702669+0000", 2024-08-29T15:24:28.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054155660000000001 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.702733+0000", 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4135999999999997e-05 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.704373+0000", 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001639291 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.292 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:180403 6.1 6:8d2eab70:::benchmark_data_smithi027_169445_object54976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:04.777732+0000", 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 22.247343496999999, 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011565583000000001, 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.293 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 180403 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.777732+0000", 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.777732+0000", 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.777733+0000", 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8929999999999999e-06 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.777741+0000", 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.486e-06 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.777759+0000", 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8270999999999998e-05 2024-08-29T15:24:28.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.777768+0000", 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6379999999999993e-06 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.777845+0000", 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6803000000000003e-05 2024-08-29T15:24:28.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.778007+0000", 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000162211 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.778129+0000", 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012257699999999999 2024-08-29T15:24:28.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.781570+0000", 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003440153 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.781756+0000", 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018610400000000001 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.782308+0000", 2024-08-29T15:24:28.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00055272800000000005 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.787703+0000", 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053944869999999999 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.787746+0000", 2024-08-29T15:24:28.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2828999999999999e-05 2024-08-29T15:24:28.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:04.789297+0000", 2024-08-29T15:24:28.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001551403 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:182891 6.1 6:85296502:::benchmark_data_smithi027_169445_object57464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:21.818581+0000", 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 5.2064937230000004, 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01556274, 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.301 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 182891 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.818581+0000", 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.818581+0000", 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.818582+0000", 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8400000000000002e-07 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.818586+0000", 2024-08-29T15:24:28.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7060000000000002e-06 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.818598+0000", 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2480000000000001e-05 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.818606+0000", 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1380000000000001e-06 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.821497+0000", 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028916549999999999 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.821614+0000", 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116804 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.822078+0000", 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046406300000000002 2024-08-29T15:24:28.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.827089+0000", 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050113379999999997 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.827108+0000", 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8913999999999999e-05 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.827138+0000", 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9328000000000001e-05 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.832810+0000", 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056720149999999999 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.832869+0000", 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9307e-05 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:21.834144+0000", 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001275008 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:183011 6.1 6:8861e441:::benchmark_data_smithi027_169445_object57584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:22.731337+0000", 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.2937384180000002, 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095553240000000005, 2024-08-29T15:24:28.308 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 183011 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731337+0000", 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731337+0000", 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731338+0000", 2024-08-29T15:24:28.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7659999999999999e-06 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731344+0000", 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7309999999999998e-06 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731358+0000", 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3901e-05 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731367+0000", 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3980000000000005e-06 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731424+0000", 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6221000000000003e-05 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731532+0000", 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010798699999999999 2024-08-29T15:24:28.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.731630+0000", 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8628000000000005e-05 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.738924+0000", 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0072942479999999997 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.738992+0000", 2024-08-29T15:24:28.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7867000000000006e-05 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.739042+0000", 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0151999999999997e-05 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.739152+0000", 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109992 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.739207+0000", 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4462000000000001e-05 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.740892+0000", 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001684971 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:183012 6.1 6:81fc81a6:::benchmark_data_smithi027_169445_object57585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:22.732603+0000", 2024-08-29T15:24:28.315 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.292472375, 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016549611999999998, 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 183012 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.732603+0000", 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.732603+0000", 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.732604+0000", 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.057e-06 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.732608+0000", 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4689999999999999e-06 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.732623+0000", 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4528e-05 2024-08-29T15:24:28.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.732630+0000", 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6229999999999997e-06 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.735010+0000", 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023796220000000001 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.735163+0000", 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015316900000000001 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.735317+0000", 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000154322 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.739083+0000", 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003765301 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.739138+0000", 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4781000000000003e-05 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.740904+0000", 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001766175 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.747198+0000", 2024-08-29T15:24:28.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062943310000000002 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.747413+0000", 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021547299999999999 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.749152+0000", 2024-08-29T15:24:28.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017387609999999999 2024-08-29T15:24:28.323 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.323 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.323 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.323 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:183027 6.e e262/234 6:7d01f711:::benchmark_data_smithi027_169445_object57600:head v 262'11478, mlcod=181'5430)", 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:22.774108+0000", 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.2509665349999999, 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014093046, 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774108+0000", 2024-08-29T15:24:28.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774108+0000", 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774110+0000", 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.697e-06 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774120+0000", 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8209999999999997e-06 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774141+0000", 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1148000000000001e-05 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774151+0000", 2024-08-29T15:24:28.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3600000000000002e-06 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774220+0000", 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9939999999999998e-05 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.774247+0000", 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6313999999999999e-05 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.788140+0000", 2024-08-29T15:24:28.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013893167 2024-08-29T15:24:28.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.788202+0000", 2024-08-29T15:24:28.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1599000000000004e-05 2024-08-29T15:24:28.329 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.329 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.329 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:183323 6.1 6:8c5e5bd9:::benchmark_data_smithi027_169445_object57896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:24.396928+0000", 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 2.6281468449999998, 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010866818, 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:24:28.330 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 183323 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.396928+0000", 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.396928+0000", 2024-08-29T15:24:28.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.396930+0000", 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5039999999999999e-06 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.396935+0000", 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2449999999999998e-06 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.396948+0000", 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2917e-05 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.396954+0000", 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2720000000000003e-06 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.397008+0000", 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3637000000000002e-05 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.397111+0000", 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010329599999999999 2024-08-29T15:24:28.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.397204+0000", 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3317000000000001e-05 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.399883+0000", 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026784310000000002 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:24:28.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.400008+0000", 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012503199999999999 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.400340+0000", 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033204600000000001 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.406049+0000", 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057094310000000001 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.406100+0000", 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0992000000000002e-05 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:24.407795+0000", 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0016946979999999999 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:183654 6.d e263/218 6:b23ee373:::benchmark_data_smithi027_169445_object58227:head v 263'11571, mlcod=263'11569)", 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:26.827515+0000", 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.197559555, 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095049339999999996, 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827515+0000", 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.338 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827515+0000", 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827517+0000", 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1659999999999999e-06 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827525+0000", 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4039999999999995e-06 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827540+0000", 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4810999999999999e-05 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827551+0000", 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1012000000000001e-05 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827639+0000", 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7925999999999998e-05 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.827667+0000", 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8413999999999999e-05 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.836987+0000", 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093202509999999999 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:26.837020+0000", 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2950000000000001e-05 2024-08-29T15:24:28.341 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.342 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.342 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.342 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:24:28.342 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:24:28.342 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:24:28.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:28.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:28.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:28.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:28.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:28.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:28.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:28.591 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:28.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:28.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:28.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:28.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:28.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:28.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:28.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:28.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:28.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:28.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:24:28.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:28.971 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:28.971 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:28.986 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:28.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:29.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:29.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:29.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:29.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:29.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:29.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:29.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:24:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:24:29] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-08-29T15:24:29.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:29.357 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:29.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:29.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:29.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:29.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:29.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:29.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:29.708 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:29.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:29.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:29.853 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:29.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:29.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:29.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:29 smithi171 ceph-mon[90031]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 29893/202089 objects degraded (14.792%); 12526/202089 objects misplaced (6.198%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:29 smithi027 ceph-mon[98970]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 29893/202089 objects degraded (14.792%); 12526/202089 objects misplaced (6.198%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:29 smithi027 ceph-mon[93921]: pgmap v866: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 29893/202089 objects degraded (14.792%); 12526/202089 objects misplaced (6.198%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:30.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:30.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:30.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:30.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:30.207 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:30.231 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:30.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:30.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:30.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:30.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:30.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:30.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:30.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:30.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:30.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:30.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:30.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:30.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:30.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:30.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:30.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:30.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:30.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:30.857 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:30.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:30.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:24:30.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:31.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:31.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:31.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:31.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:31.268 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:31.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:31.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:31.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:31.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:31.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:31.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:31.619 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:31.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:31.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:31.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:31.765 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:24:31.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:31.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:31.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:31.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:31.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:31.994 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:32.016 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:32.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:31 smithi027 ceph-mon[98970]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 29893/201492 objects degraded (14.836%); 12526/201492 objects misplaced (6.217%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3084676160' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:32.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:31 smithi027 ceph-mon[93921]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 29893/201492 objects degraded (14.836%); 12526/201492 objects misplaced (6.217%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:32.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3084676160' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:32.101 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:32.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:32.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:32.242 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:32.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:32.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:31 smithi171 ceph-mon[90031]: pgmap v867: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 29893/201492 objects degraded (14.836%); 12526/201492 objects misplaced (6.217%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3084676160' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:32.367 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:32.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:32.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:32.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:32.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:32.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:32.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:32.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:32.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:32.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.771 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:32.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:32.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:32.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:32.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:32.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:33.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:33.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:33.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:24:33.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3084676160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:33.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[98970]: osdmap e264: 8 total, 8 up, 8 in 2024-08-29T15:24:33.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29893/201492 objects degraded (14.836%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:33.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:24:33.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3084676160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:33.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[93921]: osdmap e264: 8 total, 8 up, 8 in 2024-08-29T15:24:33.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29893/201492 objects degraded (14.836%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:33.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:33.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:33.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:33.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:32 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:24:33.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3084676160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:33.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:32 smithi171 ceph-mon[90031]: osdmap e264: 8 total, 8 up, 8 in 2024-08-29T15:24:33.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29893/201492 objects degraded (14.836%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:33.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:33.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:33.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:33.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:33.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:33.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:33.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:33.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:33.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:33.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:33.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:33.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:33.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:33.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:33.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:33.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:33 smithi027 ceph-mon[93921]: pgmap v869: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 29833/199734 objects degraded (14.936%); 12412/199734 objects misplaced (6.214%); 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:24:33.999 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:33 smithi027 ceph-mon[98970]: pgmap v869: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 29833/199734 objects degraded (14.936%); 12412/199734 objects misplaced (6.214%); 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:24:34.002 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:34.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:34.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:34.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:34.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:34.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:34.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:34.253 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:24:34.253 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:24:34.265 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:33 smithi171 ceph-mon[90031]: pgmap v869: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 29833/199734 objects degraded (14.936%); 12412/199734 objects misplaced (6.214%); 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:24:34.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:34.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:24:34.288 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:24:34.313 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:24:34.313 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:24:34.333 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:24:34.333 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:24:34.356 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:24:34.356 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:24:34.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:34.377 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:24:34.378 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:24:34.398 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:24:34.398 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:24:34.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:34.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:34.417 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:24:34.417 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:24:34.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:34.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:34.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:34.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:24:34.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:34.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:34.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:34.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:34.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:34.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:34.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:35.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:35.038 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3411493415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:35.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3411493415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:35.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:35.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:35.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:35.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3411493415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:35.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:35.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:35.402 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:24:35.402 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:24:35.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:35.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:35.539 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:35.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:35.641 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:35.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:35.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:35.791 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:35.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:35.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:35.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:35.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:35.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:36.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:36.047 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:36.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:36.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:35 smithi027 ceph-mon[93921]: pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 29833/199329 objects degraded (14.967%); 12362/199329 objects misplaced (6.202%); 1.3 MiB/s, 20 objects/s recovering 2024-08-29T15:24:36.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:35 smithi027 ceph-mon[98970]: pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 29833/199329 objects degraded (14.967%); 12362/199329 objects misplaced (6.202%); 1.3 MiB/s, 20 objects/s recovering 2024-08-29T15:24:36.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:36.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:24:36.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:35 smithi171 ceph-mon[90031]: pgmap v870: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 29833/199329 objects degraded (14.967%); 12362/199329 objects misplaced (6.202%); 1.3 MiB/s, 20 objects/s recovering 2024-08-29T15:24:36.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:36.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:36.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:36.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:36.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:36.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:36.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:36.567 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:36.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:36.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:36.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:36.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:36.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:36.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:36.884 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:36.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:36.926 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:36.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:36.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:37.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:37.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:37.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:37.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:24:37.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:37.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:37.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:37.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:37.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:37.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:37.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:37.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:37.665 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:37.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:37.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:37.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:37.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:37.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:37.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:38.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:38.064 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:38.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:37 smithi027 ceph-mon[98970]: pgmap v871: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29757/196950 objects degraded (15.109%); 12263/196950 objects misplaced (6.226%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29757/196950 objects degraded (15.109%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:37 smithi027 ceph-mon[93921]: pgmap v871: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29757/196950 objects degraded (15.109%); 12263/196950 objects misplaced (6.226%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29757/196950 objects degraded (15.109%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:38.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:38.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:37 smithi171 ceph-mon[90031]: pgmap v871: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29757/196950 objects degraded (15.109%); 12263/196950 objects misplaced (6.226%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29757/196950 objects degraded (15.109%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:38.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:38.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:38.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:38.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:38.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:38.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:38.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:38.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:38.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:38.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:38.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:38.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:38.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:38.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:38.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:39.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:39.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:39.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:39.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:39.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:39.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.316 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:39.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:24:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:24:39] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-08-29T15:24:39.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:39.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:39.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:39.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:39.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:39.670 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:39.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:39.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:39.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:39.847 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:39.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:39.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:39.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:39.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:40.021 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:24:40.022 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:24:40.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:40.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:39 smithi027 ceph-mon[93921]: pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29757/196800 objects degraded (15.120%); 12212/196800 objects misplaced (6.205%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:40.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:39 smithi027 ceph-mon[98970]: pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29757/196800 objects degraded (15.120%); 12212/196800 objects misplaced (6.205%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:40.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:40.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:40.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:40.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:39 smithi171 ceph-mon[90031]: pgmap v872: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29757/196800 objects degraded (15.120%); 12212/196800 objects misplaced (6.205%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:40.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:40.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:40.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:40.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:40.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.440 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:40.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:40.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:40.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:40.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.700 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:40.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:40.798 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:24:40.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:40.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:40.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:40.950 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:40.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:41.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1324807862' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:40 smithi027 ceph-mon[98970]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:41.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1324807862' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:40 smithi027 ceph-mon[93921]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:41.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:41.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:41.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1324807862' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:24:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:40 smithi171 ceph-mon[90031]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:24:41.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:41.322 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:41.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:41.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:41.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:41.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:41.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:41.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:41.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:41.696 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:41.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:41.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:41.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:41.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:41.946 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:41.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:42.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[98970]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 29757/196071 objects degraded (15.177%); 12212/196071 objects misplaced (6.228%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1324807862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:24:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[98970]: osdmap e265: 8 total, 8 up, 8 in 2024-08-29T15:24:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[93921]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 29757/196071 objects degraded (15.177%); 12212/196071 objects misplaced (6.228%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1324807862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:24:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:41 smithi027 ceph-mon[93921]: osdmap e265: 8 total, 8 up, 8 in 2024-08-29T15:24:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.070 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:42.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:42.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:42.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:41 smithi171 ceph-mon[90031]: pgmap v873: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 29757/196071 objects degraded (15.177%); 12212/196071 objects misplaced (6.228%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:24:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:41 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1324807862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:24:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:41 smithi171 ceph-mon[90031]: osdmap e265: 8 total, 8 up, 8 in 2024-08-29T15:24:42.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:42.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:42.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:42.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:42.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:42.568 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:42.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:42.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:42.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:42.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:42.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:42.939 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:42.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:43.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:43.044 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29757/196071 objects degraded (15.177%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:43.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29757/196071 objects degraded (15.177%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:43.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:43.070 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":872,"stamp":"2024-08-29T15:24:38.611417+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4302051325,"num_objects":65600,"num_object_clones":0,"num_object_copies":196800,"num_objects_missing_on_primary":5089,"num_objects_missing":5089,"num_objects_degraded":29757,"num_objects_misplaced":12212,"num_objects_unfound":0,"num_objects_dirty":65600,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":186401,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16722,"num_bytes_recovered":974720205,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125537,"ondisk_log_size":125537,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14577228,"kb_used_data":12665544,"kb_used_omap":43,"kb_used_meta":1911572,"kb_avail":735383988,"statfs":{"total":767960285184,"available":753033203712,"internally_reserved":0,"allocated":12969517056,"data_stored":12957485006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1957450220},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-115539968,"num_objects":-1763,"num_object_clones":0,"num_object_copies":-5289,"num_objects_missing_on_primary":-101,"num_objects_missing":-101,"num_objects_degraded":-136,"num_objects_misplaced":-314,"num_objects_unfound":0,"num_objects_dirty":-1763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1763,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":238,"num_bytes_recovered":15597568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003200"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":413,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.770911+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:24:31.770911+0000","last_peered":"2024-08-29T15:24:31.770911+0000","last_clean":"2024-08-29T15:24:31.770911+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:24:31.770911+0000","last_undegraded":"2024-08-29T15:24:31.770911+0000","last_fullsized":"2024-08-29T15:24:31.770911+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":42,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:29.818024+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:24:29.818024+0000","last_peered":"2024-08-29T15:24:29.818024+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:24:29.818024+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:24:29.818024+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":131,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744414+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:24:23.744414+0000","last_peered":"2024-08-29T15:24:23.744414+0000","last_clean":"2024-08-29T15:24:23.744414+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:24:23.744414+0000","last_undegraded":"2024-08-29T15:24:23.744414+0000","last_fullsized":"2024-08-29T15:24:23.744414+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":379,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.767337+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:24:31.767337+0000","last_peered":"2024-08-29T15:24:31.767337+0000","last_clean":"2024-08-29T15:24:31.767337+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:24:31.767337+0000","last_undegraded":"2024-08-29T15:24:31.767337+0000","last_fullsized":"2024-08-29T15:24:31.767337+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":379,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.770981+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:24:31.770981+0000","last_peered":"2024-08-29T15:24:31.770981+0000","last_clean":"2024-08-29T15:24:31.770981+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:24:31.770981+0000","last_undegraded":"2024-08-29T15:24:31.770981+0000","last_fullsized":"2024-08-29T15:24:31.770981+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:29.817936+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:24:29.817936+0000","last_peered":"2024-08-29T15:24:29.817936+0000","last_clean":"2024-08-29T15:24:29.817936+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:24:29.817936+0000","last_undegraded":"2024-08-29T15:24:29.817936+0000","last_fullsized":"2024-08-29T15:24:29.817936+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":190,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769088+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:24:31.769088+0000","last_peered":"2024-08-29T15:24:31.769088+0000","last_clean":"2024-08-29T15:24:31.769088+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:24:31.769088+0000","last_undegraded":"2024-08-29T15:24:31.769088+0000","last_fullsized":"2024-08-29T15:24:31.769088+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":133,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.813761+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:24:23.813761+0000","last_peered":"2024-08-29T15:24:23.813761+0000","last_clean":"2024-08-29T15:24:23.813761+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:24:23.813761+0000","last_undegraded":"2024-08-29T15:24:23.813761+0000","last_fullsized":"2024-08-29T15:24:23.813761+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"264'11722","reported_seq":19765,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:36.070577+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:24:36.070577+0000","last_peered":"2024-08-29T15:24:36.070577+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:24:36.070577+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"176'1700","ondisk_log_start":"176'1700","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274595840,"num_objects":4190,"num_object_clones":0,"num_object_copies":12570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11722,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":1265}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":41,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:23.744655+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:24:23.744655+0000","last_peered":"2024-08-29T15:24:23.744655+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:24:23.744655+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:24:23.744655+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":150,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744137+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:24:23.744137+0000","last_peered":"2024-08-29T15:24:23.744137+0000","last_clean":"2024-08-29T15:24:23.744137+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:24:23.744137+0000","last_undegraded":"2024-08-29T15:24:23.744137+0000","last_fullsized":"2024-08-29T15:24:23.744137+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"264'11479","reported_seq":11686,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:36.314763+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:24:36.314763+0000","last_peered":"2024-08-29T15:24:36.314763+0000","last_clean":"2024-08-29T15:24:36.314763+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:24:36.314763+0000","last_undegraded":"2024-08-29T15:24:36.314763+0000","last_fullsized":"2024-08-29T15:24:36.314763+0000","mapping_epoch":168,"log_start":"180'4728","ondisk_log_start":"180'4728","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266797056,"num_objects":4071,"num_object_clones":0,"num_object_copies":12213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11479,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":41,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:23.745762+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:24:23.745762+0000","last_peered":"2024-08-29T15:24:23.745762+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:24:23.745762+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:24:23.745762+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744271+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:24:23.744271+0000","last_peered":"2024-08-29T15:24:23.744271+0000","last_clean":"2024-08-29T15:24:23.744271+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:24:23.744271+0000","last_undegraded":"2024-08-29T15:24:23.744271+0000","last_fullsized":"2024-08-29T15:24:23.744271+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"264'11685","reported_seq":11861,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:36.371054+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:24:36.371054+0000","last_peered":"2024-08-29T15:24:36.371054+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:24:36.371054+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"180'4974","ondisk_log_start":"180'4974","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263913472,"num_objects":4027,"num_object_clones":0,"num_object_copies":12081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11685,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1550}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":195,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.767988+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:24:31.767988+0000","last_peered":"2024-08-29T15:24:31.767988+0000","last_clean":"2024-08-29T15:24:31.767988+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:24:31.767988+0000","last_undegraded":"2024-08-29T15:24:31.767988+0000","last_fullsized":"2024-08-29T15:24:31.767988+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":88,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.818572+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:24:23.818572+0000","last_peered":"2024-08-29T15:24:23.818572+0000","last_clean":"2024-08-29T15:24:23.818572+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:24:23.818572+0000","last_undegraded":"2024-08-29T15:24:23.818572+0000","last_fullsized":"2024-08-29T15:24:23.818572+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"264'11674","reported_seq":17101,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:33.515749+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:24:33.515749+0000","last_peered":"2024-08-29T15:24:33.515749+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:24:33.515749+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"178'3562","ondisk_log_start":"178'3562","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8112,"log_dups_size":0,"ondisk_log_size":8112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268959744,"num_objects":4104,"num_object_clones":0,"num_object_copies":12312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11674,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4104}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":150,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744944+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:24:23.744944+0000","last_peered":"2024-08-29T15:24:23.744944+0000","last_clean":"2024-08-29T15:24:23.744944+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:24:23.744944+0000","last_undegraded":"2024-08-29T15:24:23.744944+0000","last_fullsized":"2024-08-29T15:24:23.744944+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":393,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.771609+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:24:31.771609+0000","last_peered":"2024-08-29T15:24:31.771609+0000","last_clean":"2024-08-29T15:24:31.771609+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:24:31.771609+0000","last_undegraded":"2024-08-29T15:24:31.771609+0000","last_fullsized":"2024-08-29T15:24:31.771609+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"264'11595","reported_seq":11746,"reported_epoch":264,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:34.740783+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:24:34.740783+0000","last_peered":"2024-08-29T15:24:34.740783+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:24:34.740783+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:24:34.740783+0000","mapping_epoch":234,"log_start":"179'3633","ondisk_log_start":"179'3633","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7962,"log_dups_size":0,"ondisk_log_size":7962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276496384,"num_objects":4219,"num_object_clones":0,"num_object_copies":12657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11595,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":1742},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":387,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769601+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:24:31.769601+0000","last_peered":"2024-08-29T15:24:31.769601+0000","last_clean":"2024-08-29T15:24:31.769601+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:24:31.769601+0000","last_undegraded":"2024-08-29T15:24:31.769601+0000","last_fullsized":"2024-08-29T15:24:31.769601+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745244+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:24:23.745244+0000","last_peered":"2024-08-29T15:24:23.745244+0000","last_clean":"2024-08-29T15:24:23.745244+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:24:23.745244+0000","last_undegraded":"2024-08-29T15:24:23.745244+0000","last_fullsized":"2024-08-29T15:24:23.745244+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"264'11670","reported_seq":11893,"reported_epoch":264,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:36.363678+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:24:36.363678+0000","last_peered":"2024-08-29T15:24:36.363678+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:24:36.363678+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:24:36.363678+0000","mapping_epoch":234,"log_start":"180'4968","ondisk_log_start":"180'4968","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273416192,"num_objects":4172,"num_object_clones":0,"num_object_copies":12516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11670,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":1707},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":106,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.818527+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:24:23.818527+0000","last_peered":"2024-08-29T15:24:23.818527+0000","last_clean":"2024-08-29T15:24:23.818527+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:24:23.818527+0000","last_undegraded":"2024-08-29T15:24:23.818527+0000","last_fullsized":"2024-08-29T15:24:23.818527+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745564+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:24:23.745564+0000","last_peered":"2024-08-29T15:24:23.745564+0000","last_clean":"2024-08-29T15:24:23.745564+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:24:23.745564+0000","last_undegraded":"2024-08-29T15:24:23.745564+0000","last_fullsized":"2024-08-29T15:24:23.745564+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"264'11516","reported_seq":11924,"reported_epoch":264,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:24:36.385435+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:24:36.385435+0000","last_peered":"2024-08-29T15:24:36.385435+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:24:36.385435+0000","last_undegraded":"2024-08-29T15:24:36.385435+0000","last_fullsized":"2024-08-29T15:24:36.385435+0000","mapping_epoch":202,"log_start":"180'4767","ondisk_log_start":"180'4767","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271450112,"num_objects":4142,"num_object_clones":0,"num_object_copies":12426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4142,"num_objects_unfound":0,"num_objects_dirty":4142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11516,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4142}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745123+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:24:23.745123+0000","last_peered":"2024-08-29T15:24:23.745123+0000","last_clean":"2024-08-29T15:24:23.745123+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:24:23.745123+0000","last_undegraded":"2024-08-29T15:24:23.745123+0000","last_fullsized":"2024-08-29T15:24:23.745123+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":248,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745965+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:24:23.745965+0000","last_peered":"2024-08-29T15:24:23.745965+0000","last_clean":"2024-08-29T15:24:23.745965+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:24:23.745965+0000","last_undegraded":"2024-08-29T15:24:23.745965+0000","last_fullsized":"2024-08-29T15:24:23.745965+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"264'11678","reported_seq":11823,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:36.132804+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:24:36.132804+0000","last_peered":"2024-08-29T15:24:36.132804+0000","last_clean":"2024-08-29T15:24:36.132804+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:24:36.132804+0000","last_undegraded":"2024-08-29T15:24:36.132804+0000","last_fullsized":"2024-08-29T15:24:36.132804+0000","mapping_epoch":218,"log_start":"176'1578","ondisk_log_start":"176'1578","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267452440,"num_objects":4082,"num_object_clones":0,"num_object_copies":12246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4082,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11678,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":391,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.772153+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:24:31.772153+0000","last_peered":"2024-08-29T15:24:31.772153+0000","last_clean":"2024-08-29T15:24:31.772153+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:24:31.772153+0000","last_undegraded":"2024-08-29T15:24:31.772153+0000","last_fullsized":"2024-08-29T15:24:31.772153+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":383,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769154+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:24:31.769154+0000","last_peered":"2024-08-29T15:24:31.769154+0000","last_clean":"2024-08-29T15:24:31.769154+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:24:31.769154+0000","last_undegraded":"2024-08-29T15:24:31.769154+0000","last_fullsized":"2024-08-29T15:24:31.769154+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"264'11200","reported_seq":11393,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:36.056339+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:24:36.056339+0000","last_peered":"2024-08-29T15:24:36.056339+0000","last_clean":"2024-08-29T15:24:36.056339+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:24:36.056339+0000","last_undegraded":"2024-08-29T15:24:36.056339+0000","last_fullsized":"2024-08-29T15:24:36.056339+0000","mapping_epoch":201,"log_start":"175'1100","ondisk_log_start":"175'1100","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262668288,"num_objects":4008,"num_object_clones":0,"num_object_copies":12024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11200,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":482,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.768514+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:24:31.768514+0000","last_peered":"2024-08-29T15:24:31.768514+0000","last_clean":"2024-08-29T15:24:31.768514+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:24:31.768514+0000","last_undegraded":"2024-08-29T15:24:31.768514+0000","last_fullsized":"2024-08-29T15:24:31.768514+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":734,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744455+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:24:23.744455+0000","last_peered":"2024-08-29T15:24:23.744455+0000","last_clean":"2024-08-29T15:24:23.744455+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:24:23.744455+0000","last_undegraded":"2024-08-29T15:24:23.744455+0000","last_fullsized":"2024-08-29T15:24:23.744455+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":242,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744680+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:24:23.744680+0000","last_peered":"2024-08-29T15:24:23.744680+0000","last_clean":"2024-08-29T15:24:23.744680+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:24:23.744680+0000","last_undegraded":"2024-08-29T15:24:23.744680+0000","last_fullsized":"2024-08-29T15:24:23.744680+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":104,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.814191+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:24:23.814191+0000","last_peered":"2024-08-29T15:24:23.814191+0000","last_clean":"2024-08-29T15:24:23.814191+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:24:23.814191+0000","last_undegraded":"2024-08-29T15:24:23.814191+0000","last_fullsized":"2024-08-29T15:24:23.814191+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"264'11645","reported_seq":12050,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:35.641886+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:24:35.641886+0000","last_peered":"2024-08-29T15:24:35.641886+0000","last_clean":"2024-08-29T15:24:35.641886+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:24:35.641886+0000","last_undegraded":"2024-08-29T15:24:35.641886+0000","last_fullsized":"2024-08-29T15:24:35.641886+0000","mapping_epoch":201,"log_start":"180'4653","ondisk_log_start":"180'4653","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264568832,"num_objects":4037,"num_object_clones":0,"num_object_copies":12111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11645,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":856,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.768913+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:24:31.768913+0000","last_peered":"2024-08-29T15:24:31.768913+0000","last_clean":"2024-08-29T15:24:31.768913+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:24:31.768913+0000","last_undegraded":"2024-08-29T15:24:31.768913+0000","last_fullsized":"2024-08-29T15:24:31.768913+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":394,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.770478+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:24:31.770478+0000","last_peered":"2024-08-29T15:24:31.770478+0000","last_clean":"2024-08-29T15:24:31.770478+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:24:31.770478+0000","last_undegraded":"2024-08-29T15:24:31.770478+0000","last_fullsized":"2024-08-29T15:24:31.770478+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":401,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.771267+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:24:31.771267+0000","last_peered":"2024-08-29T15:24:31.771267+0000","last_clean":"2024-08-29T15:24:31.771267+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:24:31.771267+0000","last_undegraded":"2024-08-29T15:24:31.771267+0000","last_fullsized":"2024-08-29T15:24:31.771267+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"264'11547","reported_seq":18065,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:36.402303+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:24:36.402303+0000","last_peered":"2024-08-29T15:24:36.402303+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:24:36.402303+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"180'4830","ondisk_log_start":"180'4830","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6717,"log_dups_size":0,"ondisk_log_size":6717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4620,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11547,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3171,"num_bytes_recovered":207814656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1705},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":664,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.770265+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:24:31.770265+0000","last_peered":"2024-08-29T15:24:31.770265+0000","last_clean":"2024-08-29T15:24:31.770265+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:24:31.770265+0000","last_undegraded":"2024-08-29T15:24:31.770265+0000","last_fullsized":"2024-08-29T15:24:31.770265+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":95,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.813622+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:24:23.813622+0000","last_peered":"2024-08-29T15:24:23.813622+0000","last_clean":"2024-08-29T15:24:23.813622+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:24:23.813622+0000","last_undegraded":"2024-08-29T15:24:23.813622+0000","last_fullsized":"2024-08-29T15:24:23.813622+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":428,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.768966+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:24:31.768966+0000","last_peered":"2024-08-29T15:24:31.768966+0000","last_clean":"2024-08-29T15:24:31.768966+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:24:31.768966+0000","last_undegraded":"2024-08-29T15:24:31.768966+0000","last_fullsized":"2024-08-29T15:24:31.768966+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"264'11616","reported_seq":11833,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:36.344887+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:24:36.344887+0000","last_peered":"2024-08-29T15:24:36.344887+0000","last_clean":"2024-08-29T15:24:36.344887+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:24:36.344887+0000","last_undegraded":"2024-08-29T15:24:36.344887+0000","last_fullsized":"2024-08-29T15:24:36.344887+0000","mapping_epoch":234,"log_start":"180'4897","ondisk_log_start":"180'4897","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268435456,"num_objects":4096,"num_object_clones":0,"num_object_copies":12288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11616,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":202,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744342+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:24:23.744342+0000","last_peered":"2024-08-29T15:24:23.744342+0000","last_clean":"2024-08-29T15:24:23.744342+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:24:23.744342+0000","last_undegraded":"2024-08-29T15:24:23.744342+0000","last_fullsized":"2024-08-29T15:24:23.744342+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744660+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:24:23.744660+0000","last_peered":"2024-08-29T15:24:23.744660+0000","last_clean":"2024-08-29T15:24:23.744660+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:24:23.744660+0000","last_undegraded":"2024-08-29T15:24:23.744660+0000","last_fullsized":"2024-08-29T15:24:23.744660+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":109,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744502+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:24:23.744502+0000","last_peered":"2024-08-29T15:24:23.744502+0000","last_clean":"2024-08-29T15:24:23.744502+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:24:23.744502+0000","last_undegraded":"2024-08-29T15:24:23.744502+0000","last_fullsized":"2024-08-29T15:24:23.744502+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"264'11632","reported_seq":11847,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:33.541074+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:24:33.541074+0000","last_peered":"2024-08-29T15:24:33.541074+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:24:33.541074+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"178'3442","ondisk_log_start":"178'3442","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8190,"log_dups_size":0,"ondisk_log_size":8190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274857984,"num_objects":4194,"num_object_clones":0,"num_object_copies":12582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11632,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1771}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744340+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:24:23.744340+0000","last_peered":"2024-08-29T15:24:23.744340+0000","last_clean":"2024-08-29T15:24:23.744340+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:24:23.744340+0000","last_undegraded":"2024-08-29T15:24:23.744340+0000","last_fullsized":"2024-08-29T15:24:23.744340+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":821,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769568+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:24:31.769568+0000","last_peered":"2024-08-29T15:24:31.769568+0000","last_clean":"2024-08-29T15:24:31.769568+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:24:31.769568+0000","last_undegraded":"2024-08-29T15:24:31.769568+0000","last_fullsized":"2024-08-29T15:24:31.769568+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":398,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769426+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:24:31.769426+0000","last_peered":"2024-08-29T15:24:31.769426+0000","last_clean":"2024-08-29T15:24:31.769426+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:24:31.769426+0000","last_undegraded":"2024-08-29T15:24:31.769426+0000","last_fullsized":"2024-08-29T15:24:31.769426+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"264'11602","reported_seq":12373,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:35.706082+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:24:35.706082+0000","last_peered":"2024-08-29T15:24:35.706082+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:24:35.706082+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"180'4580","ondisk_log_start":"180'4580","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266469376,"num_objects":4066,"num_object_clones":0,"num_object_copies":12198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4038,"num_objects_unfound":0,"num_objects_dirty":4066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11602,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":1915}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":403,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769476+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:24:31.769476+0000","last_peered":"2024-08-29T15:24:31.769476+0000","last_clean":"2024-08-29T15:24:31.769476+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:24:31.769476+0000","last_undegraded":"2024-08-29T15:24:31.769476+0000","last_fullsized":"2024-08-29T15:24:31.769476+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":691,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:23.788657+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:24:23.788657+0000","last_peered":"2024-08-29T15:24:23.788657+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:24:23.788657+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:24:23.788657+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":394,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.771573+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:24:31.771573+0000","last_peered":"2024-08-29T15:24:31.771573+0000","last_clean":"2024-08-29T15:24:31.771573+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:24:31.771573+0000","last_undegraded":"2024-08-29T15:24:31.771573+0000","last_fullsized":"2024-08-29T15:24:31.771573+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"264'11453","reported_seq":18602,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:34.684354+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:24:34.684354+0000","last_peered":"2024-08-29T15:24:34.684354+0000","last_clean":"2024-08-29T15:24:34.684354+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:24:34.684354+0000","last_undegraded":"2024-08-29T15:24:34.684354+0000","last_fullsized":"2024-08-29T15:24:34.684354+0000","mapping_epoch":234,"log_start":"178'3499","ondisk_log_start":"178'3499","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7954,"log_dups_size":0,"ondisk_log_size":7954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260505600,"num_objects":3975,"num_object_clones":0,"num_object_copies":11925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11453,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":388,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.772499+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:24:31.772499+0000","last_peered":"2024-08-29T15:24:31.772499+0000","last_clean":"2024-08-29T15:24:31.772499+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:24:31.772499+0000","last_undegraded":"2024-08-29T15:24:31.772499+0000","last_fullsized":"2024-08-29T15:24:31.772499+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1010,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:31.769895+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:24:31.769895+0000","last_peered":"2024-08-29T15:24:31.769895+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:24:31.769895+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":385,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769929+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:24:31.769929+0000","last_peered":"2024-08-29T15:24:31.769929+0000","last_clean":"2024-08-29T15:24:31.769929+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:24:31.769929+0000","last_undegraded":"2024-08-29T15:24:31.769929+0000","last_fullsized":"2024-08-29T15:24:31.769929+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"264'11564","reported_seq":17009,"reported_epoch":264,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:37.902035+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:24:37.902035+0000","last_peered":"2024-08-29T15:24:37.902035+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:24:37.902035+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"179'4012","ondisk_log_start":"179'4012","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7552,"log_dups_size":0,"ondisk_log_size":7552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265158656,"num_objects":4046,"num_object_clones":0,"num_object_copies":12138,"num_objects_missing_on_primary":5081,"num_objects_missing":5081,"num_objects_degraded":2313,"num_objects_misplaced":4032,"num_objects_unfound":0,"num_objects_dirty":4046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11564,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2641,"num_bytes_recovered":172949504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":1661},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":383,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.767477+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:24:31.767477+0000","last_peered":"2024-08-29T15:24:31.767477+0000","last_clean":"2024-08-29T15:24:31.767477+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:24:31.767477+0000","last_undegraded":"2024-08-29T15:24:31.767477+0000","last_fullsized":"2024-08-29T15:24:31.767477+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":115,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.814055+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:24:23.814055+0000","last_peered":"2024-08-29T15:24:23.814055+0000","last_clean":"2024-08-29T15:24:23.814055+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:24:23.814055+0000","last_undegraded":"2024-08-29T15:24:23.814055+0000","last_fullsized":"2024-08-29T15:24:23.814055+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745198+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:24:23.745198+0000","last_peered":"2024-08-29T15:24:23.745198+0000","last_clean":"2024-08-29T15:24:23.745198+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:24:23.745198+0000","last_undegraded":"2024-08-29T15:24:23.745198+0000","last_fullsized":"2024-08-29T15:24:23.745198+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":43,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:29.818128+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:24:29.818128+0000","last_peered":"2024-08-29T15:24:29.818128+0000","last_clean":"2024-08-29T15:24:29.818128+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:24:29.818128+0000","last_undegraded":"2024-08-29T15:24:29.818128+0000","last_fullsized":"2024-08-29T15:24:29.818128+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":41,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:29.818156+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:24:29.818156+0000","last_peered":"2024-08-29T15:24:29.818156+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:24:29.818156+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:24:29.818156+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":392,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769266+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:24:31.769266+0000","last_peered":"2024-08-29T15:24:31.769266+0000","last_clean":"2024-08-29T15:24:31.769266+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:24:31.769266+0000","last_undegraded":"2024-08-29T15:24:31.769266+0000","last_fullsized":"2024-08-29T15:24:31.769266+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":142,"reported_epoch":264,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:31.773631+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:24:31.773631+0000","last_peered":"2024-08-29T15:24:31.773631+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:24:31.773631+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:24:31.773631+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":131,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745049+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:24:23.745049+0000","last_peered":"2024-08-29T15:24:23.745049+0000","last_clean":"2024-08-29T15:24:23.745049+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:24:23.745049+0000","last_undegraded":"2024-08-29T15:24:23.745049+0000","last_fullsized":"2024-08-29T15:24:23.745049+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":351,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.768544+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:24:31.768544+0000","last_peered":"2024-08-29T15:24:31.768544+0000","last_clean":"2024-08-29T15:24:31.768544+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:24:31.768544+0000","last_undegraded":"2024-08-29T15:24:31.768544+0000","last_fullsized":"2024-08-29T15:24:31.768544+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":405,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.767886+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:24:31.767886+0000","last_peered":"2024-08-29T15:24:31.767886+0000","last_clean":"2024-08-29T15:24:31.767886+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:24:31.767886+0000","last_undegraded":"2024-08-29T15:24:31.767886+0000","last_fullsized":"2024-08-29T15:24:31.767886+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":403,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769835+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:24:31.769835+0000","last_peered":"2024-08-29T15:24:31.769835+0000","last_clean":"2024-08-29T15:24:31.769835+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:24:31.769835+0000","last_undegraded":"2024-08-29T15:24:31.769835+0000","last_fullsized":"2024-08-29T15:24:31.769835+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.814124+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:24:23.814124+0000","last_peered":"2024-08-29T15:24:23.814124+0000","last_clean":"2024-08-29T15:24:23.814124+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:24:23.814124+0000","last_undegraded":"2024-08-29T15:24:23.814124+0000","last_fullsized":"2024-08-29T15:24:23.814124+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":260,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.768425+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:24:31.768425+0000","last_peered":"2024-08-29T15:24:31.768425+0000","last_clean":"2024-08-29T15:24:31.768425+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:24:31.768425+0000","last_undegraded":"2024-08-29T15:24:31.768425+0000","last_fullsized":"2024-08-29T15:24:31.768425+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":380,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.771146+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:24:31.771146+0000","last_peered":"2024-08-29T15:24:31.771146+0000","last_clean":"2024-08-29T15:24:31.771146+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:24:31.771146+0000","last_undegraded":"2024-08-29T15:24:31.771146+0000","last_fullsized":"2024-08-29T15:24:31.771146+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":383,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.770943+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:24:31.770943+0000","last_peered":"2024-08-29T15:24:31.770943+0000","last_clean":"2024-08-29T15:24:31.770943+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:24:31.770943+0000","last_undegraded":"2024-08-29T15:24:31.770943+0000","last_fullsized":"2024-08-29T15:24:31.770943+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":388,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.773502+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:24:31.773502+0000","last_peered":"2024-08-29T15:24:31.773502+0000","last_clean":"2024-08-29T15:24:31.773502+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:24:31.773502+0000","last_undegraded":"2024-08-29T15:24:31.773502+0000","last_fullsized":"2024-08-29T15:24:31.773502+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":42,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745333+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:24:23.745333+0000","last_peered":"2024-08-29T15:24:23.745333+0000","last_clean":"2024-08-29T15:24:23.745333+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:24:23.745333+0000","last_undegraded":"2024-08-29T15:24:23.745333+0000","last_fullsized":"2024-08-29T15:24:23.745333+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":408,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.769655+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:24:31.769655+0000","last_peered":"2024-08-29T15:24:31.769655+0000","last_clean":"2024-08-29T15:24:31.769655+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:24:31.769655+0000","last_undegraded":"2024-08-29T15:24:31.769655+0000","last_fullsized":"2024-08-29T15:24:31.769655+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":371,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.771471+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:24:31.771471+0000","last_peered":"2024-08-29T15:24:31.771471+0000","last_clean":"2024-08-29T15:24:31.771471+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:24:31.771471+0000","last_undegraded":"2024-08-29T15:24:31.771471+0000","last_fullsized":"2024-08-29T15:24:31.771471+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":42,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744579+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:24:23.744579+0000","last_peered":"2024-08-29T15:24:23.744579+0000","last_clean":"2024-08-29T15:24:23.744579+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:24:23.744579+0000","last_undegraded":"2024-08-29T15:24:23.744579+0000","last_fullsized":"2024-08-29T15:24:23.744579+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":155,"reported_epoch":263,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:23.744663+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:24:23.744663+0000","last_peered":"2024-08-29T15:24:23.744663+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:24:23.744663+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:24:23.744663+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745371+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:24:23.745371+0000","last_peered":"2024-08-29T15:24:23.745371+0000","last_clean":"2024-08-29T15:24:23.745371+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:24:23.745371+0000","last_undegraded":"2024-08-29T15:24:23.745371+0000","last_fullsized":"2024-08-29T15:24:23.745371+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":161,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.745155+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:24:23.745155+0000","last_peered":"2024-08-29T15:24:23.745155+0000","last_clean":"2024-08-29T15:24:23.745155+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:24:23.745155+0000","last_undegraded":"2024-08-29T15:24:23.745155+0000","last_fullsized":"2024-08-29T15:24:23.745155+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":419,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.770460+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:24:31.770460+0000","last_peered":"2024-08-29T15:24:31.770460+0000","last_clean":"2024-08-29T15:24:31.770460+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:24:31.770460+0000","last_undegraded":"2024-08-29T15:24:31.770460+0000","last_fullsized":"2024-08-29T15:24:31.770460+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":420,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.770252+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:24:31.770252+0000","last_peered":"2024-08-29T15:24:31.770252+0000","last_clean":"2024-08-29T15:24:31.770252+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:24:31.770252+0000","last_undegraded":"2024-08-29T15:24:31.770252+0000","last_fullsized":"2024-08-29T15:24:31.770252+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":42,"reported_epoch":263,"state":"active+clean","last_fresh":"2024-08-29T15:24:23.744150+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:24:23.744150+0000","last_peered":"2024-08-29T15:24:23.744150+0000","last_clean":"2024-08-29T15:24:23.744150+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:24:23.744150+0000","last_undegraded":"2024-08-29T15:24:23.744150+0000","last_fullsized":"2024-08-29T15:24:23.744150+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":428,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-08-29T15:24:31.767782+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:24:31.767782+0000","last_peered":"2024-08-29T15:24:31.767782+0000","last_clean":"2024-08-29T15:24:31.767782+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:24:31.767782+0000","last_undegraded":"2024-08-29T15:24:31.767782+0000","last_fullsized":"2024-08-29T15:24:31.767782+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4297261080,"num_objects":65572,"num_object_clones":0,"num_object_copies":196716,"num_objects_missing_on_primary":5081,"num_objects_missing":5081,"num_objects_degraded":29728,"num_objects_misplaced":12212,"num_objects_unfound":0,"num_objects_dirty":65572,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":185278,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16613,"num_bytes_recovered":968753152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12910211072,"data_stored":12910198856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124355,"ondisk_log_size":124355,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234359,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2438872,"kb_used_data":2110800,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":91306280,"statfs":{"total":95995035648,"available":93497630720,"internally_reserved":0,"allocated":2161459200,"data_stored":2159943547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":335935947},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495991,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265492,"kb_used_data":94092,"kb_used_omap":1,"kb_used_meta":171390,"kb_avail":93479660,"statfs":{"total":95995035648,"available":95723171840,"internally_reserved":0,"allocated":96350208,"data_stored":94838355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":175503815},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426577,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1398152,"kb_used_data":1316800,"kb_used_omap":8,"kb_used_meta":81335,"kb_avail":92347000,"statfs":{"total":95995035648,"available":94563328000,"internally_reserved":0,"allocated":1348403200,"data_stored":1346898813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":83287946},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019269,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1648696,"kb_used_data":1320624,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":92096456,"statfs":{"total":95995035648,"available":94306770944,"internally_reserved":0,"allocated":1352318976,"data_stored":1350830372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":335935942},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435454,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2907140,"kb_used_data":2618684,"kb_used_omap":4,"kb_used_meta":288443,"kb_avail":90838012,"statfs":{"total":95995035648,"available":93018124288,"internally_reserved":0,"allocated":2681532416,"data_stored":2680031969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":295365897},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903306,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2510664,"kb_used_data":2387264,"kb_used_omap":10,"kb_used_meta":123381,"kb_avail":91234488,"statfs":{"total":95995035648,"available":93424115712,"internally_reserved":0,"allocated":2444558336,"data_stored":2443057825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":126342672},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762507,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010500,"kb_used_data":1744188,"kb_used_omap":7,"kb_used_meta":266296,"kb_avail":91734652,"statfs":{"total":95995035648,"available":93936283648,"internally_reserved":0,"allocated":1786048512,"data_stored":1784546825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":272687458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926033,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1397712,"kb_used_data":1073092,"kb_used_omap":7,"kb_used_meta":324600,"kb_avail":92347440,"statfs":{"total":95995035648,"available":94563778560,"internally_reserved":0,"allocated":1098846208,"data_stored":1097337300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":332390543},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1090195456,"data_stored":1090191384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1777078272,"data_stored":1777074200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2439118848,"data_stored":2439118848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2672164864,"data_stored":2672164864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1345716224,"data_stored":1345716224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1340542976,"data_stored":1340538904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":92209152,"data_stored":92209152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2153185280,"data_stored":2153185280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:24:43.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:24:43.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:43.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:43.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29757/196071 objects degraded (15.177%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:43.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:43.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:43.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:43.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:43.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:43.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:43.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:24:43.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:43.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:43.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:43.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:43.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:43.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:43.874 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:43 smithi027 ceph-mon[98970]: pgmap v875: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 29709/194493 objects degraded (15.275%); 12114/194493 objects misplaced (6.229%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:43.874 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:43.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:43 smithi027 ceph-mon[93921]: pgmap v875: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 29709/194493 objects degraded (15.275%); 12114/194493 objects misplaced (6.229%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:43.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:43.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:43.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:43.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:44.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:44.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:44.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:44.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:44.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:43 smithi171 ceph-mon[90031]: pgmap v875: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 29709/194493 objects degraded (15.275%); 12114/194493 objects misplaced (6.229%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:44.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:44.316 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:44.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:44.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:44.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:44.467 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:24:44.467 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:24:44.467 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:24:44.467 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:24:44.467 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:24:44.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:44.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:44.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:44.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:44.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:44.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:44.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:44.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:44.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:44.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:45.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:45.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:45.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2300727182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2300727182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:45.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:45.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:45.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2300727182' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:45.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:45.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:24:45.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:45.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:45.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:45.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:45.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:45.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:45.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:24:45.681 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:24:45.682 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:24:45.682 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:24:45.682 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:24:45.682 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:24:45.682 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:24:45.682 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:24:45.683 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:24:45.683 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:24:45.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:45.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:45.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:45.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:45.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:45.936 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:45.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:46.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:45 smithi027 ceph-mon[98970]: pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 29709/194118 objects degraded (15.305%); 12061/194118 objects misplaced (6.213%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:46.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:45 smithi027 ceph-mon[93921]: pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 29709/194118 objects degraded (15.305%); 12061/194118 objects misplaced (6.213%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:46.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:46.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:46.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:46.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:45 smithi171 ceph-mon[90031]: pgmap v876: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 29709/194118 objects degraded (15.305%); 12061/194118 objects misplaced (6.213%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:46.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:46.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:46.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:46.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:46.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:46.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:46.466 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:46.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:46.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:46.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:46.668 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:46.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:46.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:46.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:46.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:46.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:46.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:46.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:46.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:47.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:47.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:47.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:47.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:47.296 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:47.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:47.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:47.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:47.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:47.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:47.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:47.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:47.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:47.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:47.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:47.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:47.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:47.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:47.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:47.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:47.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:48.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:48.064 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:47 smithi027 ceph-mon[93921]: pgmap v877: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 29671/192645 objects degraded (15.402%); 11977/192645 objects misplaced (6.217%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:48.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29671/192645 objects degraded (15.402%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:48.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:47 smithi027 ceph-mon[98970]: pgmap v877: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 29671/192645 objects degraded (15.402%); 11977/192645 objects misplaced (6.217%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:48.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29671/192645 objects degraded (15.402%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:48.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:48.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:48.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:48.189 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:48.202 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:47 smithi171 ceph-mon[90031]: pgmap v877: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 29671/192645 objects degraded (15.402%); 11977/192645 objects misplaced (6.217%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:24:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29671/192645 objects degraded (15.402%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:48.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:48.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:48.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:48.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:48.442 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:48.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:48.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:24:48.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:48.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:48.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:48.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:48.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:48.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:48.815 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:24:48.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:48.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:48.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:48.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:48.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:48.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:49.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/250545719' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:49.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/250545719' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:49.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:49.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:49.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:49.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:49.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/250545719' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:24:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:24:49] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-08-29T15:24:49.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:49.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:49.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:49.476 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:49.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:49.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:49.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:49.740 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:49.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:49.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:49.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:49.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[98970]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 29671/192486 objects degraded (15.415%); 11924/192486 objects misplaced (6.195%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:49.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:49.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/250545719' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:49.965 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[98970]: osdmap e266: 8 total, 8 up, 8 in 2024-08-29T15:24:49.965 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[93921]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 29671/192486 objects degraded (15.415%); 11924/192486 objects misplaced (6.195%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:49.965 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:49.965 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/250545719' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:49.966 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:49 smithi027 ceph-mon[93921]: osdmap e266: 8 total, 8 up, 8 in 2024-08-29T15:24:49.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:49.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:49.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:50.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:50.115 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:50.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:50.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:50.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:50.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:50.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:49 smithi171 ceph-mon[90031]: pgmap v878: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 143 op/s; 29671/192486 objects degraded (15.415%); 11924/192486 objects misplaced (6.195%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:49 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/250545719' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:24:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:49 smithi171 ceph-mon[90031]: osdmap e266: 8 total, 8 up, 8 in 2024-08-29T15:24:50.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:50.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:50.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:50.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:50.468 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:50.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:50.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:50.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:50.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:50.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:50.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:50.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:50.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:50.844 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:50.867 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:50.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:50.936 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:50.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:51.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:51.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:51.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:51.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:51.183 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:24:51.183 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 29671/191766 objects degraded (15.473%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1273,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":266,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":63922,"data_bytes":4192081917,"bytes_used":14588751872,"bytes_avail":753371533312,"bytes_total":767960285184,"degraded_objects":29671,"degraded_total":191766,"degraded_ratio":0.15472502946299135,"misplaced_objects":11924,"misplaced_total":191766,"misplaced_ratio":0.062179948478875299,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1275159,"recovering_keys_per_sec":0,"num_objects_recovered":191,"num_bytes_recovered":12517376,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":136},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-29T15:19:38.525651+0000","services":{}},"progress_events":{}} 2024-08-29T15:24:51.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:51.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:51.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:51.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:51.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:51.506 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:51.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:51.608 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:51.631 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:51.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:51.754 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:51.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:51.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:51.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:51.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:52.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:51 smithi027 ceph-mon[93921]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 29671/191766 objects degraded (15.473%); 11924/191766 objects misplaced (6.218%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1761791512' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:24:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:51 smithi027 ceph-mon[98970]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 29671/191766 objects degraded (15.473%); 11924/191766 objects misplaced (6.218%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1761791512' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:24:52.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.127 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:52.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:52.228 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:52.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:51 smithi171 ceph-mon[90031]: pgmap v880: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 29671/191766 objects degraded (15.473%); 11924/191766 objects misplaced (6.218%); 1.2 MiB/s, 19 objects/s recovering 2024-08-29T15:24:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1761791512' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:24:52.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:52.354 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:24:52.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:24:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:52.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:52.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:52.541 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:52.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:52.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:24:52.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:52.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:52.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:52.791 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:52.893 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:52.915 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:52.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:52.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:52.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29671/191766 objects degraded (15.473%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:52.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29671/191766 objects degraded (15.473%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:53.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:53.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:53.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:53.169 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:53.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29671/191766 objects degraded (15.473%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:53.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:53.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:53.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:53.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:53.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:53.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:53.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:53.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:53.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:53.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:53.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:53.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:53.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:53.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:53.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:53.923 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:53.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:53.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:54.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:54.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:54.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:54.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:53 smithi027 ceph-mon[98970]: pgmap v881: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 29596/189681 objects degraded (15.603%); 11777/189681 objects misplaced (6.209%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:24:54.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:53 smithi027 ceph-mon[93921]: pgmap v881: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 29596/189681 objects degraded (15.603%); 11777/189681 objects misplaced (6.209%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:24:54.123 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:54.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:54.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:54.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:53 smithi171 ceph-mon[90031]: pgmap v881: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 29596/189681 objects degraded (15.603%); 11777/189681 objects misplaced (6.209%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:24:54.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:54.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:54.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:54.423 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:54.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:54.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:54.552 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:54.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:54.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:54.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:54.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:54.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:54.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:54.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:54.912 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:54.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:54.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:54.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:55.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:55.061 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:55.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:55.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:55.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:55.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:55.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:55.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:55.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.435 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:55.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:55.461 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:24:55.461 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:24:55.477 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:24:55.478 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:24:55.494 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:24:55.495 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:24:55.510 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:24:55.511 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:24:55.527 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:24:55.528 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:24:55.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:55.545 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:24:55.545 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:24:55.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:55.565 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:24:55.565 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:24:55.581 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:24:55.582 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:24:55.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:55.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:55.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:55.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:55.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:55.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:55.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:55.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:55.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:55.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:56.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:56.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:56.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:56.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:56.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:56.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:56.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:55 smithi027 ceph-mon[98970]: pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 29596/189234 objects degraded (15.640%); 11724/189234 objects misplaced (6.196%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:24:56.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/628383125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:56.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:55 smithi027 ceph-mon[93921]: pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 29596/189234 objects degraded (15.640%); 11724/189234 objects misplaced (6.196%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:24:56.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/628383125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:55 smithi171 ceph-mon[90031]: pgmap v882: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 29596/189234 objects degraded (15.640%); 11724/189234 objects misplaced (6.196%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:24:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/628383125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:24:56.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.309 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:56.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:56.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:56.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:56.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:56.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:56.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:56.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:56.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:56.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:56.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:56.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:56.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:56.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:56.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:56.962 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:56 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29596/189234 objects degraded (15.640%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:56.963 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:56 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29596/189234 objects degraded (15.640%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:56.995 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:24:56.995 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:24:57.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:57.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:57.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:57.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:56 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29596/189234 objects degraded (15.640%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:24:57.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:57.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:57.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:57.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:24:57.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:57.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:24:57.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:24:57.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:57.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:24:57.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:57.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:57.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:24:57.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:24:57.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:57.947 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:57.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:58.036 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:57 smithi027 ceph-mon[98970]: pgmap v883: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29553/187359 objects degraded (15.773%); 11623/187359 objects misplaced (6.204%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:58.036 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:57 smithi027 ceph-mon[93921]: pgmap v883: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29553/187359 objects degraded (15.773%); 11623/187359 objects misplaced (6.204%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:58.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:58.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:58.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:58.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:58.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:58.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:58.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:58.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:58.253 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:24:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:57 smithi171 ceph-mon[90031]: pgmap v883: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29553/187359 objects degraded (15.773%); 11623/187359 objects misplaced (6.204%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:58.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:58.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:58.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:58.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:58.454 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:58.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:58.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:58.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:58.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:58.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:58.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:24:58.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:24:58.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:58.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:58.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:24:58.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:58.869 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:24:58.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:58.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:58.926 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:24:58.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1232258582' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:24:58.927 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:58.928 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1232258582' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:24:58.928 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:58.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:59.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:59.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1232258582' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:24:59.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:24:59.105 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:59.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:59.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:59.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:59.233 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:24:59] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:24:59.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:59.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:59.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:24:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.383 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:24:59.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:24:59.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:59.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:59.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:24:59.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:24:59.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:24:59.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:24:59.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:24:59.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:59.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:24:59.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:24:59.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:24:59.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:24:59.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[93921]: pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29553/187191 objects degraded (15.788%); 11569/187191 objects misplaced (6.180%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:59.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:59.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1232258582' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:24:59.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[93921]: osdmap e267: 8 total, 8 up, 8 in 2024-08-29T15:24:59.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[98970]: pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29553/187191 objects degraded (15.788%); 11569/187191 objects misplaced (6.180%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:24:59.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:24:59.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1232258582' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:24:59.999 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:24:59 smithi027 ceph-mon[98970]: osdmap e267: 8 total, 8 up, 8 in 2024-08-29T15:25:00.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:59 smithi171 ceph-mon[90031]: pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 176 op/s; 29553/187191 objects degraded (15.788%); 11569/187191 objects misplaced (6.180%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:59 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1232258582' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:25:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:24:59 smithi171 ceph-mon[90031]: osdmap e267: 8 total, 8 up, 8 in 2024-08-29T15:25:00.115 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:00.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.139 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:00.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:00.240 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:00.263 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:00.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:00.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:00.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:00.489 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.516 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:00.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:00.617 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:00.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:00.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:00.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:00.765 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:00.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:00.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:00.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:00.994 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:01.009 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:25:01.009 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:25:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:01.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:01.081 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:01.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:01.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:01.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:01.246 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:01.270 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:25:01.271 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:25:01.272 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:01.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:01.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:01.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:01.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:01.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:01.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:01.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:01.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:01.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:25:01.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:01.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:01.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:01.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:01.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:01.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:01.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:01.979 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:01 smithi171 ceph-mon[90031]: pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 29553/186588 objects degraded (15.839%); 11569/186588 objects misplaced (6.200%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:01 smithi171 ceph-mon[90031]: from='client.45791 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:01 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29553/186588 objects degraded (15.839%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:02.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:02.047 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:02.127 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:01 smithi027 ceph-mon[98970]: pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 29553/186588 objects degraded (15.839%); 11569/186588 objects misplaced (6.200%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:02.127 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:01 smithi027 ceph-mon[98970]: from='client.45791 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:02.127 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:01 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29553/186588 objects degraded (15.839%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:02.128 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:01 smithi027 ceph-mon[93921]: pgmap v886: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 29553/186588 objects degraded (15.839%); 11569/186588 objects misplaced (6.200%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:02.128 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:01 smithi027 ceph-mon[93921]: from='client.45791 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:02.128 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:01 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29553/186588 objects degraded (15.839%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:02.128 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":886,"stamp":"2024-08-29T15:25:00.617799+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4078966781,"num_objects":62196,"num_object_clones":0,"num_object_copies":186588,"num_objects_missing_on_primary":4876,"num_objects_missing":4876,"num_objects_degraded":29553,"num_objects_misplaced":11569,"num_objects_unfound":0,"num_objects_dirty":62196,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":189805,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17140,"num_bytes_recovered":1002114253,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125441,"ondisk_log_size":125441,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13918768,"kb_used_data":12015272,"kb_used_omap":43,"kb_used_meta":1903380,"kb_avail":736042448,"statfs":{"total":767960285184,"available":753707466752,"internally_reserved":0,"allocated":12303638528,"data_stored":12291468814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1949061612},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,3,1,3,0,2,6,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-113115136,"num_objects":-1726,"num_object_clones":0,"num_object_copies":-5178,"num_objects_missing_on_primary":-107,"num_objects_missing":-107,"num_objects_degraded":-118,"num_objects_misplaced":-355,"num_objects_unfound":0,"num_objects_dirty":-1726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1726,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":227,"num_bytes_recovered":14876672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002921"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":417,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820884+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:24:48.820884+0000","last_peered":"2024-08-29T15:24:48.820884+0000","last_clean":"2024-08-29T15:24:48.820884+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:24:48.820884+0000","last_undegraded":"2024-08-29T15:24:48.820884+0000","last_fullsized":"2024-08-29T15:24:48.820884+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":45,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:48.817803+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:24:48.817803+0000","last_peered":"2024-08-29T15:24:48.817803+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:24:48.817803+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:24:48.817803+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.816280+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:24:48.816280+0000","last_peered":"2024-08-29T15:24:48.816280+0000","last_clean":"2024-08-29T15:24:48.816280+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:24:48.816280+0000","last_undegraded":"2024-08-29T15:24:48.816280+0000","last_fullsized":"2024-08-29T15:24:48.816280+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":385,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875398+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:24:58.875398+0000","last_peered":"2024-08-29T15:24:58.875398+0000","last_clean":"2024-08-29T15:24:58.875398+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:24:58.875398+0000","last_undegraded":"2024-08-29T15:24:58.875398+0000","last_fullsized":"2024-08-29T15:24:58.875398+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":385,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875396+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:24:58.875396+0000","last_peered":"2024-08-29T15:24:58.875396+0000","last_clean":"2024-08-29T15:24:58.875396+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:24:58.875396+0000","last_undegraded":"2024-08-29T15:24:58.875396+0000","last_fullsized":"2024-08-29T15:24:58.875396+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817584+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:24:48.817584+0000","last_peered":"2024-08-29T15:24:48.817584+0000","last_clean":"2024-08-29T15:24:48.817584+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:24:48.817584+0000","last_undegraded":"2024-08-29T15:24:48.817584+0000","last_fullsized":"2024-08-29T15:24:48.817584+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":194,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.826171+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:24:48.826171+0000","last_peered":"2024-08-29T15:24:48.826171+0000","last_clean":"2024-08-29T15:24:48.826171+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:24:48.826171+0000","last_undegraded":"2024-08-29T15:24:48.826171+0000","last_fullsized":"2024-08-29T15:24:48.826171+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":135,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.816512+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:24:40.816512+0000","last_peered":"2024-08-29T15:24:40.816512+0000","last_clean":"2024-08-29T15:24:40.816512+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:24:40.816512+0000","last_undegraded":"2024-08-29T15:24:40.816512+0000","last_fullsized":"2024-08-29T15:24:40.816512+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"266'11926","reported_seq":19971,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:55.773675+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:24:55.773675+0000","last_peered":"2024-08-29T15:24:55.773675+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:24:55.773675+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"176'1900","ondisk_log_start":"176'1900","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261226496,"num_objects":3986,"num_object_clones":0,"num_object_copies":11958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11926,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":1061}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":44,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:48.817793+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:24:48.817793+0000","last_peered":"2024-08-29T15:24:48.817793+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:24:48.817793+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:24:48.817793+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":152,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.799410+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:24:40.799410+0000","last_peered":"2024-08-29T15:24:40.799410+0000","last_clean":"2024-08-29T15:24:40.799410+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:24:40.799410+0000","last_undegraded":"2024-08-29T15:24:40.799410+0000","last_fullsized":"2024-08-29T15:24:40.799410+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"266'11649","reported_seq":11860,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:56.318897+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:24:56.318897+0000","last_peered":"2024-08-29T15:24:56.318897+0000","last_clean":"2024-08-29T15:24:56.318897+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:24:56.318897+0000","last_undegraded":"2024-08-29T15:24:56.318897+0000","last_fullsized":"2024-08-29T15:24:56.318897+0000","mapping_epoch":168,"log_start":"180'4928","ondisk_log_start":"180'4928","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255655936,"num_objects":3901,"num_object_clones":0,"num_object_copies":11703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11649,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":44,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:48.821354+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:24:48.821354+0000","last_peered":"2024-08-29T15:24:48.821354+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:24:48.821354+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:24:48.821354+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817464+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:24:48.817464+0000","last_peered":"2024-08-29T15:24:48.817464+0000","last_clean":"2024-08-29T15:24:48.817464+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:24:48.817464+0000","last_undegraded":"2024-08-29T15:24:48.817464+0000","last_fullsized":"2024-08-29T15:24:48.817464+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"266'11873","reported_seq":12054,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:56.221929+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:24:56.221929+0000","last_peered":"2024-08-29T15:24:56.221929+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:24:56.221929+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"180'5174","ondisk_log_start":"180'5174","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251592704,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11873,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1362}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":201,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.877633+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:24:58.877633+0000","last_peered":"2024-08-29T15:24:58.877633+0000","last_clean":"2024-08-29T15:24:58.877633+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:24:58.877633+0000","last_undegraded":"2024-08-29T15:24:58.877633+0000","last_fullsized":"2024-08-29T15:24:58.877633+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":90,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.816930+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:24:40.816930+0000","last_peered":"2024-08-29T15:24:40.816930+0000","last_clean":"2024-08-29T15:24:40.816930+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:24:40.816930+0000","last_undegraded":"2024-08-29T15:24:40.816930+0000","last_fullsized":"2024-08-29T15:24:40.816930+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"267'11896","reported_seq":17329,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:58.940229+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:24:58.940229+0000","last_peered":"2024-08-29T15:24:58.940229+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:24:58.940229+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"179'3762","ondisk_log_start":"179'3762","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8134,"log_dups_size":0,"ondisk_log_size":8134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254410752,"num_objects":3882,"num_object_clones":0,"num_object_copies":11646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11896,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3882}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":152,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.799953+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:24:40.799953+0000","last_peered":"2024-08-29T15:24:40.799953+0000","last_clean":"2024-08-29T15:24:40.799953+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:24:40.799953+0000","last_undegraded":"2024-08-29T15:24:40.799953+0000","last_fullsized":"2024-08-29T15:24:40.799953+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":399,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.876472+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:24:58.876472+0000","last_peered":"2024-08-29T15:24:58.876472+0000","last_clean":"2024-08-29T15:24:58.876472+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:24:58.876472+0000","last_undegraded":"2024-08-29T15:24:58.876472+0000","last_fullsized":"2024-08-29T15:24:58.876472+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"267'11861","reported_seq":12015,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:59.736180+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:24:59.736180+0000","last_peered":"2024-08-29T15:24:59.736180+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:24:59.736180+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:24:59.736180+0000","mapping_epoch":234,"log_start":"179'3933","ondisk_log_start":"179'3933","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7928,"log_dups_size":0,"ondisk_log_size":7928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259063808,"num_objects":3953,"num_object_clones":0,"num_object_copies":11859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11861,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":1476},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":392,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820639+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:24:48.820639+0000","last_peered":"2024-08-29T15:24:48.820639+0000","last_clean":"2024-08-29T15:24:48.820639+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:24:48.820639+0000","last_undegraded":"2024-08-29T15:24:48.820639+0000","last_fullsized":"2024-08-29T15:24:48.820639+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.818204+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:24:48.818204+0000","last_peered":"2024-08-29T15:24:48.818204+0000","last_clean":"2024-08-29T15:24:48.818204+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:24:48.818204+0000","last_undegraded":"2024-08-29T15:24:48.818204+0000","last_fullsized":"2024-08-29T15:24:48.818204+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"266'11868","reported_seq":12095,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:56.232372+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:24:56.232372+0000","last_peered":"2024-08-29T15:24:56.232372+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:24:56.232372+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:24:56.232372+0000","mapping_epoch":234,"log_start":"180'5168","ondisk_log_start":"180'5168","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260440064,"num_objects":3974,"num_object_clones":0,"num_object_copies":11922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11868,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":1509},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.816848+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:24:40.816848+0000","last_peered":"2024-08-29T15:24:40.816848+0000","last_clean":"2024-08-29T15:24:40.816848+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:24:40.816848+0000","last_undegraded":"2024-08-29T15:24:40.816848+0000","last_fullsized":"2024-08-29T15:24:40.816848+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817867+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:24:48.817867+0000","last_peered":"2024-08-29T15:24:48.817867+0000","last_clean":"2024-08-29T15:24:48.817867+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:24:48.817867+0000","last_undegraded":"2024-08-29T15:24:48.817867+0000","last_fullsized":"2024-08-29T15:24:48.817867+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"266'11719","reported_seq":12131,"reported_epoch":266,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:24:55.825802+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:24:55.825802+0000","last_peered":"2024-08-29T15:24:55.825802+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:24:55.825802+0000","last_undegraded":"2024-08-29T15:24:55.825802+0000","last_fullsized":"2024-08-29T15:24:55.825802+0000","mapping_epoch":202,"log_start":"180'4967","ondisk_log_start":"180'4967","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258146304,"num_objects":3939,"num_object_clones":0,"num_object_copies":11817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3939,"num_objects_unfound":0,"num_objects_dirty":3939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11719,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":3939}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817507+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:24:48.817507+0000","last_peered":"2024-08-29T15:24:48.817507+0000","last_clean":"2024-08-29T15:24:48.817507+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:24:48.817507+0000","last_undegraded":"2024-08-29T15:24:48.817507+0000","last_fullsized":"2024-08-29T15:24:48.817507+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":252,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:54.821240+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:24:54.821240+0000","last_peered":"2024-08-29T15:24:54.821240+0000","last_clean":"2024-08-29T15:24:54.821240+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:24:54.821240+0000","last_undegraded":"2024-08-29T15:24:54.821240+0000","last_fullsized":"2024-08-29T15:24:54.821240+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"266'11892","reported_seq":12039,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:55.808082+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:24:55.808082+0000","last_peered":"2024-08-29T15:24:55.808082+0000","last_clean":"2024-08-29T15:24:55.808082+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:24:55.808082+0000","last_undegraded":"2024-08-29T15:24:55.808082+0000","last_fullsized":"2024-08-29T15:24:55.808082+0000","mapping_epoch":218,"log_start":"176'1878","ondisk_log_start":"176'1878","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10014,"log_dups_size":0,"ondisk_log_size":10014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253427736,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11892,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":395,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820890+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:24:48.820890+0000","last_peered":"2024-08-29T15:24:48.820890+0000","last_clean":"2024-08-29T15:24:48.820890+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:24:48.820890+0000","last_undegraded":"2024-08-29T15:24:48.820890+0000","last_fullsized":"2024-08-29T15:24:48.820890+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":387,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.826174+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:24:48.826174+0000","last_peered":"2024-08-29T15:24:48.826174+0000","last_clean":"2024-08-29T15:24:48.826174+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:24:48.826174+0000","last_undegraded":"2024-08-29T15:24:48.826174+0000","last_fullsized":"2024-08-29T15:24:48.826174+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"266'11361","reported_seq":11556,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:55.796779+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:24:55.796779+0000","last_peered":"2024-08-29T15:24:55.796779+0000","last_clean":"2024-08-29T15:24:55.796779+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:24:55.796779+0000","last_undegraded":"2024-08-29T15:24:55.796779+0000","last_fullsized":"2024-08-29T15:24:55.796779+0000","mapping_epoch":201,"log_start":"175'1300","ondisk_log_start":"175'1300","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10061,"log_dups_size":0,"ondisk_log_size":10061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252116992,"num_objects":3847,"num_object_clones":0,"num_object_copies":11541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11361,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":488,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.879938+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:24:58.879938+0000","last_peered":"2024-08-29T15:24:58.879938+0000","last_clean":"2024-08-29T15:24:58.879938+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:24:58.879938+0000","last_undegraded":"2024-08-29T15:24:58.879938+0000","last_fullsized":"2024-08-29T15:24:58.879938+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":736,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.799616+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:24:40.799616+0000","last_peered":"2024-08-29T15:24:40.799616+0000","last_clean":"2024-08-29T15:24:40.799616+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:24:40.799616+0000","last_undegraded":"2024-08-29T15:24:40.799616+0000","last_fullsized":"2024-08-29T15:24:40.799616+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":244,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.800020+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:24:40.800020+0000","last_peered":"2024-08-29T15:24:40.800020+0000","last_clean":"2024-08-29T15:24:40.800020+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:24:40.800020+0000","last_undegraded":"2024-08-29T15:24:40.800020+0000","last_fullsized":"2024-08-29T15:24:40.800020+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.818421+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:24:40.818421+0000","last_peered":"2024-08-29T15:24:40.818421+0000","last_clean":"2024-08-29T15:24:40.818421+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:24:40.818421+0000","last_undegraded":"2024-08-29T15:24:40.818421+0000","last_fullsized":"2024-08-29T15:24:40.818421+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"266'11823","reported_seq":12232,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:55.235888+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:24:55.235888+0000","last_peered":"2024-08-29T15:24:55.235888+0000","last_clean":"2024-08-29T15:24:55.235888+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:24:55.235888+0000","last_undegraded":"2024-08-29T15:24:55.235888+0000","last_fullsized":"2024-08-29T15:24:55.235888+0000","mapping_epoch":201,"log_start":"180'4753","ondisk_log_start":"180'4753","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252903424,"num_objects":3859,"num_object_clones":0,"num_object_copies":11577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11823,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":860,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.824404+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:24:48.824404+0000","last_peered":"2024-08-29T15:24:48.824404+0000","last_clean":"2024-08-29T15:24:48.824404+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:24:48.824404+0000","last_undegraded":"2024-08-29T15:24:48.824404+0000","last_fullsized":"2024-08-29T15:24:48.824404+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.877526+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:24:58.877526+0000","last_peered":"2024-08-29T15:24:58.877526+0000","last_clean":"2024-08-29T15:24:58.877526+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:24:58.877526+0000","last_undegraded":"2024-08-29T15:24:58.877526+0000","last_fullsized":"2024-08-29T15:24:58.877526+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":408,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875650+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:24:58.875650+0000","last_peered":"2024-08-29T15:24:58.875650+0000","last_clean":"2024-08-29T15:24:58.875650+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:24:58.875650+0000","last_undegraded":"2024-08-29T15:24:58.875650+0000","last_fullsized":"2024-08-29T15:24:58.875650+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"266'11751","reported_seq":18682,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:56.350557+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:24:56.350557+0000","last_peered":"2024-08-29T15:24:56.350557+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:24:56.350557+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"180'5030","ondisk_log_start":"180'5030","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258146304,"num_objects":3939,"num_object_clones":0,"num_object_copies":11817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4416,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11751,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3375,"num_bytes_recovered":221184000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1501},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":669,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820469+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:24:48.820469+0000","last_peered":"2024-08-29T15:24:48.820469+0000","last_clean":"2024-08-29T15:24:48.820469+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:24:48.820469+0000","last_undegraded":"2024-08-29T15:24:48.820469+0000","last_fullsized":"2024-08-29T15:24:48.820469+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":97,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.816393+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:24:40.816393+0000","last_peered":"2024-08-29T15:24:40.816393+0000","last_clean":"2024-08-29T15:24:40.816393+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:24:40.816393+0000","last_undegraded":"2024-08-29T15:24:40.816393+0000","last_fullsized":"2024-08-29T15:24:40.816393+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":432,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.826051+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:24:48.826051+0000","last_peered":"2024-08-29T15:24:48.826051+0000","last_clean":"2024-08-29T15:24:48.826051+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:24:48.826051+0000","last_undegraded":"2024-08-29T15:24:48.826051+0000","last_fullsized":"2024-08-29T15:24:48.826051+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"266'11824","reported_seq":12045,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:55.864900+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:24:55.864900+0000","last_peered":"2024-08-29T15:24:55.864900+0000","last_clean":"2024-08-29T15:24:55.864900+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:24:55.864900+0000","last_undegraded":"2024-08-29T15:24:55.864900+0000","last_fullsized":"2024-08-29T15:24:55.864900+0000","mapping_epoch":234,"log_start":"180'5097","ondisk_log_start":"180'5097","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6727,"log_dups_size":0,"ondisk_log_size":6727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254803968,"num_objects":3888,"num_object_clones":0,"num_object_copies":11664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11824,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":204,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.799438+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:24:40.799438+0000","last_peered":"2024-08-29T15:24:40.799438+0000","last_clean":"2024-08-29T15:24:40.799438+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:24:40.799438+0000","last_undegraded":"2024-08-29T15:24:40.799438+0000","last_fullsized":"2024-08-29T15:24:40.799438+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817694+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:24:48.817694+0000","last_peered":"2024-08-29T15:24:48.817694+0000","last_clean":"2024-08-29T15:24:48.817694+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:24:48.817694+0000","last_undegraded":"2024-08-29T15:24:48.817694+0000","last_fullsized":"2024-08-29T15:24:48.817694+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.799627+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:24:40.799627+0000","last_peered":"2024-08-29T15:24:40.799627+0000","last_clean":"2024-08-29T15:24:40.799627+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:24:40.799627+0000","last_undegraded":"2024-08-29T15:24:40.799627+0000","last_fullsized":"2024-08-29T15:24:40.799627+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"267'11905","reported_seq":12126,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:58.932568+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:24:58.932568+0000","last_peered":"2024-08-29T15:24:58.932568+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:24:58.932568+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"179'3742","ondisk_log_start":"179'3742","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8163,"log_dups_size":0,"ondisk_log_size":8163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256966656,"num_objects":3921,"num_object_clones":0,"num_object_copies":11763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11905,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1498}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.799688+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:24:40.799688+0000","last_peered":"2024-08-29T15:24:40.799688+0000","last_clean":"2024-08-29T15:24:40.799688+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:24:40.799688+0000","last_undegraded":"2024-08-29T15:24:40.799688+0000","last_fullsized":"2024-08-29T15:24:40.799688+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":825,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.825314+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:24:48.825314+0000","last_peered":"2024-08-29T15:24:48.825314+0000","last_clean":"2024-08-29T15:24:48.825314+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:24:48.825314+0000","last_undegraded":"2024-08-29T15:24:48.825314+0000","last_fullsized":"2024-08-29T15:24:48.825314+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":402,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.824816+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:24:48.824816+0000","last_peered":"2024-08-29T15:24:48.824816+0000","last_clean":"2024-08-29T15:24:48.824816+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:24:48.824816+0000","last_undegraded":"2024-08-29T15:24:48.824816+0000","last_fullsized":"2024-08-29T15:24:48.824816+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"266'11829","reported_seq":12604,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:55.255193+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:24:55.255193+0000","last_peered":"2024-08-29T15:24:55.255193+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:24:55.255193+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"180'4780","ondisk_log_start":"180'4780","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7049,"log_dups_size":0,"ondisk_log_size":7049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251592704,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":3811,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11829,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":1688}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":407,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.825219+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:24:48.825219+0000","last_peered":"2024-08-29T15:24:48.825219+0000","last_clean":"2024-08-29T15:24:48.825219+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:24:48.825219+0000","last_undegraded":"2024-08-29T15:24:48.825219+0000","last_fullsized":"2024-08-29T15:24:48.825219+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":694,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:48.838882+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:24:48.838882+0000","last_peered":"2024-08-29T15:24:48.838882+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:24:48.838882+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:24:48.838882+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":400,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.876387+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:24:58.876387+0000","last_peered":"2024-08-29T15:24:58.876387+0000","last_clean":"2024-08-29T15:24:58.876387+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:24:58.876387+0000","last_undegraded":"2024-08-29T15:24:58.876387+0000","last_fullsized":"2024-08-29T15:24:58.876387+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"267'11726","reported_seq":18878,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:59.750843+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:24:59.750843+0000","last_peered":"2024-08-29T15:24:59.750843+0000","last_clean":"2024-08-29T15:24:59.750843+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:24:59.750843+0000","last_undegraded":"2024-08-29T15:24:59.750843+0000","last_fullsized":"2024-08-29T15:24:59.750843+0000","mapping_epoch":234,"log_start":"179'3799","ondisk_log_start":"179'3799","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7927,"log_dups_size":0,"ondisk_log_size":7927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242614272,"num_objects":3702,"num_object_clones":0,"num_object_copies":11106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11726,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":392,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.822985+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:24:48.822985+0000","last_peered":"2024-08-29T15:24:48.822985+0000","last_clean":"2024-08-29T15:24:48.822985+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:24:48.822985+0000","last_undegraded":"2024-08-29T15:24:48.822985+0000","last_fullsized":"2024-08-29T15:24:48.822985+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1014,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:48.824273+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:24:48.824273+0000","last_peered":"2024-08-29T15:24:48.824273+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:24:48.824273+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":389,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820324+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:24:48.820324+0000","last_peered":"2024-08-29T15:24:48.820324+0000","last_clean":"2024-08-29T15:24:48.820324+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:24:48.820324+0000","last_undegraded":"2024-08-29T15:24:48.820324+0000","last_fullsized":"2024-08-29T15:24:48.820324+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"266'11779","reported_seq":17654,"reported_epoch":266,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:58.124481+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:24:58.124481+0000","last_peered":"2024-08-29T15:24:58.124481+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:24:58.124481+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"179'4212","ondisk_log_start":"179'4212","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7567,"log_dups_size":0,"ondisk_log_size":7567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251068416,"num_objects":3831,"num_object_clones":0,"num_object_copies":11493,"num_objects_missing_on_primary":4868,"num_objects_missing":4868,"num_objects_degraded":2313,"num_objects_misplaced":3819,"num_objects_unfound":0,"num_objects_dirty":3831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11779,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2855,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":12},{"shards":"1,2,7","objects":1448},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":390,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875278+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:24:58.875278+0000","last_peered":"2024-08-29T15:24:58.875278+0000","last_clean":"2024-08-29T15:24:58.875278+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:24:58.875278+0000","last_undegraded":"2024-08-29T15:24:58.875278+0000","last_fullsized":"2024-08-29T15:24:58.875278+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":117,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.816950+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:24:40.816950+0000","last_peered":"2024-08-29T15:24:40.816950+0000","last_clean":"2024-08-29T15:24:40.816950+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:24:40.816950+0000","last_undegraded":"2024-08-29T15:24:40.816950+0000","last_fullsized":"2024-08-29T15:24:40.816950+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817545+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:24:48.817545+0000","last_peered":"2024-08-29T15:24:48.817545+0000","last_clean":"2024-08-29T15:24:48.817545+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:24:48.817545+0000","last_undegraded":"2024-08-29T15:24:48.817545+0000","last_fullsized":"2024-08-29T15:24:48.817545+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817943+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:24:48.817943+0000","last_peered":"2024-08-29T15:24:48.817943+0000","last_clean":"2024-08-29T15:24:48.817943+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:24:48.817943+0000","last_undegraded":"2024-08-29T15:24:48.817943+0000","last_fullsized":"2024-08-29T15:24:48.817943+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":44,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:48.818313+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:24:48.818313+0000","last_peered":"2024-08-29T15:24:48.818313+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:24:48.818313+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:24:48.818313+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":396,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.824728+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:24:48.824728+0000","last_peered":"2024-08-29T15:24:48.824728+0000","last_clean":"2024-08-29T15:24:48.824728+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:24:48.824728+0000","last_undegraded":"2024-08-29T15:24:48.824728+0000","last_fullsized":"2024-08-29T15:24:48.824728+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":146,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:48.821423+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:24:48.821423+0000","last_peered":"2024-08-29T15:24:48.821423+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:24:48.821423+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:24:48.821423+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":134,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.816127+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:24:48.816127+0000","last_peered":"2024-08-29T15:24:48.816127+0000","last_clean":"2024-08-29T15:24:48.816127+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:24:48.816127+0000","last_undegraded":"2024-08-29T15:24:48.816127+0000","last_fullsized":"2024-08-29T15:24:48.816127+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":355,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820191+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:24:48.820191+0000","last_peered":"2024-08-29T15:24:48.820191+0000","last_clean":"2024-08-29T15:24:48.820191+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:24:48.820191+0000","last_undegraded":"2024-08-29T15:24:48.820191+0000","last_fullsized":"2024-08-29T15:24:48.820191+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":411,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875867+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:24:58.875867+0000","last_peered":"2024-08-29T15:24:58.875867+0000","last_clean":"2024-08-29T15:24:58.875867+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:24:58.875867+0000","last_undegraded":"2024-08-29T15:24:58.875867+0000","last_fullsized":"2024-08-29T15:24:58.875867+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":407,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820227+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:24:48.820227+0000","last_peered":"2024-08-29T15:24:48.820227+0000","last_clean":"2024-08-29T15:24:48.820227+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:24:48.820227+0000","last_undegraded":"2024-08-29T15:24:48.820227+0000","last_fullsized":"2024-08-29T15:24:48.820227+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":105,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.817019+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:24:40.817019+0000","last_peered":"2024-08-29T15:24:40.817019+0000","last_clean":"2024-08-29T15:24:40.817019+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:24:40.817019+0000","last_undegraded":"2024-08-29T15:24:40.817019+0000","last_fullsized":"2024-08-29T15:24:40.817019+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":266,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.879805+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:24:58.879805+0000","last_peered":"2024-08-29T15:24:58.879805+0000","last_clean":"2024-08-29T15:24:58.879805+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:24:58.879805+0000","last_undegraded":"2024-08-29T15:24:58.879805+0000","last_fullsized":"2024-08-29T15:24:58.879805+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":386,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875564+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:24:58.875564+0000","last_peered":"2024-08-29T15:24:58.875564+0000","last_clean":"2024-08-29T15:24:58.875564+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:24:58.875564+0000","last_undegraded":"2024-08-29T15:24:58.875564+0000","last_fullsized":"2024-08-29T15:24:58.875564+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":388,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.824609+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:24:48.824609+0000","last_peered":"2024-08-29T15:24:48.824609+0000","last_clean":"2024-08-29T15:24:48.824609+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:24:48.824609+0000","last_undegraded":"2024-08-29T15:24:48.824609+0000","last_fullsized":"2024-08-29T15:24:48.824609+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":392,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.821268+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:24:48.821268+0000","last_peered":"2024-08-29T15:24:48.821268+0000","last_clean":"2024-08-29T15:24:48.821268+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:24:48.821268+0000","last_undegraded":"2024-08-29T15:24:48.821268+0000","last_fullsized":"2024-08-29T15:24:48.821268+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:49.820420+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:24:49.820420+0000","last_peered":"2024-08-29T15:24:49.820420+0000","last_clean":"2024-08-29T15:24:49.820420+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:24:49.820420+0000","last_undegraded":"2024-08-29T15:24:49.820420+0000","last_fullsized":"2024-08-29T15:24:49.820420+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":412,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.825418+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:24:48.825418+0000","last_peered":"2024-08-29T15:24:48.825418+0000","last_clean":"2024-08-29T15:24:48.825418+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:24:48.825418+0000","last_undegraded":"2024-08-29T15:24:48.825418+0000","last_fullsized":"2024-08-29T15:24:48.825418+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":378,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.876289+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:24:58.876289+0000","last_peered":"2024-08-29T15:24:58.876289+0000","last_clean":"2024-08-29T15:24:58.876289+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:24:58.876289+0000","last_undegraded":"2024-08-29T15:24:58.876289+0000","last_fullsized":"2024-08-29T15:24:58.876289+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:49.820479+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:24:49.820479+0000","last_peered":"2024-08-29T15:24:49.820479+0000","last_clean":"2024-08-29T15:24:49.820479+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:24:49.820479+0000","last_undegraded":"2024-08-29T15:24:49.820479+0000","last_fullsized":"2024-08-29T15:24:49.820479+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":157,"reported_epoch":265,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:40.799792+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:24:40.799792+0000","last_peered":"2024-08-29T15:24:40.799792+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:24:40.799792+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:24:40.799792+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:49.820452+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:24:49.820452+0000","last_peered":"2024-08-29T15:24:49.820452+0000","last_clean":"2024-08-29T15:24:49.820452+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:24:49.820452+0000","last_undegraded":"2024-08-29T15:24:49.820452+0000","last_fullsized":"2024-08-29T15:24:49.820452+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":163,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-08-29T15:24:40.799849+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:24:40.799849+0000","last_peered":"2024-08-29T15:24:40.799849+0000","last_clean":"2024-08-29T15:24:40.799849+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:24:40.799849+0000","last_undegraded":"2024-08-29T15:24:40.799849+0000","last_fullsized":"2024-08-29T15:24:40.799849+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":423,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.820523+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:24:48.820523+0000","last_peered":"2024-08-29T15:24:48.820523+0000","last_clean":"2024-08-29T15:24:48.820523+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:24:48.820523+0000","last_undegraded":"2024-08-29T15:24:48.820523+0000","last_fullsized":"2024-08-29T15:24:48.820523+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":424,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.821127+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:24:48.821127+0000","last_peered":"2024-08-29T15:24:48.821127+0000","last_clean":"2024-08-29T15:24:48.821127+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:24:48.821127+0000","last_undegraded":"2024-08-29T15:24:48.821127+0000","last_fullsized":"2024-08-29T15:24:48.821127+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":46,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.817359+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:24:48.817359+0000","last_peered":"2024-08-29T15:24:48.817359+0000","last_clean":"2024-08-29T15:24:48.817359+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:24:48.817359+0000","last_undegraded":"2024-08-29T15:24:48.817359+0000","last_fullsized":"2024-08-29T15:24:48.817359+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":434,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875758+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:24:58.875758+0000","last_peered":"2024-08-29T15:24:58.875758+0000","last_clean":"2024-08-29T15:24:58.875758+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:24:58.875758+0000","last_undegraded":"2024-08-29T15:24:58.875758+0000","last_fullsized":"2024-08-29T15:24:58.875758+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4074176536,"num_objects":62168,"num_object_clones":0,"num_object_copies":186504,"num_objects_missing_on_primary":4868,"num_objects_missing":4868,"num_objects_degraded":29524,"num_objects_misplaced":11569,"num_objects_unfound":0,"num_objects_dirty":62168,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188682,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17031,"num_bytes_recovered":996147200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12250329088,"data_stored":12250316872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124259,"ondisk_log_size":124259,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234364,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2298252,"kb_used_data":1978368,"kb_used_omap":1,"kb_used_meta":319870,"kb_avail":91446900,"statfs":{"total":95995035648,"available":93641625600,"internally_reserved":0,"allocated":2025848832,"data_stored":2024310235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":327547339},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495995,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":255988,"kb_used_data":80492,"kb_used_omap":1,"kb_used_meta":175486,"kb_avail":93489164,"statfs":{"total":95995035648,"available":95732903936,"internally_reserved":0,"allocated":82423808,"data_stored":80896659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":179698119},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3,1,3,0,2,6,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426581,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1338216,"kb_used_data":1252768,"kb_used_omap":8,"kb_used_meta":85431,"kb_avail":92406936,"statfs":{"total":95995035648,"available":94624702464,"internally_reserved":0,"allocated":1282834432,"data_stored":1281314749,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":87482250},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019274,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1558636,"kb_used_data":1238752,"kb_used_omap":1,"kb_used_meta":319870,"kb_avail":92186516,"statfs":{"total":95995035648,"available":94398992384,"internally_reserved":0,"allocated":1268482048,"data_stored":1266970500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":327547334},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435458,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2772260,"kb_used_data":2492124,"kb_used_omap":4,"kb_used_meta":280123,"kb_avail":90972892,"statfs":{"total":95995035648,"available":93156241408,"internally_reserved":0,"allocated":2551934976,"data_stored":2550419233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":286846217},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903310,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2414184,"kb_used_data":2286624,"kb_used_omap":10,"kb_used_meta":127541,"kb_avail":91330968,"statfs":{"total":95995035648,"available":93522911232,"internally_reserved":0,"allocated":2341502976,"data_stored":2339987169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":130602512},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762511,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1945124,"kb_used_data":1678812,"kb_used_omap":7,"kb_used_meta":266296,"kb_avail":91800028,"statfs":{"total":95995035648,"available":94003228672,"internally_reserved":0,"allocated":1719103488,"data_stored":1717586505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":272687458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926038,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1336108,"kb_used_data":1007332,"kb_used_omap":7,"kb_used_meta":328760,"kb_avail":92409044,"statfs":{"total":95995035648,"available":94626861056,"internally_reserved":0,"allocated":1031507968,"data_stored":1029983764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":336650383},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1021841408,"data_stored":1021837336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1710821376,"data_stored":1710817304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2336358400,"data_stored":2336358400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2546860032,"data_stored":2546860032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1261633536,"data_stored":1261633536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1277825024,"data_stored":1277820952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":77856768,"data_stored":77856768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2017132544,"data_stored":2017132544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:25:02.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:02.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:02.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:02.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:02.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:02.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:02.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:02.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:02.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:02.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:02.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:25:02.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:02.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:02.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:02.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:02.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:02.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:02.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:03.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:03.057 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:03.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:03.159 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:03.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:03.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:03.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:03.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:03.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:03.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:03.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:03.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:03.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:03.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:03.701 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:03.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:03.716 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:25:03.717 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:25:03.740 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:03.744 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:25:03.744 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:25:03.772 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:25:03.772 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:25:03.800 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:25:03.801 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:25:03.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:03.818 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:25:03.818 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:25:03.835 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:25:03.835 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:25:03.858 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:25:03.859 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:25:03.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:03.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:03.883 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:25:03.883 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:25:03.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:03.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:03.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:04.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:03 smithi027 ceph-mon[93921]: pgmap v887: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 29488/184788 objects degraded (15.958%); 11462/184788 objects misplaced (6.203%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:25:04.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:03 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3130153385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:04.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:03 smithi027 ceph-mon[98970]: pgmap v887: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 29488/184788 objects degraded (15.958%); 11462/184788 objects misplaced (6.203%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:25:04.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:03 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3130153385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:04.036 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:25:04.036 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:25:04.036 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:25:04.036 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:25:04.036 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:25:04.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:04.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:04.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:04.239 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:04.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:03 smithi171 ceph-mon[90031]: pgmap v887: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 29488/184788 objects degraded (15.958%); 11462/184788 objects misplaced (6.203%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:25:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:03 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3130153385' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:04.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:04.362 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:04.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:04.487 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:04.589 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:04.611 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:04.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:04.713 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:04.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:04.838 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:04.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:25:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:04.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:04.966 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:04.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:04.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:05.078 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:05.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:05.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:05.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:05.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:25:05.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:05.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:05.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:05.364 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:05.466 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:05.488 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:05.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:05.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:05.715 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:05.736 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:05.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:05.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:05.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:05.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:05.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:05.987 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:05.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:05.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:06.055 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:06.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:06.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:06.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:06.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:06.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:05 smithi027 ceph-mon[98970]: pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 29488/184329 objects degraded (15.997%); 11410/184329 objects misplaced (6.190%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:25:06.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:05 smithi027 ceph-mon[93921]: pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 29488/184329 objects degraded (15.997%); 11410/184329 objects misplaced (6.190%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:25:06.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:06.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:06.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:05 smithi171 ceph-mon[90031]: pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 163 op/s; 29488/184329 objects degraded (15.997%); 11410/184329 objects misplaced (6.190%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:25:06.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:06.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:06.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:06.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:06.462 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:06.485 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:06.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:06.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:06.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:06.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:06.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:06.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:06.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:06.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:06.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:06.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:25:06.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:06.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:06.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:06.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:07.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:07.073 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:06 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29488/184329 objects degraded (15.997%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:07.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:07.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:07.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:07.179 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:06 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29488/184329 objects degraded (15.997%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:07.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:06 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29488/184329 objects degraded (15.997%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:07.216 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:154446 6.2 6:49f57ff2:::benchmark_data_smithi027_169445_object29019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e243)", 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:20:45.735650+0000", 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 261.44348895500002, 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021025988999999998, 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 154446 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735650+0000", 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735650+0000", 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735648+0000", 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735657+0000", 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0589999999999992e-06 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735673+0000", 2024-08-29T15:25:07.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6521999999999999e-05 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735683+0000", 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3279999999999995e-06 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735764+0000", 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1403999999999995e-05 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735853+0000", 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9492000000000003e-05 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.735922+0000", 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8484000000000004e-05 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.738928+0000", 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030056670000000001 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.738967+0000", 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9134000000000002e-05 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.756607+0000", 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017640427 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.756645+0000", 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8203e-05 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:20:45.756676+0000", 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0215000000000001e-05 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.223 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:159001 6.2 6:43e3c641:::benchmark_data_smithi027_169445_object33574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:24.412053+0000", 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 222.76708597999999, 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022453205, 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 159001 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412053+0000", 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412053+0000", 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412051+0000", 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412058+0000", 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1799999999999999e-06 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412071+0000", 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2894000000000001e-05 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412077+0000", 2024-08-29T15:25:07.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0820000000000004e-06 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412141+0000", 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3800999999999994e-05 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412232+0000", 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0545999999999997e-05 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:25:07.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.412314+0000", 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2047999999999998e-05 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.415477+0000", 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031632259999999999 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.415885+0000", 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00040792600000000002 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.434420+0000", 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018534761 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.434463+0000", 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3479000000000002e-05 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:24.434506+0000", 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2688999999999997e-05 2024-08-29T15:25:07.229 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:159421 6.e 6:73864256:::benchmark_data_smithi027_169445_object33994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:28.183950+0000", 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 218.995188402, 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014066255999999999, 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 159421 2024-08-29T15:25:07.230 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183950+0000", 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183950+0000", 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183949+0000", 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183955+0000", 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5939999999999999e-06 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183969+0000", 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.328e-05 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.232 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.183976+0000", 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.878e-06 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.184006+0000", 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0870000000000001e-05 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.184097+0000", 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0328000000000007e-05 2024-08-29T15:25:07.233 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.184173+0000", 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6341000000000007e-05 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.187293+0000", 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00312013 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.190702+0000", 2024-08-29T15:25:07.234 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034091999999999998 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.196255+0000", 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055525920000000003 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.196292+0000", 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.752e-05 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:28.198016+0000", 2024-08-29T15:25:07.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017239309999999999 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:161957 6.e 6:7da24517:::benchmark_data_smithi027_169445_object36530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e248)", 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:21:48.349106+0000", 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 198.83003248099999, 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016709833, 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.236 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 161957 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349106+0000", 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349106+0000", 2024-08-29T15:25:07.237 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349104+0000", 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349114+0000", 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.662e-06 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349148+0000", 2024-08-29T15:25:07.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4424000000000001e-05 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349159+0000", 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0261000000000001e-05 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349207+0000", 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8610999999999999e-05 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349312+0000", 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000104647 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.349399+0000", 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7540000000000006e-05 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.352696+0000", 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003296326 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.240 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.352734+0000", 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8361000000000001e-05 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.364229+0000", 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011494881 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.364264+0000", 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5108999999999999e-05 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:21:48.365816+0000", 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015519189999999999 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:164550 6.e 6:7edb1c1a:::benchmark_data_smithi027_169445_object39123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:08.491336+0000", 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 178.68780251999999, 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014258033999999999, 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.242 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 164550 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491336+0000", 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491336+0000", 2024-08-29T15:25:07.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491334+0000", 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491342+0000", 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1869999999999997e-06 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491358+0000", 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6212e-05 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491366+0000", 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.43e-06 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491414+0000", 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8507000000000002e-05 2024-08-29T15:25:07.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491520+0000", 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010611300000000001 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.491609+0000", 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8382999999999997e-05 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.497448+0000", 2024-08-29T15:25:07.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0058391609999999998 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.497464+0000", 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6127e-05 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.503546+0000", 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006082222 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.503602+0000", 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6103999999999997e-05 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:08.505594+0000", 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001991629 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:170812 6.e 6:763b5d9e:::benchmark_data_smithi027_169445_object45385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:57.382005+0000", 2024-08-29T15:25:07.248 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.79713366199999, 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017293317999999998, 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 170812 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382005+0000", 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382005+0000", 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382002+0000", 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382013+0000", 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0265000000000001e-05 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382032+0000", 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9544000000000001e-05 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382041+0000", 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1309999999999995e-06 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382092+0000", 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0677999999999998e-05 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382197+0000", 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000104923 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.382290+0000", 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2849999999999994e-05 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.385679+0000", 2024-08-29T15:25:07.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033892560000000002 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.393595+0000", 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079158749999999993 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.397530+0000", 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039352179999999999 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.397583+0000", 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2981e-05 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.399298+0000", 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017150189999999999 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:170880 6.2 6:478d2f2e:::benchmark_data_smithi027_169445_object45453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2024-08-29T15:25:07.254 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:57.623428+0000", 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 129.555710602, 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014640800000000001, 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 170880 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623428+0000", 2024-08-29T15:25:07.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623428+0000", 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623427+0000", 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623434+0000", 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8429999999999993e-06 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623450+0000", 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5948999999999999e-05 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623458+0000", 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5599999999999996e-06 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623502+0000", 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4481000000000003e-05 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623598+0000", 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5366999999999997e-05 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.623678+0000", 2024-08-29T15:25:07.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0079000000000004e-05 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.626816+0000", 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031378420000000001 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.627641+0000", 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00082584800000000001 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.637972+0000", 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010330191000000001 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.638024+0000", 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2534000000000002e-05 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:57.638069+0000", 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4566e-05 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.260 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:179558 6.e 6:735a4e7b:::benchmark_data_smithi027_169445_object54131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:23:59.572275+0000", 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 67.606863619999999, 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015845703999999999, 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.261 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 179558 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572275+0000", 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572275+0000", 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572274+0000", 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572280+0000", 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3459999999999999e-06 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572294+0000", 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4069e-05 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572300+0000", 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4819999999999999e-06 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572343+0000", 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2277999999999997e-05 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572437+0000", 2024-08-29T15:25:07.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4266999999999997e-05 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.572522+0000", 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5161999999999995e-05 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.575647+0000", 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003124492 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.265 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.576885+0000", 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001238561 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.586359+0000", 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.009473904 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.586416+0000", 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7157999999999997e-05 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:23:59.588121+0000", 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017044250000000001 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:180392 6.e 6:7adf1623:::benchmark_data_smithi027_169445_object54965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:04.748421+0000", 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 62.430717162000001, 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015938539000000002, 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.267 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 180392 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748421+0000", 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748421+0000", 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748420+0000", 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748428+0000", 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9829999999999996e-06 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748443+0000", 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.483e-05 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748451+0000", 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6379999999999993e-06 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748507+0000", 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5587999999999997e-05 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748649+0000", 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014250100000000001 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.748787+0000", 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000138166 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.756767+0000", 2024-08-29T15:25:07.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079795379999999996 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.756820+0000", 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2580000000000001e-05 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.762258+0000", 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0054387339999999998 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.762291+0000", 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2978000000000002e-05 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.764360+0000", 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020686239999999998 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:180393 6.e 6:79273320:::benchmark_data_smithi027_169445_object54966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:04.749945+0000", 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 62.429193859999998, 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016278353999999998, 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 180393 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.749945+0000", 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.749945+0000", 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.749944+0000", 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.749949+0000", 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3410000000000002e-06 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.749961+0000", 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1659e-05 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.749966+0000", 2024-08-29T15:25:07.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4750000000000001e-06 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.752637+0000", 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026702290000000001 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.752792+0000", 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015563400000000001 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.752940+0000", 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014811499999999999 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.756955+0000", 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004014948 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.757049+0000", 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3877999999999998e-05 2024-08-29T15:25:07.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.764431+0000", 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0073820090000000001 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.764548+0000", 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011657199999999999 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:04.766223+0000", 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016753600000000001 2024-08-29T15:25:07.279 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:183027 6.e 6:7d01f711:::benchmark_data_smithi027_169445_object57600:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:22.773601+0000", 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 44.405537844000001, 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019156830999999999, 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 183027 2024-08-29T15:25:07.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773601+0000", 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773601+0000", 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773599+0000", 2024-08-29T15:25:07.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773607+0000", 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7009999999999996e-06 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773625+0000", 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8023999999999999e-05 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773633+0000", 2024-08-29T15:25:07.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9510000000000006e-06 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773690+0000", 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7160999999999997e-05 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773846+0000", 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015623899999999999 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.773990+0000", 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000143993 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.777747+0000", 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037565440000000001 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.788432+0000", 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01068527 2024-08-29T15:25:07.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.790702+0000", 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022698850000000001 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.790765+0000", 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3039000000000004e-05 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:22.792758+0000", 2024-08-29T15:25:07.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019926119999999999 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:183343 6.2 6:4783b0ce:::benchmark_data_smithi027_169445_object57916:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e263)", 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:24.443069+0000", 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 42.736069207, 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010789266, 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.286 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 183343 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443069+0000", 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443069+0000", 2024-08-29T15:25:07.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443068+0000", 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443077+0000", 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6860000000000001e-06 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.288 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443096+0000", 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8910999999999999e-05 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443103+0000", 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0899999999999999e-06 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443147+0000", 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3970000000000001e-05 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443294+0000", 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000147138 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.443437+0000", 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000142609 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.446892+0000", 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034549659999999999 2024-08-29T15:25:07.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.447179+0000", 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000286932 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.453798+0000", 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0066189300000000003 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.453830+0000", 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2262999999999999e-05 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:24.453859+0000", 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8521999999999999e-05 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:184646 6.e 6:78a602d0:::benchmark_data_smithi027_169445_object59219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-08-29T15:25:07.292 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:32.356798+0000", 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 34.822340707000002, 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0122124, 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 184646 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.356798+0000", 2024-08-29T15:25:07.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.356798+0000", 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.356796+0000", 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.294 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.356805+0000", 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6800000000000005e-06 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.356824+0000", 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8039000000000001e-05 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.356833+0000", 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6760000000000002e-06 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.356897+0000", 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3360000000000003e-05 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.357038+0000", 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014122200000000001 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.357167+0000", 2024-08-29T15:25:07.296 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012964000000000001 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.364273+0000", 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0071057669999999998 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.364310+0000", 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7188999999999999e-05 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.297 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.367236+0000", 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029253259999999998 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.367272+0000", 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6384999999999998e-05 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.369010+0000", 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001738173 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.298 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:184648 6.e 6:756d2cf2:::benchmark_data_smithi027_169445_object59221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:32.360297+0000", 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 34.818841272, 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010389812999999999, 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 184648 2024-08-29T15:25:07.299 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360297+0000", 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360297+0000", 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.300 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360295+0000", 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360304+0000", 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7790000000000001e-06 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360322+0000", 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8618999999999999e-05 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.301 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360332+0000", 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0127e-05 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360822+0000", 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00048980599999999997 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.360935+0000", 2024-08-29T15:25:07.302 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000112878 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.361068+0000", 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000132545 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.364397+0000", 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003329383 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.364631+0000", 2024-08-29T15:25:07.303 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00023437299999999999 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.369068+0000", 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044361030000000003 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.369123+0000", 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5517999999999999e-05 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.304 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:32.370687+0000", 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015640599999999999 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:187022 6.2 6:4bc178ed:::benchmark_data_smithi027_169445_object61595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:47.395011+0000", 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 19.784127130999998, 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011300693000000001, 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.305 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 187022 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.395011+0000", 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.395011+0000", 2024-08-29T15:25:07.306 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.395011+0000", 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.395014+0000", 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.745e-06 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.307 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.395018+0000", 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.685e-06 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.395022+0000", 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2820000000000001e-06 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.399187+0000", 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041657539999999998 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.308 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.399345+0000", 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015778100000000001 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.399488+0000", 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014314800000000001 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.403307+0000", 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038185929999999999 2024-08-29T15:25:07.309 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.403408+0000", 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000101266 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.406258+0000", 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028502599999999999 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.406289+0000", 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0769999999999998e-05 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.310 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:47.406312+0000", 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2880000000000001e-05 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:187416 6.e 6:710498ad:::benchmark_data_smithi027_169445_object61989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:49.507049+0000", 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.672090004000001, 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011017328999999999, 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.311 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 187416 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.507049+0000", 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.507049+0000", 2024-08-29T15:25:07.312 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.507047+0000", 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.507054+0000", 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3030000000000002e-06 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.313 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.507077+0000", 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2748e-05 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.507084+0000", 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0199999999999997e-06 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.508537+0000", 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001452933 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.314 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.508650+0000", 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000113338 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.508759+0000", 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010891 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.515483+0000", 2024-08-29T15:25:07.315 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067240989999999999 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.515536+0000", 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2849000000000002e-05 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.516048+0000", 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00051175899999999998 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.316 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.516083+0000", 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4579000000000002e-05 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.518066+0000", 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001983271 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:187419 6.e 6:7cb2346e:::benchmark_data_smithi027_169445_object61992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2024-08-29T15:25:07.317 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:49.510553+0000", 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.668585123, 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013986106999999999, 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 187419 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.510553+0000", 2024-08-29T15:25:07.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.510553+0000", 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.510552+0000", 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.510560+0000", 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1510000000000004e-06 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.510577+0000", 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6793e-05 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.510586+0000", 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6409999999999997e-06 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.512004+0000", 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014176729999999999 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.512158+0000", 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000153547 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.512290+0000", 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000132793 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.515578+0000", 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032874179999999999 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.518133+0000", 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00255478 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.522354+0000", 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042215040000000001 2024-08-29T15:25:07.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.522396+0000", 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2144000000000002e-05 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.524540+0000", 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021433020000000001 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.323 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:187536 6.e 6:732ee646:::benchmark_data_smithi027_169445_object62109:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:49.755211+0000", 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.423927242000001, 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011941851, 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 187536 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.324 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.755211+0000", 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.755211+0000", 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.755211+0000", 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.325 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.755213+0000", 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2979999999999999e-06 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.755217+0000", 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9339999999999999e-06 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.755220+0000", 2024-08-29T15:25:07.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9909999999999998e-06 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.759146+0000", 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039258660000000001 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.759298+0000", 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015170599999999999 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.759442+0000", 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000144339 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.765436+0000", 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0059938270000000002 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.765489+0000", 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2923999999999997e-05 2024-08-29T15:25:07.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.765499+0000", 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0190999999999999e-05 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.765552+0000", 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2921999999999999e-05 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.767153+0000", 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016011230000000001 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:187537 6.e 6:7b888883:::benchmark_data_smithi027_169445_object62110:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:49.757684+0000", 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.421454398000002, 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017684841, 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.330 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 187537 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.757684+0000", 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.757684+0000", 2024-08-29T15:25:07.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.757683+0000", 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.757691+0000", 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0309999999999998e-06 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.757717+0000", 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.599e-05 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.757726+0000", 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2920000000000001e-06 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.767207+0000", 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0094814789999999993 2024-08-29T15:25:07.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.767352+0000", 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014415900000000001 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.767493+0000", 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014140499999999999 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.771230+0000", 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003736802 2024-08-29T15:25:07.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.771673+0000", 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00044340299999999998 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.773238+0000", 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001565108 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.773277+0000", 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8287999999999997e-05 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:49.775369+0000", 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020924120000000001 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:188364 6.e 6:78c784ae:::benchmark_data_smithi027_169445_object62937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:54.752379+0000", 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 12.426759904000001, 2024-08-29T15:25:07.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0137629, 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 188364 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.752379+0000", 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.752379+0000", 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.752377+0000", 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.752384+0000", 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1190000000000002e-06 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.752398+0000", 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4131000000000001e-05 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.752406+0000", 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7980000000000006e-06 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.754066+0000", 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016605089999999999 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.754209+0000", 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014339100000000001 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.754355+0000", 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014577599999999999 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.757991+0000", 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036358839999999998 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.760963+0000", 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002971942 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.764286+0000", 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033226390000000001 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.764337+0000", 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1811000000000003e-05 2024-08-29T15:25:07.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:54.766142+0000", 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001804066 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:07.342 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:25:07.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:07.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:07.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:07.490 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:07.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:07.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:07.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:07.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:07.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:07.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:07.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:07.864 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:07.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:07.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:07 smithi027 ceph-mon[93921]: pgmap v889: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 29436/182331 objects degraded (16.144%); 11327/182331 objects misplaced (6.212%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:07.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2192460558' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:07.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:07 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:07.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:07 smithi027 ceph-mon[98970]: pgmap v889: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 29436/182331 objects degraded (16.144%); 11327/182331 objects misplaced (6.212%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:07.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2192460558' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:07.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:07 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:07.897 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:25:07.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:07.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:07.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:07.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:08.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:08.117 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:08.148 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:07 smithi171 ceph-mon[90031]: pgmap v889: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 29436/182331 objects degraded (16.144%); 11327/182331 objects misplaced (6.212%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:08.148 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2192460558' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:08.148 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:07 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:08.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:25:08.219 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:08.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:08.343 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:08.354 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:08.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:08.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:08.391 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:08.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:08.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:08.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:08.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.620 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:08.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:08.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:08.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:08.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:08.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:08.869 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:08.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:08.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:08.994 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:08.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:08.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:09.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:09.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:09.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:09.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:09.147 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:08 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:25:09.147 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:08 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:09.147 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:08 smithi027 ceph-mon[93921]: osdmap e268: 8 total, 8 up, 8 in 2024-08-29T15:25:09.147 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:08 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:25:09.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:08 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:09.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:08 smithi027 ceph-mon[98970]: osdmap e268: 8 total, 8 up, 8 in 2024-08-29T15:25:09.148 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:25:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:25:09] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:25:09.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:09.247 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:08 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:25:09.247 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:08 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:09.247 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:08 smithi171 ceph-mon[90031]: osdmap e268: 8 total, 8 up, 8 in 2024-08-29T15:25:09.253 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:09.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:09.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:09.355 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:09.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:09.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:09.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:09.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:09.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:09.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:09.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:09.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:09.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:09.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:09.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:09.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:09.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:09.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:09.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:10.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:10.009 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:09 smithi171 ceph-mon[90031]: pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 29436/182172 objects degraded (16.158%); 11271/182172 objects misplaced (6.187%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:10.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:09 smithi027 ceph-mon[98970]: pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 29436/182172 objects degraded (16.158%); 11271/182172 objects misplaced (6.187%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:09 smithi027 ceph-mon[93921]: pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 29436/182172 objects degraded (16.158%); 11271/182172 objects misplaced (6.187%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:10.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:10.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:10.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:10.230 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:10.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:10.264 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:25:10.264 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:25:10.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.264 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190841 6.3 6:cb805add:::benchmark_data_smithi027_169445_object65414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.264 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:10.178977+0000", 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.083271655, 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083295542, 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for missing object", 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190841 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.265 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.178977+0000", 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.178977+0000", 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.178976+0000", 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.178985+0000", 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8990000000000007e-06 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.179000+0000", 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5537999999999999e-05 2024-08-29T15:25:10.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.179008+0000", 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3129999999999999e-06 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.214537+0000", 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035528271 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.220337+0000", 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0057999219999999999 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190809 6.3 6:c71c2c98:::benchmark_data_smithi027_169445_object65382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.269 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.734131+0000", 2024-08-29T15:25:10.270 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.52811808000000005, 2024-08-29T15:25:10.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.52818748100000001, 2024-08-29T15:25:10.270 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.270 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.270 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.270 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.270 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190809 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.734131+0000", 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.734131+0000", 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.734129+0000", 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.734137+0000", 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9049999999999997e-06 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.734149+0000", 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2183e-05 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.734156+0000", 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1010000000000003e-06 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.753901+0000", 2024-08-29T15:25:10.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019744782999999998 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.760950+0000", 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070485110000000004 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.841504+0000", 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080553979999999997 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.274 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.847026+0000", 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055222220000000002 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPush(6.3 268/218 [PushOp(6:cb805add:::benchmark_data_smithi027_169445_object65414:head, version: 179'4116, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:cb805add:::benchmark_data_smithi027_169445_object65414:head@179'4116, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:10.221029+0000", 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.041219719000000002, 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041350676000000003, 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:25:10.275 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.221029+0000", 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.221029+0000", 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.221028+0000", 2024-08-29T15:25:10.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.221098+0000", 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9739000000000004e-05 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.221114+0000", 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5848000000000001e-05 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.221125+0000", 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.077e-05 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190810 6.3 6:cc04dbe0:::benchmark_data_smithi027_169445_object65383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.736011+0000", 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.52623816199999995, 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.526396219, 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.278 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.279 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.279 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.279 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.279 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190810 2024-08-29T15:25:10.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.736011+0000", 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.736011+0000", 2024-08-29T15:25:10.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.736010+0000", 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.736017+0000", 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9199999999999998e-06 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.736029+0000", 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2193e-05 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.736036+0000", 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8600000000000004e-06 2024-08-29T15:25:10.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.761970+0000", 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025934123999999999 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.769365+0000", 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00739552 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.815016+0000", 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.045650700000000002 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.820409+0000", 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0053935659999999998 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPush(6.3 268/218 [PushOp(6:cc75485f:::benchmark_data_smithi027_169445_object65425:head, version: 179'4117, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:cc75485f:::benchmark_data_smithi027_169445_object65425:head@179'4117, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-29T15:25:10.284 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:10.226843+0000", 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.035405963999999998, 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035618041000000003, 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.226843+0000", 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.226843+0000", 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.226842+0000", 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.286 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.226870+0000", 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7263999999999998e-05 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.226885+0000", 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5140000000000001e-05 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.226895+0000", 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0645999999999999e-05 2024-08-29T15:25:10.287 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190711 6.3 6:c6a306de:::benchmark_data_smithi027_169445_object65284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.209765+0000", 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.052484551, 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0527229090000001, 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.288 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190711 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.209765+0000", 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.209765+0000", 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.209763+0000", 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.209772+0000", 2024-08-29T15:25:10.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8170000000000005e-06 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.209788+0000", 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6262000000000001e-05 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.209796+0000", 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2309999999999996e-06 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.284598+0000", 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.074801714000000005 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.292 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.290430+0000", 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005831943 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.329160+0000", 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038730555 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.334812+0000", 2024-08-29T15:25:10.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056521879999999998 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190784 6.3 6:cd2d56dc:::benchmark_data_smithi027_169445_object65357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.388722+0000", 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.87352684999999997, 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.87380590400000002, 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.294 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190784 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.388722+0000", 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.388722+0000", 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.388720+0000", 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.388728+0000", 2024-08-29T15:25:10.296 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1270000000000003e-06 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.388743+0000", 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4894e-05 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.388750+0000", 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0589999999999997e-06 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.297 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.487550+0000", 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098799725000000005 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.493971+0000", 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006421263 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.554488+0000", 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.060516552000000001 2024-08-29T15:25:10.298 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.559811+0000", 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0053229940000000002 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190785 6.3 6:c34ad1b1:::benchmark_data_smithi027_169445_object65358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.392432+0000", 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.86981693800000004, 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.87013968900000005, 2024-08-29T15:25:10.299 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190785 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.392432+0000", 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.300 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.392432+0000", 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.392431+0000", 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.301 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.392436+0000", 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.536e-06 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.392447+0000", 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0504e-05 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.302 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.392453+0000", 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7139999999999998e-06 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.494685+0000", 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10223212600000001 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.500347+0000", 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056622130000000001 2024-08-29T15:25:10.303 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.581340+0000", 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080992854000000003 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.586866+0000", 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055258149999999999 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.304 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190852 6.3 6.fa12ae33 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:10.246060+0000", 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.016189188, 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016551443999999998, 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190852 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.305 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.246060+0000", 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.246060+0000", 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.306 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.246058+0000", 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.246067+0000", 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.106e-06 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.307 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.246079+0000", 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2062000000000001e-05 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.246086+0000", 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1790000000000002e-06 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44912.0:190691 6.3 e268/218)", 2024-08-29T15:25:10.308 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:10.260096+0000", 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.002153557, 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025425560000000001, 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260096+0000", 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.309 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260096+0000", 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260094+0000", 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260102+0000", 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1729999999999998e-06 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.310 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260116+0000", 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4936999999999999e-05 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260125+0000", 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7579999999999998e-06 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.311 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.44912.0:190691 6.3 e268/218)", 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:10.260848+0000", 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0014009129999999999, 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001812662, 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260848+0000", 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.312 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260848+0000", 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260847+0000", 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260853+0000", 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1259999999999997e-06 2024-08-29T15:25:10.313 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260876+0000", 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3563000000000001e-05 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.260882+0000", 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4010000000000002e-06 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.314 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190726 6.3 6:cc9077ba:::benchmark_data_smithi027_169445_object65299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.235036+0000", 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.0272134020000001, 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.027651206, 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190726 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.315 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.235036+0000", 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.235036+0000", 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.235034+0000", 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.316 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.235042+0000", 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9719999999999992e-06 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.235057+0000", 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4949e-05 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.235064+0000", 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4930000000000001e-06 2024-08-29T15:25:10.317 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.291995+0000", 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.056931041000000002 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.297719+0000", 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0057232949999999998 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.318 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.393133+0000", 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095413964000000004 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.398696+0000", 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055630419999999998 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190817 6.3 6:c1521d05:::benchmark_data_smithi027_169445_object65390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.945782+0000", 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.316466633, 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.31695804999999999, 2024-08-29T15:25:10.319 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190817 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.945782+0000", 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.945782+0000", 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.945781+0000", 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.945789+0000", 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7549999999999994e-06 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.945801+0000", 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2208999999999999e-05 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.945808+0000", 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.635e-06 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.985001+0000", 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039193191000000002 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.991011+0000", 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060099209999999997 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.032677+0000", 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041666584999999999 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.040060+0000", 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007383049 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190733 6.3 6:c8e70caf:::benchmark_data_smithi027_169445_object65306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.249071+0000", 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.0131784939999999, 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0137109209999999, 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190733 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.324 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.325 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.325 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.325 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.249071+0000", 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.249071+0000", 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.249069+0000", 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.249077+0000", 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1240000000000005e-06 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.249090+0000", 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3332e-05 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.249098+0000", 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5820000000000004e-06 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.298509+0000", 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049411412000000002 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.304093+0000", 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055841529999999997 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.356434+0000", 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.052340790999999998 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.365471+0000", 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0090370769999999993 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190821 6.3 6:ce5a0085:::benchmark_data_smithi027_169445_object65394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.962668+0000", 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.29958107099999998, 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.30015700899999997, 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.329 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190821 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.962668+0000", 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.962668+0000", 2024-08-29T15:25:10.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.962667+0000", 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.962673+0000", 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8510000000000001e-06 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.962686+0000", 2024-08-29T15:25:10.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2485e-05 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.962693+0000", 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0659999999999998e-06 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.040582+0000", 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.077889271999999996 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.046063+0000", 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0054802510000000002 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.100333+0000", 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.054270279999999997 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.105821+0000", 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00548841 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190793 6.3 6:c72dc212:::benchmark_data_smithi027_169445_object65366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.507602+0000", 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.75464735299999997, 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.75527528099999997, 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.334 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190793 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.507602+0000", 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.507602+0000", 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.507600+0000", 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.507608+0000", 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.259e-06 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.507623+0000", 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4630999999999999e-05 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.507632+0000", 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.628e-06 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.605261+0000", 2024-08-29T15:25:10.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.097628554000000006 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.611421+0000", 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0061604579999999997 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.650061+0000", 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038639104000000001 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.655596+0000", 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055353579999999998 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190744 6.3 6:cadbbb79:::benchmark_data_smithi027_169445_object65317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.267045+0000", 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.99520361199999996, 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99587236000000001, 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.339 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190744 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.267045+0000", 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.267045+0000", 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.267044+0000", 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.267051+0000", 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.328e-06 2024-08-29T15:25:10.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.267066+0000", 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5384e-05 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.267074+0000", 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7400000000000004e-06 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.400744+0000", 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.1336696 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.406383+0000", 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056392719999999999 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.462044+0000", 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.055660702999999999 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.467641+0000", 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055977470000000001 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190829 6.3 6:c76cfe0d:::benchmark_data_smithi027_169445_object65402:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:10.048971+0000", 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.21327822699999999, 2024-08-29T15:25:10.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.213987444, 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190829 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.048971+0000", 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.048971+0000", 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.048970+0000", 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.048975+0000", 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4589999999999997e-06 2024-08-29T15:25:10.346 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.048986+0000", 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.043e-05 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.048990+0000", 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8579999999999999e-06 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.125497+0000", 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.076506478000000003 2024-08-29T15:25:10.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.131549+0000", 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060526540000000002 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.189153+0000", 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.057603999000000003 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.194788+0000", 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056346900000000004 2024-08-29T15:25:10.348 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190691 6.3 6:c0c97af7:::benchmark_data_smithi027_169445_object65264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.176834+0000", 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.085414892, 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.086179395, 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.349 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190691 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.176834+0000", 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.176834+0000", 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.176832+0000", 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.176841+0000", 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3179999999999996e-06 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.176858+0000", 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6086000000000002e-05 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.176868+0000", 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0305e-05 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.212994+0000", 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036126449999999997 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.219850+0000", 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0068552650000000001 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.277310+0000", 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.057459986999999997 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.283865+0000", 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0065554239999999998 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.241689+0000", 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95782387099999999 2024-08-29T15:25:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.247696+0000", 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060066870000000001 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:10.256681+0000", 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0089855890000000004 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:190806 6.3 6:cc50dad5:::benchmark_data_smithi027_169445_object65379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:25:09.668446+0000", 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.59380322900000004, 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.594614054, 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for rw locks", 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 190806 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.668446+0000", 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.668446+0000", 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.668444+0000", 2024-08-29T15:25:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.668451+0000", 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9280000000000004e-06 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.668464+0000", 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2738999999999999e-05 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.668471+0000", 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.063e-06 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.747510+0000", 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.079039148000000004 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.753177+0000", 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056669340000000002 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.790116+0000", 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036938805999999998 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:25:09.795515+0000", 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0053989989999999998 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 20 2024-08-29T15:25:10.359 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:25:10.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:10.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:10.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:10.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:10.506 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:10.608 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:10.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:10.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:10.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:10.757 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:10.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:10.864 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:25:10.865 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:25:10.880 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:10.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:10.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:10.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:25:10.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:11.004 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:11.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:11.130 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:11.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:25:11.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:11.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:11.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:11.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:11.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:11.439 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:11.488 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:11.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:11.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:11.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:11.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:11.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:11.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:11.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:11.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:11.868 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":892,"stamp":"2024-08-29T15:25:10.621151+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3969718269,"num_objects":60529,"num_object_clones":0,"num_object_copies":181587,"num_objects_missing_on_primary":4768,"num_objects_missing":4768,"num_objects_degraded":29436,"num_objects_misplaced":11271,"num_objects_unfound":0,"num_objects_dirty":60529,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":191472,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17363,"num_bytes_recovered":1016728781,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125508,"ondisk_log_size":125508,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13641280,"kb_used_data":11711288,"kb_used_omap":43,"kb_used_meta":1929876,"kb_avail":736319936,"statfs":{"total":767960285184,"available":753991614464,"internally_reserved":0,"allocated":11992358912,"data_stored":11980120366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1976193516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-109248512,"num_objects":-1667,"num_object_clones":0,"num_object_copies":-5001,"num_objects_missing_on_primary":-108,"num_objects_missing":-108,"num_objects_degraded":-117,"num_objects_misplaced":-298,"num_objects_unfound":0,"num_objects_dirty":-1667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1667,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":223,"num_bytes_recovered":14614528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003351"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":419,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873512+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:24:58.873512+0000","last_peered":"2024-08-29T15:24:58.873512+0000","last_clean":"2024-08-29T15:24:58.873512+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:24:58.873512+0000","last_undegraded":"2024-08-29T15:24:58.873512+0000","last_fullsized":"2024-08-29T15:24:58.873512+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":46,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:58.873260+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:24:58.873260+0000","last_peered":"2024-08-29T15:24:58.873260+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:24:58.873260+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:24:58.873260+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":135,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:51.182281+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:24:51.182281+0000","last_peered":"2024-08-29T15:24:51.182281+0000","last_clean":"2024-08-29T15:24:51.182281+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:24:51.182281+0000","last_undegraded":"2024-08-29T15:24:51.182281+0000","last_fullsized":"2024-08-29T15:24:51.182281+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":387,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893744+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:25:07.893744+0000","last_peered":"2024-08-29T15:25:07.893744+0000","last_clean":"2024-08-29T15:25:07.893744+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:25:07.893744+0000","last_undegraded":"2024-08-29T15:25:07.893744+0000","last_fullsized":"2024-08-29T15:25:07.893744+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":387,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893430+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:25:07.893430+0000","last_peered":"2024-08-29T15:25:07.893430+0000","last_clean":"2024-08-29T15:25:07.893430+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:25:07.893430+0000","last_undegraded":"2024-08-29T15:25:07.893430+0000","last_fullsized":"2024-08-29T15:25:07.893430+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872964+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:24:58.872964+0000","last_peered":"2024-08-29T15:24:58.872964+0000","last_clean":"2024-08-29T15:24:58.872964+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:24:58.872964+0000","last_undegraded":"2024-08-29T15:24:58.872964+0000","last_fullsized":"2024-08-29T15:24:58.872964+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":196,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.876244+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:24:58.876244+0000","last_peered":"2024-08-29T15:24:58.876244+0000","last_clean":"2024-08-29T15:24:58.876244+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:24:58.876244+0000","last_undegraded":"2024-08-29T15:24:58.876244+0000","last_fullsized":"2024-08-29T15:24:58.876244+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":137,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.884607+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:24:58.884607+0000","last_peered":"2024-08-29T15:24:58.884607+0000","last_clean":"2024-08-29T15:24:58.884607+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:24:58.884607+0000","last_undegraded":"2024-08-29T15:24:58.884607+0000","last_fullsized":"2024-08-29T15:24:58.884607+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"267'12048","reported_seq":20094,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:05.899444+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:25:05.899444+0000","last_peered":"2024-08-29T15:25:05.899444+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:25:05.899444+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"176'2000","ondisk_log_start":"176'2000","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10048,"log_dups_size":0,"ondisk_log_size":10048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253231104,"num_objects":3864,"num_object_clones":0,"num_object_copies":11592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12048,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":939}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":45,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:58.873642+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:24:58.873642+0000","last_peered":"2024-08-29T15:24:58.873642+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:24:58.873642+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:24:58.873642+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":153,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.815478+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:24:48.815478+0000","last_peered":"2024-08-29T15:24:48.815478+0000","last_clean":"2024-08-29T15:24:48.815478+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:24:48.815478+0000","last_undegraded":"2024-08-29T15:24:48.815478+0000","last_fullsized":"2024-08-29T15:24:48.815478+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"267'11759","reported_seq":11972,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:25:05.687485+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:25:05.687485+0000","last_peered":"2024-08-29T15:25:05.687485+0000","last_clean":"2024-08-29T15:25:05.687485+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:25:05.687485+0000","last_undegraded":"2024-08-29T15:25:05.687485+0000","last_fullsized":"2024-08-29T15:25:05.687485+0000","mapping_epoch":168,"log_start":"180'5028","ondisk_log_start":"180'5028","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248446976,"num_objects":3791,"num_object_clones":0,"num_object_copies":11373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11759,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":45,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:58.877779+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:24:58.877779+0000","last_peered":"2024-08-29T15:24:58.877779+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:24:58.877779+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:24:58.877779+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872749+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:24:58.872749+0000","last_peered":"2024-08-29T15:24:58.872749+0000","last_clean":"2024-08-29T15:24:58.872749+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:24:58.872749+0000","last_undegraded":"2024-08-29T15:24:58.872749+0000","last_fullsized":"2024-08-29T15:24:58.872749+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"267'11967","reported_seq":12150,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:05.823895+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:25:05.823895+0000","last_peered":"2024-08-29T15:25:05.823895+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:25:05.823895+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"180'5274","ondisk_log_start":"180'5274","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6693,"log_dups_size":0,"ondisk_log_size":6693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245432320,"num_objects":3745,"num_object_clones":0,"num_object_copies":11235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11967,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1268}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":203,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893790+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:25:07.893790+0000","last_peered":"2024-08-29T15:25:07.893790+0000","last_clean":"2024-08-29T15:25:07.893790+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:25:07.893790+0000","last_undegraded":"2024-08-29T15:25:07.893790+0000","last_fullsized":"2024-08-29T15:25:07.893790+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.885195+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:24:58.885195+0000","last_peered":"2024-08-29T15:24:58.885195+0000","last_clean":"2024-08-29T15:24:58.885195+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:24:58.885195+0000","last_undegraded":"2024-08-29T15:24:58.885195+0000","last_fullsized":"2024-08-29T15:24:58.885195+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"268'12001","reported_seq":17436,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:08.927873+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:25:08.927873+0000","last_peered":"2024-08-29T15:25:08.927873+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:25:08.927873+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"179'3862","ondisk_log_start":"179'3862","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8139,"log_dups_size":0,"ondisk_log_size":8139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247529472,"num_objects":3777,"num_object_clones":0,"num_object_copies":11331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12001,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3777}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":153,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.816153+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:24:48.816153+0000","last_peered":"2024-08-29T15:24:48.816153+0000","last_clean":"2024-08-29T15:24:48.816153+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:24:48.816153+0000","last_undegraded":"2024-08-29T15:24:48.816153+0000","last_fullsized":"2024-08-29T15:24:48.816153+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":401,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893381+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:25:07.893381+0000","last_peered":"2024-08-29T15:25:07.893381+0000","last_clean":"2024-08-29T15:25:07.893381+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:25:07.893381+0000","last_undegraded":"2024-08-29T15:25:07.893381+0000","last_fullsized":"2024-08-29T15:25:07.893381+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"268'11969","reported_seq":12124,"reported_epoch":268,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:09.649083+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:25:09.649083+0000","last_peered":"2024-08-29T15:25:09.649083+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:25:09.649083+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:25:09.649083+0000","mapping_epoch":234,"log_start":"179'4033","ondisk_log_start":"179'4033","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7936,"log_dups_size":0,"ondisk_log_size":7936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251985920,"num_objects":3845,"num_object_clones":0,"num_object_copies":11535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11969,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":1368},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":394,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873007+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:24:58.873007+0000","last_peered":"2024-08-29T15:24:58.873007+0000","last_clean":"2024-08-29T15:24:58.873007+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:24:58.873007+0000","last_undegraded":"2024-08-29T15:24:58.873007+0000","last_fullsized":"2024-08-29T15:24:58.873007+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872970+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:24:58.872970+0000","last_peered":"2024-08-29T15:24:58.872970+0000","last_clean":"2024-08-29T15:24:58.872970+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:24:58.872970+0000","last_undegraded":"2024-08-29T15:24:58.872970+0000","last_fullsized":"2024-08-29T15:24:58.872970+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"267'11958","reported_seq":12187,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:06.394188+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:25:06.394188+0000","last_peered":"2024-08-29T15:25:06.394188+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:25:06.394188+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:25:06.394188+0000","mapping_epoch":234,"log_start":"180'5268","ondisk_log_start":"180'5268","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254541824,"num_objects":3884,"num_object_clones":0,"num_object_copies":11652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11958,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":1419},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.885125+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:24:58.885125+0000","last_peered":"2024-08-29T15:24:58.885125+0000","last_clean":"2024-08-29T15:24:58.885125+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:24:58.885125+0000","last_undegraded":"2024-08-29T15:24:58.885125+0000","last_fullsized":"2024-08-29T15:24:58.885125+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873408+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:24:58.873408+0000","last_peered":"2024-08-29T15:24:58.873408+0000","last_clean":"2024-08-29T15:24:58.873408+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:24:58.873408+0000","last_undegraded":"2024-08-29T15:24:58.873408+0000","last_fullsized":"2024-08-29T15:24:58.873408+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"267'11815","reported_seq":12229,"reported_epoch":267,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:25:06.254184+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:25:06.254184+0000","last_peered":"2024-08-29T15:25:06.254184+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:25:06.254184+0000","last_undegraded":"2024-08-29T15:25:06.254184+0000","last_fullsized":"2024-08-29T15:25:06.254184+0000","mapping_epoch":202,"log_start":"180'5067","ondisk_log_start":"180'5067","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6748,"log_dups_size":0,"ondisk_log_size":6748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251854848,"num_objects":3843,"num_object_clones":0,"num_object_copies":11529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3843,"num_objects_unfound":0,"num_objects_dirty":3843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11815,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":3843}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872825+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:24:58.872825+0000","last_peered":"2024-08-29T15:24:58.872825+0000","last_clean":"2024-08-29T15:24:58.872825+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:24:58.872825+0000","last_undegraded":"2024-08-29T15:24:58.872825+0000","last_fullsized":"2024-08-29T15:24:58.872825+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":253,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.877874+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:24:58.877874+0000","last_peered":"2024-08-29T15:24:58.877874+0000","last_clean":"2024-08-29T15:24:58.877874+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:24:58.877874+0000","last_undegraded":"2024-08-29T15:24:58.877874+0000","last_fullsized":"2024-08-29T15:24:58.877874+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"267'11996","reported_seq":12144,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:25:05.891609+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:25:05.891609+0000","last_peered":"2024-08-29T15:25:05.891609+0000","last_clean":"2024-08-29T15:25:05.891609+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:25:05.891609+0000","last_undegraded":"2024-08-29T15:25:05.891609+0000","last_fullsized":"2024-08-29T15:25:05.891609+0000","mapping_epoch":218,"log_start":"176'1978","ondisk_log_start":"176'1978","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246611992,"num_objects":3764,"num_object_clones":0,"num_object_copies":11292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3764,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11996,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":397,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873680+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:24:58.873680+0000","last_peered":"2024-08-29T15:24:58.873680+0000","last_clean":"2024-08-29T15:24:58.873680+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:24:58.873680+0000","last_undegraded":"2024-08-29T15:24:58.873680+0000","last_fullsized":"2024-08-29T15:24:58.873680+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":389,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.876328+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:24:58.876328+0000","last_peered":"2024-08-29T15:24:58.876328+0000","last_clean":"2024-08-29T15:24:58.876328+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:24:58.876328+0000","last_undegraded":"2024-08-29T15:24:58.876328+0000","last_fullsized":"2024-08-29T15:24:58.876328+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"267'11480","reported_seq":11676,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:25:05.990117+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:25:05.990117+0000","last_peered":"2024-08-29T15:25:05.990117+0000","last_clean":"2024-08-29T15:25:05.990117+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:25:05.990117+0000","last_undegraded":"2024-08-29T15:25:05.990117+0000","last_fullsized":"2024-08-29T15:25:05.990117+0000","mapping_epoch":201,"log_start":"176'1400","ondisk_log_start":"176'1400","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244318208,"num_objects":3728,"num_object_clones":0,"num_object_copies":11184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11480,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":490,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.894391+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:25:07.894391+0000","last_peered":"2024-08-29T15:25:07.894391+0000","last_clean":"2024-08-29T15:25:07.894391+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:25:07.894391+0000","last_undegraded":"2024-08-29T15:25:07.894391+0000","last_fullsized":"2024-08-29T15:25:07.894391+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":737,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.815696+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:24:48.815696+0000","last_peered":"2024-08-29T15:24:48.815696+0000","last_clean":"2024-08-29T15:24:48.815696+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:24:48.815696+0000","last_undegraded":"2024-08-29T15:24:48.815696+0000","last_fullsized":"2024-08-29T15:24:48.815696+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":245,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.815848+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:24:48.815848+0000","last_peered":"2024-08-29T15:24:48.815848+0000","last_clean":"2024-08-29T15:24:48.815848+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:24:48.815848+0000","last_undegraded":"2024-08-29T15:24:48.815848+0000","last_fullsized":"2024-08-29T15:24:48.815848+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.884769+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:24:58.884769+0000","last_peered":"2024-08-29T15:24:58.884769+0000","last_clean":"2024-08-29T15:24:58.884769+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:24:58.884769+0000","last_undegraded":"2024-08-29T15:24:58.884769+0000","last_fullsized":"2024-08-29T15:24:58.884769+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"267'11922","reported_seq":12333,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:25:05.390224+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:25:05.390224+0000","last_peered":"2024-08-29T15:25:05.390224+0000","last_clean":"2024-08-29T15:25:05.390224+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:25:05.390224+0000","last_undegraded":"2024-08-29T15:25:05.390224+0000","last_fullsized":"2024-08-29T15:25:05.390224+0000","mapping_epoch":201,"log_start":"180'4853","ondisk_log_start":"180'4853","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246415360,"num_objects":3760,"num_object_clones":0,"num_object_copies":11280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11922,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":862,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875009+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:24:58.875009+0000","last_peered":"2024-08-29T15:24:58.875009+0000","last_clean":"2024-08-29T15:24:58.875009+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:24:58.875009+0000","last_undegraded":"2024-08-29T15:24:58.875009+0000","last_fullsized":"2024-08-29T15:24:58.875009+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":402,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893283+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:25:07.893283+0000","last_peered":"2024-08-29T15:25:07.893283+0000","last_clean":"2024-08-29T15:25:07.893283+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:25:07.893283+0000","last_undegraded":"2024-08-29T15:25:07.893283+0000","last_fullsized":"2024-08-29T15:25:07.893283+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":410,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893654+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:25:07.893654+0000","last_peered":"2024-08-29T15:25:07.893654+0000","last_clean":"2024-08-29T15:25:07.893654+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:25:07.893654+0000","last_undegraded":"2024-08-29T15:25:07.893654+0000","last_fullsized":"2024-08-29T15:25:07.893654+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"267'11868","reported_seq":19035,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:06.146628+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:25:06.146628+0000","last_peered":"2024-08-29T15:25:06.146628+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:25:06.146628+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"180'5130","ondisk_log_start":"180'5130","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6738,"log_dups_size":0,"ondisk_log_size":6738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250478592,"num_objects":3822,"num_object_clones":0,"num_object_copies":11466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11868,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3492,"num_bytes_recovered":228851712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1384},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":671,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872751+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:24:58.872751+0000","last_peered":"2024-08-29T15:24:58.872751+0000","last_clean":"2024-08-29T15:24:58.872751+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:24:58.872751+0000","last_undegraded":"2024-08-29T15:24:58.872751+0000","last_fullsized":"2024-08-29T15:24:58.872751+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":99,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.884363+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:24:58.884363+0000","last_peered":"2024-08-29T15:24:58.884363+0000","last_clean":"2024-08-29T15:24:58.884363+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:24:58.884363+0000","last_undegraded":"2024-08-29T15:24:58.884363+0000","last_fullsized":"2024-08-29T15:24:58.884363+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":434,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.876108+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:24:58.876108+0000","last_peered":"2024-08-29T15:24:58.876108+0000","last_clean":"2024-08-29T15:24:58.876108+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:24:58.876108+0000","last_undegraded":"2024-08-29T15:24:58.876108+0000","last_fullsized":"2024-08-29T15:24:58.876108+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"267'11926","reported_seq":12149,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:25:06.413542+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:25:06.413542+0000","last_peered":"2024-08-29T15:25:06.413542+0000","last_clean":"2024-08-29T15:25:06.413542+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:25:06.413542+0000","last_undegraded":"2024-08-29T15:25:06.413542+0000","last_fullsized":"2024-08-29T15:25:06.413542+0000","mapping_epoch":234,"log_start":"180'5197","ondisk_log_start":"180'5197","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248119296,"num_objects":3786,"num_object_clones":0,"num_object_copies":11358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11926,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":205,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.815615+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:24:48.815615+0000","last_peered":"2024-08-29T15:24:48.815615+0000","last_clean":"2024-08-29T15:24:48.815615+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:24:48.815615+0000","last_undegraded":"2024-08-29T15:24:48.815615+0000","last_fullsized":"2024-08-29T15:24:48.815615+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873162+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:24:58.873162+0000","last_peered":"2024-08-29T15:24:58.873162+0000","last_clean":"2024-08-29T15:24:58.873162+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:24:58.873162+0000","last_undegraded":"2024-08-29T15:24:58.873162+0000","last_fullsized":"2024-08-29T15:24:58.873162+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":112,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.815532+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:24:48.815532+0000","last_peered":"2024-08-29T15:24:48.815532+0000","last_clean":"2024-08-29T15:24:48.815532+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:24:48.815532+0000","last_undegraded":"2024-08-29T15:24:48.815532+0000","last_fullsized":"2024-08-29T15:24:48.815532+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"268'12003","reported_seq":12226,"reported_epoch":268,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:08.906033+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:25:08.906033+0000","last_peered":"2024-08-29T15:25:08.906033+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:25:08.906033+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"179'3842","ondisk_log_start":"179'3842","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8161,"log_dups_size":0,"ondisk_log_size":8161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250544128,"num_objects":3823,"num_object_clones":0,"num_object_copies":11469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12003,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1400}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":137,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.816216+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:24:48.816216+0000","last_peered":"2024-08-29T15:24:48.816216+0000","last_clean":"2024-08-29T15:24:48.816216+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:24:48.816216+0000","last_undegraded":"2024-08-29T15:24:48.816216+0000","last_fullsized":"2024-08-29T15:24:48.816216+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":827,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875571+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:24:58.875571+0000","last_peered":"2024-08-29T15:24:58.875571+0000","last_clean":"2024-08-29T15:24:58.875571+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:24:58.875571+0000","last_undegraded":"2024-08-29T15:24:58.875571+0000","last_fullsized":"2024-08-29T15:24:58.875571+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":404,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875491+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:24:58.875491+0000","last_peered":"2024-08-29T15:24:58.875491+0000","last_clean":"2024-08-29T15:24:58.875491+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:24:58.875491+0000","last_undegraded":"2024-08-29T15:24:58.875491+0000","last_fullsized":"2024-08-29T15:24:58.875491+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"267'11926","reported_seq":12703,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:05.614210+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:25:05.614210+0000","last_peered":"2024-08-29T15:25:05.614210+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:25:05.614210+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"180'4880","ondisk_log_start":"180'4880","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7046,"log_dups_size":0,"ondisk_log_size":7046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245235712,"num_objects":3742,"num_object_clones":0,"num_object_copies":11226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":3717,"num_objects_unfound":0,"num_objects_dirty":3742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11926,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":1591}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":409,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875468+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:24:58.875468+0000","last_peered":"2024-08-29T15:24:58.875468+0000","last_clean":"2024-08-29T15:24:58.875468+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:24:58.875468+0000","last_undegraded":"2024-08-29T15:24:58.875468+0000","last_fullsized":"2024-08-29T15:24:58.875468+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":695,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:58.884731+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:24:58.884731+0000","last_peered":"2024-08-29T15:24:58.884731+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:24:58.884731+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:24:58.884731+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":402,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893284+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:25:07.893284+0000","last_peered":"2024-08-29T15:25:07.893284+0000","last_clean":"2024-08-29T15:25:07.893284+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:25:07.893284+0000","last_undegraded":"2024-08-29T15:25:07.893284+0000","last_fullsized":"2024-08-29T15:25:07.893284+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"268'11827","reported_seq":18980,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:09.388035+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:25:09.388035+0000","last_peered":"2024-08-29T15:25:09.388035+0000","last_clean":"2024-08-29T15:25:09.388035+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:25:09.388035+0000","last_undegraded":"2024-08-29T15:25:09.388035+0000","last_fullsized":"2024-08-29T15:25:09.388035+0000","mapping_epoch":234,"log_start":"179'3899","ondisk_log_start":"179'3899","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7928,"log_dups_size":0,"ondisk_log_size":7928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235995136,"num_objects":3601,"num_object_clones":0,"num_object_copies":10803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11827,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":394,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.874364+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:24:58.874364+0000","last_peered":"2024-08-29T15:24:58.874364+0000","last_clean":"2024-08-29T15:24:58.874364+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:24:58.874364+0000","last_undegraded":"2024-08-29T15:24:58.874364+0000","last_fullsized":"2024-08-29T15:24:58.874364+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1016,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:24:58.874585+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:24:58.874585+0000","last_peered":"2024-08-29T15:24:58.874585+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:24:58.874585+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":392,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872296+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:24:58.872296+0000","last_peered":"2024-08-29T15:24:58.872296+0000","last_clean":"2024-08-29T15:24:58.872296+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:24:58.872296+0000","last_undegraded":"2024-08-29T15:24:58.872296+0000","last_fullsized":"2024-08-29T15:24:58.872296+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"268'11884","reported_seq":17975,"reported_epoch":268,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:08.201424+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:25:08.201424+0000","last_peered":"2024-08-29T15:25:08.201424+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:25:08.201424+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"179'4312","ondisk_log_start":"179'4312","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244187136,"num_objects":3726,"num_object_clones":0,"num_object_copies":11178,"num_objects_missing_on_primary":4760,"num_objects_missing":4760,"num_objects_degraded":2313,"num_objects_misplaced":3711,"num_objects_unfound":0,"num_objects_dirty":3726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11884,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2961,"num_bytes_recovered":193921024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":1340},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":392,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893328+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:25:07.893328+0000","last_peered":"2024-08-29T15:25:07.893328+0000","last_clean":"2024-08-29T15:25:07.893328+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:25:07.893328+0000","last_undegraded":"2024-08-29T15:25:07.893328+0000","last_fullsized":"2024-08-29T15:25:07.893328+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":119,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.884770+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:24:58.884770+0000","last_peered":"2024-08-29T15:24:58.884770+0000","last_clean":"2024-08-29T15:24:58.884770+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:24:58.884770+0000","last_undegraded":"2024-08-29T15:24:58.884770+0000","last_fullsized":"2024-08-29T15:24:58.884770+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872892+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:24:58.872892+0000","last_peered":"2024-08-29T15:24:58.872892+0000","last_clean":"2024-08-29T15:24:58.872892+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:24:58.872892+0000","last_undegraded":"2024-08-29T15:24:58.872892+0000","last_fullsized":"2024-08-29T15:24:58.872892+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873398+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:24:58.873398+0000","last_peered":"2024-08-29T15:24:58.873398+0000","last_clean":"2024-08-29T15:24:58.873398+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:24:58.873398+0000","last_undegraded":"2024-08-29T15:24:58.873398+0000","last_fullsized":"2024-08-29T15:24:58.873398+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":45,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:58.873075+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:24:58.873075+0000","last_peered":"2024-08-29T15:24:58.873075+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:24:58.873075+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:24:58.873075+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":398,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875337+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:24:58.875337+0000","last_peered":"2024-08-29T15:24:58.875337+0000","last_clean":"2024-08-29T15:24:58.875337+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:24:58.875337+0000","last_undegraded":"2024-08-29T15:24:58.875337+0000","last_fullsized":"2024-08-29T15:24:58.875337+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":148,"reported_epoch":267,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:58.876142+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:24:58.876142+0000","last_peered":"2024-08-29T15:24:58.876142+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:24:58.876142+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:24:58.876142+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":135,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:51.182306+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:24:51.182306+0000","last_peered":"2024-08-29T15:24:51.182306+0000","last_clean":"2024-08-29T15:24:51.182306+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:24:51.182306+0000","last_undegraded":"2024-08-29T15:24:51.182306+0000","last_fullsized":"2024-08-29T15:24:51.182306+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":357,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872348+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:24:58.872348+0000","last_peered":"2024-08-29T15:24:58.872348+0000","last_clean":"2024-08-29T15:24:58.872348+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:24:58.872348+0000","last_undegraded":"2024-08-29T15:24:58.872348+0000","last_fullsized":"2024-08-29T15:24:58.872348+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":413,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.894551+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:25:07.894551+0000","last_peered":"2024-08-29T15:25:07.894551+0000","last_clean":"2024-08-29T15:25:07.894551+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:25:07.894551+0000","last_undegraded":"2024-08-29T15:25:07.894551+0000","last_fullsized":"2024-08-29T15:25:07.894551+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":409,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872152+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:24:58.872152+0000","last_peered":"2024-08-29T15:24:58.872152+0000","last_clean":"2024-08-29T15:24:58.872152+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:24:58.872152+0000","last_undegraded":"2024-08-29T15:24:58.872152+0000","last_fullsized":"2024-08-29T15:24:58.872152+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.884838+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:24:58.884838+0000","last_peered":"2024-08-29T15:24:58.884838+0000","last_clean":"2024-08-29T15:24:58.884838+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:24:58.884838+0000","last_undegraded":"2024-08-29T15:24:58.884838+0000","last_fullsized":"2024-08-29T15:24:58.884838+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":268,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.894299+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:25:07.894299+0000","last_peered":"2024-08-29T15:25:07.894299+0000","last_clean":"2024-08-29T15:25:07.894299+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:25:07.894299+0000","last_undegraded":"2024-08-29T15:25:07.894299+0000","last_fullsized":"2024-08-29T15:25:07.894299+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":388,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893571+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:25:07.893571+0000","last_peered":"2024-08-29T15:25:07.893571+0000","last_clean":"2024-08-29T15:25:07.893571+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:25:07.893571+0000","last_undegraded":"2024-08-29T15:25:07.893571+0000","last_fullsized":"2024-08-29T15:25:07.893571+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":390,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.880179+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:24:58.880179+0000","last_peered":"2024-08-29T15:24:58.880179+0000","last_clean":"2024-08-29T15:24:58.880179+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:24:58.880179+0000","last_undegraded":"2024-08-29T15:24:58.880179+0000","last_fullsized":"2024-08-29T15:24:58.880179+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":394,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875983+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:24:58.875983+0000","last_peered":"2024-08-29T15:24:58.875983+0000","last_clean":"2024-08-29T15:24:58.875983+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:24:58.875983+0000","last_undegraded":"2024-08-29T15:24:58.875983+0000","last_fullsized":"2024-08-29T15:24:58.875983+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872872+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:24:58.872872+0000","last_peered":"2024-08-29T15:24:58.872872+0000","last_clean":"2024-08-29T15:24:58.872872+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:24:58.872872+0000","last_undegraded":"2024-08-29T15:24:58.872872+0000","last_fullsized":"2024-08-29T15:24:58.872872+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":414,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.875668+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:24:58.875668+0000","last_peered":"2024-08-29T15:24:58.875668+0000","last_clean":"2024-08-29T15:24:58.875668+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:24:58.875668+0000","last_undegraded":"2024-08-29T15:24:58.875668+0000","last_fullsized":"2024-08-29T15:24:58.875668+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":380,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893181+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:25:07.893181+0000","last_peered":"2024-08-29T15:25:07.893181+0000","last_clean":"2024-08-29T15:25:07.893181+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:25:07.893181+0000","last_undegraded":"2024-08-29T15:25:07.893181+0000","last_fullsized":"2024-08-29T15:25:07.893181+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873090+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:24:58.873090+0000","last_peered":"2024-08-29T15:24:58.873090+0000","last_clean":"2024-08-29T15:24:58.873090+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:24:58.873090+0000","last_undegraded":"2024-08-29T15:24:58.873090+0000","last_fullsized":"2024-08-29T15:24:58.873090+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":158,"reported_epoch":266,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:24:48.816570+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:24:48.816570+0000","last_peered":"2024-08-29T15:24:48.816570+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:24:48.816570+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:24:48.816570+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872956+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:24:58.872956+0000","last_peered":"2024-08-29T15:24:58.872956+0000","last_clean":"2024-08-29T15:24:58.872956+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:24:58.872956+0000","last_undegraded":"2024-08-29T15:24:58.872956+0000","last_fullsized":"2024-08-29T15:24:58.872956+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":164,"reported_epoch":266,"state":"active+clean","last_fresh":"2024-08-29T15:24:48.816364+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:24:48.816364+0000","last_peered":"2024-08-29T15:24:48.816364+0000","last_clean":"2024-08-29T15:24:48.816364+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:24:48.816364+0000","last_undegraded":"2024-08-29T15:24:48.816364+0000","last_fullsized":"2024-08-29T15:24:48.816364+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":425,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872869+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:24:58.872869+0000","last_peered":"2024-08-29T15:24:58.872869+0000","last_clean":"2024-08-29T15:24:58.872869+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:24:58.872869+0000","last_undegraded":"2024-08-29T15:24:58.872869+0000","last_fullsized":"2024-08-29T15:24:58.872869+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":427,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.873126+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:24:58.873126+0000","last_peered":"2024-08-29T15:24:58.873126+0000","last_clean":"2024-08-29T15:24:58.873126+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:24:58.873126+0000","last_undegraded":"2024-08-29T15:24:58.873126+0000","last_fullsized":"2024-08-29T15:24:58.873126+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":47,"reported_epoch":267,"state":"active+clean","last_fresh":"2024-08-29T15:24:58.872547+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:24:58.872547+0000","last_peered":"2024-08-29T15:24:58.872547+0000","last_clean":"2024-08-29T15:24:58.872547+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:24:58.872547+0000","last_undegraded":"2024-08-29T15:24:58.872547+0000","last_fullsized":"2024-08-29T15:24:58.872547+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":436,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.894444+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:25:07.894444+0000","last_peered":"2024-08-29T15:25:07.894444+0000","last_clean":"2024-08-29T15:25:07.894444+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:25:07.894444+0000","last_undegraded":"2024-08-29T15:25:07.894444+0000","last_fullsized":"2024-08-29T15:25:07.894444+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3964928024,"num_objects":60501,"num_object_clones":0,"num_object_copies":181503,"num_objects_missing_on_primary":4760,"num_objects_missing":4760,"num_objects_degraded":29407,"num_objects_misplaced":11271,"num_objects_unfound":0,"num_objects_dirty":60501,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190349,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17254,"num_bytes_recovered":1010761728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11938770944,"data_stored":11938758728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124326,"ondisk_log_size":124326,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234366,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2250844,"kb_used_data":1926864,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":91494308,"statfs":{"total":95995035648,"available":93690171392,"internally_reserved":0,"allocated":1973108736,"data_stored":1971562491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":331741643},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495997,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252612,"kb_used_data":73020,"kb_used_omap":1,"kb_used_meta":179582,"kb_avail":93492540,"statfs":{"total":95995035648,"available":95736360960,"internally_reserved":0,"allocated":74772480,"data_stored":73237683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":183892423},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426583,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1286968,"kb_used_data":1217904,"kb_used_omap":8,"kb_used_meta":69047,"kb_avail":92458184,"statfs":{"total":95995035648,"available":94677180416,"internally_reserved":0,"allocated":1247133696,"data_stored":1245606365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":70705034},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019276,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527352,"kb_used_data":1207472,"kb_used_omap":1,"kb_used_meta":319870,"kb_avail":92217800,"statfs":{"total":95995035648,"available":94431027200,"internally_reserved":0,"allocated":1236451328,"data_stored":1234932132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":327547334},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435460,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2729204,"kb_used_data":2426860,"kb_used_omap":4,"kb_used_meta":302331,"kb_avail":91015948,"statfs":{"total":95995035648,"available":93200330752,"internally_reserved":0,"allocated":2485104640,"data_stored":2483581249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":309587209},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903312,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2366392,"kb_used_data":2234672,"kb_used_omap":10,"kb_used_meta":131701,"kb_avail":91378760,"statfs":{"total":95995035648,"available":93571850240,"internally_reserved":0,"allocated":2288304128,"data_stored":2286780673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":134862352},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762513,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1914228,"kb_used_data":1643756,"kb_used_omap":7,"kb_used_meta":270456,"kb_avail":91830924,"statfs":{"total":95995035648,"available":94034866176,"internally_reserved":0,"allocated":1683206144,"data_stored":1681681513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":276947298},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926040,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1313680,"kb_used_data":980740,"kb_used_omap":7,"kb_used_meta":332920,"kb_avail":92431472,"statfs":{"total":95995035648,"available":94649827328,"internally_reserved":0,"allocated":1004277760,"data_stored":1002738260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":340910223},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":995102720,"data_stored":995098648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1675431936,"data_stored":1675427864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2282356736,"data_stored":2282356736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2479816704,"data_stored":2479816704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1228537856,"data_stored":1228537856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1242435584,"data_stored":1242431512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":70516736,"data_stored":70516736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1964572672,"data_stored":1964572672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:25:11.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:25:11.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:11.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:11.986 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:12.007 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:12.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:12.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:11 smithi027 ceph-mon[98970]: pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29436/181587 objects degraded (16.210%); 11271/181587 objects misplaced (6.207%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:12.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:11 smithi027 ceph-mon[98970]: from='client.45809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:12.093 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:11 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29436/181587 objects degraded (16.210%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:12.093 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:11 smithi027 ceph-mon[93921]: pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29436/181587 objects degraded (16.210%); 11271/181587 objects misplaced (6.207%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:12.093 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:11 smithi027 ceph-mon[93921]: from='client.45809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:12.094 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:11 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29436/181587 objects degraded (16.210%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:12.105 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:25:12.106 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:25:12.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:12.126 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:25:12.127 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:25:12.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:12.145 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:25:12.145 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:25:12.146 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:12.165 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:25:12.165 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:25:12.183 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:25:12.183 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:25:12.189 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:12.200 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:25:12.201 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:25:12.222 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:25:12.223 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:25:12.239 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:25:12.239 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:25:12.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:12.260 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:12.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:11 smithi171 ceph-mon[90031]: pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29436/181587 objects degraded (16.210%); 11271/181587 objects misplaced (6.207%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:11 smithi171 ceph-mon[90031]: from='client.45809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:11 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29436/181587 objects degraded (16.210%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:12.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:12.386 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:12.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:12.488 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:12.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:12.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:12.638 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:12.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:12.767 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:12.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:12.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:12.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:12.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:13.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:13.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3809843455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:13.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3809843455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:13.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:13.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:13.250 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3809843455' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:13.276 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:13.377 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:25:13.377 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:25:13.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:13.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:13.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:13.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:13.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:13.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:13.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:13.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:13.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:13.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:13.880 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:13.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:13.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:13.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:14.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:13 smithi171 ceph-mon[90031]: pgmap v893: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 29380/179937 objects degraded (16.328%); 11166/179937 objects misplaced (6.206%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:25:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:14.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:14.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:25:14.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:14.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.160 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:14.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:14.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:14.274 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:14.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:13 smithi027 ceph-mon[93921]: pgmap v893: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 29380/179937 objects degraded (16.328%); 11166/179937 objects misplaced (6.206%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:25:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:13 smithi027 ceph-mon[98970]: pgmap v893: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 29380/179937 objects degraded (16.328%); 11166/179937 objects misplaced (6.206%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:25:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:14.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:14.408 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:14.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:14.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:14.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:14.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:14.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:14.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:14.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:14.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:14.889 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:14.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:14.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:15.012 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:15.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:15.135 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:15.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:15.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:15.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:15.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:15.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:15.413 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:15.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:15.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:25:15.515 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:15.537 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:15.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:15.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:15.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:15.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:15.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:15.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:15.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:15.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:15.910 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:25:15.912 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:15.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:15.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:16.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:16.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:15 smithi027 ceph-mon[98970]: pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 29380/179388 objects degraded (16.378%); 11113/179388 objects misplaced (6.195%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:25:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:15 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:25:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:15 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1712507344' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:16.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:15 smithi027 ceph-mon[93921]: pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 29380/179388 objects degraded (16.378%); 11113/179388 objects misplaced (6.195%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:25:16.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:15 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:25:16.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:15 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1712507344' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:16.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:16.161 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:15 smithi171 ceph-mon[90031]: pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 29380/179388 objects degraded (16.378%); 11113/179388 objects misplaced (6.195%); 1.4 MiB/s, 21 objects/s recovering 2024-08-29T15:25:16.161 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:15 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:25:16.161 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:15 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1712507344' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:16.168 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:16.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:16.270 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:25:16.270 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:25:16.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:16.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:16.415 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:16.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:16.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:16.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:16.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:16.664 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:16.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:16.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:16.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:16.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:16.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:16.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:16.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:16.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:16.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:16.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1712507344' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:25:16.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[93921]: osdmap e269: 8 total, 8 up, 8 in 2024-08-29T15:25:16.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29380/179388 objects degraded (16.378%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:16.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:16.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1712507344' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:25:16.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[98970]: osdmap e269: 8 total, 8 up, 8 in 2024-08-29T15:25:16.947 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:16 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29380/179388 objects degraded (16.378%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:17.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:17.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:17.078 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:25:17.079 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 29323/177393 objects degraded (16.530%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1298,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":269,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":59131,"data_bytes":3878098941,"bytes_used":13745156096,"bytes_avail":754215129088,"bytes_total":767960285184,"degraded_objects":29323,"degraded_total":177393,"degraded_ratio":0.16529964542005604,"misplaced_objects":11019,"misplaced_total":177393,"misplaced_ratio":0.062116318005783766,"recovering_objects_per_sec":25,"recovering_bytes_per_sec":1664524,"recovering_keys_per_sec":0,"num_objects_recovered":222,"num_bytes_recovered":14548992,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":188},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:25:17.135 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:16 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:17.136 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:16 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1712507344' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:25:17.136 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:16 smithi171 ceph-mon[90031]: osdmap e269: 8 total, 8 up, 8 in 2024-08-29T15:25:17.136 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:16 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29380/179388 objects degraded (16.378%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:17.145 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:17.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:25:17.171 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:17.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:17.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:17.299 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:17.401 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:17.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:17.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:17.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:17.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:17.676 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:17.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:17.757 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:17.779 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:17.800 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:17.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:17.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:17.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:17.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:17.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:17.958 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:17 smithi171 ceph-mon[90031]: pgmap v896: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 29323/177393 objects degraded (16.530%); 11019/177393 objects misplaced (6.212%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:25:17.958 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3363237654' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:25:18.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:18.054 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:18.156 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:18.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:18.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:18.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:17 smithi027 ceph-mon[93921]: pgmap v896: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 29323/177393 objects degraded (16.530%); 11019/177393 objects misplaced (6.212%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:25:18.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3363237654' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:25:18.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:17 smithi027 ceph-mon[98970]: pgmap v896: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 29323/177393 objects degraded (16.530%); 11019/177393 objects misplaced (6.212%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:25:18.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3363237654' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:25:18.233 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:18.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:18.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:18.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:18.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:18.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:18.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:18.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:18.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:18.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:18.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:18.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:18.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:18.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:18.936 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:18.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:19.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:19.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:19.111 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:25:19.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:25:19.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:19.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:19.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:19.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:19.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:25:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:25:19] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:25:19.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:19.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:19.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:25:19.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:19.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:19.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:19.687 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:19.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:19.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:19.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:19.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:19.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:19.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:19.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:20.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:20.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:20.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:20.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:20.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:19 smithi171 ceph-mon[90031]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 29323/177234 objects degraded (16.545%); 10963/177234 objects misplaced (6.186%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:20.283 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:19 smithi027 ceph-mon[93921]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 29323/177234 objects degraded (16.545%); 10963/177234 objects misplaced (6.186%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:20.284 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:19 smithi027 ceph-mon[98970]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 29323/177234 objects degraded (16.545%); 10963/177234 objects misplaced (6.186%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:20.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:20.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:20.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:20.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:20.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:20.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:20.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:20.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:20.676 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:25:20.676 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:25:20.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:20.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:20.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:20.714 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:20.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:20.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:20.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:20.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:20.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:20.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:21.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:21.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:21.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:21.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:21.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:21.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:21.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:21.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:21.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:21.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:21.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:21.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:25:21.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:21.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:21.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:21.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:21.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:21.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:21.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:21.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:21.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:21.941 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:21.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:21.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:21.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:22.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:22.080 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:22.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:22.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:22.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:22.201 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:21 smithi027 ceph-mon[98970]: pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29323/176586 objects degraded (16.606%); 10963/176586 objects misplaced (6.208%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:22.201 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:21 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29323/176586 objects degraded (16.606%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:22.202 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:21 smithi027 ceph-mon[93921]: pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29323/176586 objects degraded (16.606%); 10963/176586 objects misplaced (6.208%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:22.202 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:21 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29323/176586 objects degraded (16.606%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:22.206 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:21 smithi171 ceph-mon[90031]: pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29323/176586 objects degraded (16.606%); 10963/176586 objects misplaced (6.208%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:22.206 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:21 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29323/176586 objects degraded (16.606%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:22.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:22.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:22.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:22.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:22.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:22.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:22.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:22.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:22.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:22.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:22.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:22.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:22.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:22.894 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:25:22.894 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:25:22.913 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:25:22.914 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:25:22.934 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:25:22.934 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:25:22.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:22.951 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:25:22.952 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:25:22.974 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:25:22.975 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:25:22.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:22.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:22.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:22.993 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:25:22.994 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:25:23.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:25:23.011 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:25:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/880416420' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:23.020 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/880416420' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:23.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/880416420' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:23.028 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:25:23.028 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:25:23.077 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:23.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:23.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:23.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:23.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:23.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:23.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:23.373 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:25:23.373 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:25:23.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:23.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:23.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:23.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:23.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:23.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:23.730 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:23.832 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:23.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:23.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:23.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:23.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:23.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:23.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:23 smithi027 ceph-mon[98970]: pgmap v899: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 29263/174720 objects degraded (16.749%); 10840/174720 objects misplaced (6.204%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:23 smithi027 ceph-mon[93921]: pgmap v899: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 29263/174720 objects degraded (16.749%); 10840/174720 objects misplaced (6.204%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:24.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:24.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:24.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:24.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:24.241 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:23 smithi171 ceph-mon[90031]: pgmap v899: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 29263/174720 objects degraded (16.749%); 10840/174720 objects misplaced (6.204%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:24.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:24.357 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:24.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:24.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:24.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:24.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:24.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:24.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:24.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:24.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:24.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:24.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:24.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:24.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:24 smithi027 ceph-mon[93921]: from='client.45839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:25.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:24 smithi027 ceph-mon[98970]: from='client.45839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:25.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:25.109 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:25.208 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:24 smithi171 ceph-mon[90031]: from='client.45839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:25.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:25.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:25.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:25.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:25.359 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:25.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:25.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:25.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:25.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:25.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:25.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:25.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:25.796 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:25:25.796 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:25:25.797 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:25:25.797 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:25:25.797 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:25:25.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:25.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:25.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:25.987 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:25.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:25.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:26.077 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":899,"stamp":"2024-08-29T15:25:22.630391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3819706365,"num_objects":58240,"num_object_clones":0,"num_object_copies":174720,"num_objects_missing_on_primary":4659,"num_objects_missing":4659,"num_objects_degraded":29263,"num_objects_misplaced":10840,"num_objects_unfound":0,"num_objects_dirty":58240,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":193761,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17645,"num_bytes_recovered":1035209933,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125397,"ondisk_log_size":125397,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13259080,"kb_used_data":11308552,"kb_used_omap":43,"kb_used_meta":1950420,"kb_avail":736702136,"statfs":{"total":767960285184,"available":754382987264,"internally_reserved":0,"allocated":11579957248,"data_stored":11567619278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1997230572},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-113967104,"num_objects":-1739,"num_object_clones":0,"num_object_copies":-5217,"num_objects_missing_on_primary":-109,"num_objects_missing":-109,"num_objects_degraded":-117,"num_objects_misplaced":-326,"num_objects_unfound":0,"num_objects_dirty":-1739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1739,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":226,"num_bytes_recovered":14811136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008659"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":424,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.915537+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:25:15.915537+0000","last_peered":"2024-08-29T15:25:15.915537+0000","last_clean":"2024-08-29T15:25:15.915537+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:25:15.915537+0000","last_undegraded":"2024-08-29T15:25:15.915537+0000","last_fullsized":"2024-08-29T15:25:15.915537+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":47,"reported_epoch":268,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:07.895542+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:25:07.895542+0000","last_peered":"2024-08-29T15:25:07.895542+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:25:07.895542+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:25:07.895542+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":137,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893654+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:25:07.893654+0000","last_peered":"2024-08-29T15:25:07.893654+0000","last_clean":"2024-08-29T15:25:07.893654+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:25:07.893654+0000","last_undegraded":"2024-08-29T15:25:07.893654+0000","last_fullsized":"2024-08-29T15:25:07.893654+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":389,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918974+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:25:15.918974+0000","last_peered":"2024-08-29T15:25:15.918974+0000","last_clean":"2024-08-29T15:25:15.918974+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:25:15.918974+0000","last_undegraded":"2024-08-29T15:25:15.918974+0000","last_fullsized":"2024-08-29T15:25:15.918974+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":389,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917885+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:25:15.917885+0000","last_peered":"2024-08-29T15:25:15.917885+0000","last_clean":"2024-08-29T15:25:15.917885+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:25:15.917885+0000","last_undegraded":"2024-08-29T15:25:15.917885+0000","last_fullsized":"2024-08-29T15:25:15.917885+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895244+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:25:07.895244+0000","last_peered":"2024-08-29T15:25:07.895244+0000","last_clean":"2024-08-29T15:25:07.895244+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:25:07.895244+0000","last_undegraded":"2024-08-29T15:25:07.895244+0000","last_fullsized":"2024-08-29T15:25:07.895244+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":200,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918283+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:25:15.918283+0000","last_peered":"2024-08-29T15:25:15.918283+0000","last_clean":"2024-08-29T15:25:15.918283+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:25:15.918283+0000","last_undegraded":"2024-08-29T15:25:15.918283+0000","last_fullsized":"2024-08-29T15:25:15.918283+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":138,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.890868+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:25:07.890868+0000","last_peered":"2024-08-29T15:25:07.890868+0000","last_clean":"2024-08-29T15:25:07.890868+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:25:07.890868+0000","last_undegraded":"2024-08-29T15:25:07.890868+0000","last_fullsized":"2024-08-29T15:25:07.890868+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"269'12210","reported_seq":20258,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:21.161207+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:25:21.161207+0000","last_peered":"2024-08-29T15:25:21.161207+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:25:21.161207+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"176'2200","ondisk_log_start":"176'2200","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10010,"log_dups_size":0,"ondisk_log_size":10010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242614272,"num_objects":3702,"num_object_clones":0,"num_object_copies":11106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12210,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":777}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":46,"reported_epoch":268,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:07.895947+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:25:07.895947+0000","last_peered":"2024-08-29T15:25:07.895947+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:25:07.895947+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:25:07.895947+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":155,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893426+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:25:07.893426+0000","last_peered":"2024-08-29T15:25:07.893426+0000","last_clean":"2024-08-29T15:25:07.893426+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:25:07.893426+0000","last_undegraded":"2024-08-29T15:25:07.893426+0000","last_fullsized":"2024-08-29T15:25:07.893426+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"269'11911","reported_seq":12128,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:21.342483+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:25:21.342483+0000","last_peered":"2024-08-29T15:25:21.342483+0000","last_clean":"2024-08-29T15:25:21.342483+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:25:21.342483+0000","last_undegraded":"2024-08-29T15:25:21.342483+0000","last_fullsized":"2024-08-29T15:25:21.342483+0000","mapping_epoch":168,"log_start":"180'5228","ondisk_log_start":"180'5228","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238485504,"num_objects":3639,"num_object_clones":0,"num_object_copies":10917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11911,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":46,"reported_epoch":268,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:07.898597+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:25:07.898597+0000","last_peered":"2024-08-29T15:25:07.898597+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:25:07.898597+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:25:07.898597+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895140+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:25:07.895140+0000","last_peered":"2024-08-29T15:25:07.895140+0000","last_clean":"2024-08-29T15:25:07.895140+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:25:07.895140+0000","last_undegraded":"2024-08-29T15:25:07.895140+0000","last_fullsized":"2024-08-29T15:25:07.895140+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"269'12112","reported_seq":12299,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:21.057414+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:25:21.057414+0000","last_peered":"2024-08-29T15:25:21.057414+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:25:21.057414+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"181'5374","ondisk_log_start":"181'5374","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6738,"log_dups_size":0,"ondisk_log_size":6738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235929600,"num_objects":3600,"num_object_clones":0,"num_object_copies":10800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12112,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1123}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":205,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917560+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:25:15.917560+0000","last_peered":"2024-08-29T15:25:15.917560+0000","last_clean":"2024-08-29T15:25:15.917560+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:25:15.917560+0000","last_undegraded":"2024-08-29T15:25:15.917560+0000","last_fullsized":"2024-08-29T15:25:15.917560+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":93,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.891425+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:25:07.891425+0000","last_peered":"2024-08-29T15:25:07.891425+0000","last_clean":"2024-08-29T15:25:07.891425+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:25:07.891425+0000","last_undegraded":"2024-08-29T15:25:07.891425+0000","last_fullsized":"2024-08-29T15:25:07.891425+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"269'12107","reported_seq":17544,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:18.990771+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:25:18.990771+0000","last_peered":"2024-08-29T15:25:18.990771+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:25:18.990771+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"179'3962","ondisk_log_start":"179'3962","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240582656,"num_objects":3671,"num_object_clones":0,"num_object_copies":11013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12107,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3671}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":155,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893810+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:25:07.893810+0000","last_peered":"2024-08-29T15:25:07.893810+0000","last_clean":"2024-08-29T15:25:07.893810+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:25:07.893810+0000","last_undegraded":"2024-08-29T15:25:07.893810+0000","last_fullsized":"2024-08-29T15:25:07.893810+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":404,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917740+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:25:15.917740+0000","last_peered":"2024-08-29T15:25:15.917740+0000","last_clean":"2024-08-29T15:25:15.917740+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:25:15.917740+0000","last_undegraded":"2024-08-29T15:25:15.917740+0000","last_fullsized":"2024-08-29T15:25:15.917740+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"269'12099","reported_seq":12255,"reported_epoch":269,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:19.475848+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:25:19.475848+0000","last_peered":"2024-08-29T15:25:19.475848+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:25:19.475848+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:25:19.475848+0000","mapping_epoch":234,"log_start":"179'4133","ondisk_log_start":"179'4133","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7966,"log_dups_size":0,"ondisk_log_size":7966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243466240,"num_objects":3715,"num_object_clones":0,"num_object_copies":11145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12099,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":1238},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":398,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.914572+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:25:15.914572+0000","last_peered":"2024-08-29T15:25:15.914572+0000","last_clean":"2024-08-29T15:25:15.914572+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:25:15.914572+0000","last_undegraded":"2024-08-29T15:25:15.914572+0000","last_fullsized":"2024-08-29T15:25:15.914572+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895595+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:25:07.895595+0000","last_peered":"2024-08-29T15:25:07.895595+0000","last_clean":"2024-08-29T15:25:07.895595+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:25:07.895595+0000","last_undegraded":"2024-08-29T15:25:07.895595+0000","last_fullsized":"2024-08-29T15:25:07.895595+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"269'12123","reported_seq":12356,"reported_epoch":269,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:20.975480+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:25:20.975480+0000","last_peered":"2024-08-29T15:25:20.975480+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:25:20.975480+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:25:20.975480+0000","mapping_epoch":234,"log_start":"180'5368","ondisk_log_start":"180'5368","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6755,"log_dups_size":0,"ondisk_log_size":6755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243728384,"num_objects":3719,"num_object_clones":0,"num_object_copies":11157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12123,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":1254},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.891294+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:25:07.891294+0000","last_peered":"2024-08-29T15:25:07.891294+0000","last_clean":"2024-08-29T15:25:07.891294+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:25:07.891294+0000","last_undegraded":"2024-08-29T15:25:07.891294+0000","last_fullsized":"2024-08-29T15:25:07.891294+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895837+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:25:07.895837+0000","last_peered":"2024-08-29T15:25:07.895837+0000","last_clean":"2024-08-29T15:25:07.895837+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:25:07.895837+0000","last_undegraded":"2024-08-29T15:25:07.895837+0000","last_fullsized":"2024-08-29T15:25:07.895837+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"269'11973","reported_seq":12391,"reported_epoch":269,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:25:21.320274+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:25:21.320274+0000","last_peered":"2024-08-29T15:25:21.320274+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:25:21.320274+0000","last_undegraded":"2024-08-29T15:25:21.320274+0000","last_fullsized":"2024-08-29T15:25:21.320274+0000","mapping_epoch":202,"log_start":"180'5267","ondisk_log_start":"180'5267","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241500160,"num_objects":3685,"num_object_clones":0,"num_object_copies":11055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3685,"num_objects_unfound":0,"num_objects_dirty":3685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11973,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":3685}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895155+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:25:07.895155+0000","last_peered":"2024-08-29T15:25:07.895155+0000","last_clean":"2024-08-29T15:25:07.895155+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:25:07.895155+0000","last_undegraded":"2024-08-29T15:25:07.895155+0000","last_fullsized":"2024-08-29T15:25:07.895155+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":254,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.898695+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:25:07.898695+0000","last_peered":"2024-08-29T15:25:07.898695+0000","last_clean":"2024-08-29T15:25:07.898695+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:25:07.898695+0000","last_undegraded":"2024-08-29T15:25:07.898695+0000","last_fullsized":"2024-08-29T15:25:07.898695+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"269'12155","reported_seq":12305,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:20.928467+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:25:20.928467+0000","last_peered":"2024-08-29T15:25:20.928467+0000","last_clean":"2024-08-29T15:25:20.928467+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:25:20.928467+0000","last_undegraded":"2024-08-29T15:25:20.928467+0000","last_fullsized":"2024-08-29T15:25:20.928467+0000","mapping_epoch":218,"log_start":"176'2078","ondisk_log_start":"176'2078","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10077,"log_dups_size":0,"ondisk_log_size":10077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236191768,"num_objects":3605,"num_object_clones":0,"num_object_copies":10815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3605,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12155,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":401,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.915865+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:25:15.915865+0000","last_peered":"2024-08-29T15:25:15.915865+0000","last_clean":"2024-08-29T15:25:15.915865+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:25:15.915865+0000","last_undegraded":"2024-08-29T15:25:15.915865+0000","last_fullsized":"2024-08-29T15:25:15.915865+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":393,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918363+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:25:15.918363+0000","last_peered":"2024-08-29T15:25:15.918363+0000","last_clean":"2024-08-29T15:25:15.918363+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:25:15.918363+0000","last_undegraded":"2024-08-29T15:25:15.918363+0000","last_fullsized":"2024-08-29T15:25:15.918363+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"269'11616","reported_seq":11814,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:21.084300+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:25:21.084300+0000","last_peered":"2024-08-29T15:25:21.084300+0000","last_clean":"2024-08-29T15:25:21.084300+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:25:21.084300+0000","last_undegraded":"2024-08-29T15:25:21.084300+0000","last_fullsized":"2024-08-29T15:25:21.084300+0000","mapping_epoch":201,"log_start":"176'1600","ondisk_log_start":"176'1600","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10016,"log_dups_size":0,"ondisk_log_size":10016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235405312,"num_objects":3592,"num_object_clones":0,"num_object_copies":10776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11616,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":492,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918680+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:25:15.918680+0000","last_peered":"2024-08-29T15:25:15.918680+0000","last_clean":"2024-08-29T15:25:15.918680+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:25:15.918680+0000","last_undegraded":"2024-08-29T15:25:15.918680+0000","last_fullsized":"2024-08-29T15:25:15.918680+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":739,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893549+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:25:07.893549+0000","last_peered":"2024-08-29T15:25:07.893549+0000","last_clean":"2024-08-29T15:25:07.893549+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:25:07.893549+0000","last_undegraded":"2024-08-29T15:25:07.893549+0000","last_fullsized":"2024-08-29T15:25:07.893549+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":247,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893589+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:25:07.893589+0000","last_peered":"2024-08-29T15:25:07.893589+0000","last_clean":"2024-08-29T15:25:07.893589+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:25:07.893589+0000","last_undegraded":"2024-08-29T15:25:07.893589+0000","last_fullsized":"2024-08-29T15:25:07.893589+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.891292+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:25:07.891292+0000","last_peered":"2024-08-29T15:25:07.891292+0000","last_clean":"2024-08-29T15:25:07.891292+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:25:07.891292+0000","last_undegraded":"2024-08-29T15:25:07.891292+0000","last_fullsized":"2024-08-29T15:25:07.891292+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"269'12066","reported_seq":12481,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:20.703516+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:25:20.703516+0000","last_peered":"2024-08-29T15:25:20.703516+0000","last_clean":"2024-08-29T15:25:20.703516+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:25:20.703516+0000","last_undegraded":"2024-08-29T15:25:20.703516+0000","last_fullsized":"2024-08-29T15:25:20.703516+0000","mapping_epoch":201,"log_start":"180'5053","ondisk_log_start":"180'5053","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236978176,"num_objects":3616,"num_object_clones":0,"num_object_copies":10848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12066,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":866,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917288+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:25:15.917288+0000","last_peered":"2024-08-29T15:25:15.917288+0000","last_clean":"2024-08-29T15:25:15.917288+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:25:15.917288+0000","last_undegraded":"2024-08-29T15:25:15.917288+0000","last_fullsized":"2024-08-29T15:25:15.917288+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":404,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.919581+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:25:15.919581+0000","last_peered":"2024-08-29T15:25:15.919581+0000","last_clean":"2024-08-29T15:25:15.919581+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:25:15.919581+0000","last_undegraded":"2024-08-29T15:25:15.919581+0000","last_fullsized":"2024-08-29T15:25:15.919581+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":412,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918093+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:25:15.918093+0000","last_peered":"2024-08-29T15:25:15.918093+0000","last_clean":"2024-08-29T15:25:15.918093+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:25:15.918093+0000","last_undegraded":"2024-08-29T15:25:15.918093+0000","last_fullsized":"2024-08-29T15:25:15.918093+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"269'12041","reported_seq":19558,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:20.995494+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:25:20.995494+0000","last_peered":"2024-08-29T15:25:20.995494+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:25:20.995494+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"180'5330","ondisk_log_start":"180'5330","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239140864,"num_objects":3649,"num_object_clones":0,"num_object_copies":10947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4126,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12041,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3665,"num_bytes_recovered":240189440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1211},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":675,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.915093+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:25:15.915093+0000","last_peered":"2024-08-29T15:25:15.915093+0000","last_clean":"2024-08-29T15:25:15.915093+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:25:15.915093+0000","last_undegraded":"2024-08-29T15:25:15.915093+0000","last_fullsized":"2024-08-29T15:25:15.915093+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":100,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.890701+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:25:07.890701+0000","last_peered":"2024-08-29T15:25:07.890701+0000","last_clean":"2024-08-29T15:25:07.890701+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:25:07.890701+0000","last_undegraded":"2024-08-29T15:25:07.890701+0000","last_fullsized":"2024-08-29T15:25:07.890701+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":439,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918151+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:25:15.918151+0000","last_peered":"2024-08-29T15:25:15.918151+0000","last_clean":"2024-08-29T15:25:15.918151+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:25:15.918151+0000","last_undegraded":"2024-08-29T15:25:15.918151+0000","last_fullsized":"2024-08-29T15:25:15.918151+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"269'12085","reported_seq":12312,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:21.177422+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:25:21.177422+0000","last_peered":"2024-08-29T15:25:21.177422+0000","last_clean":"2024-08-29T15:25:21.177422+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:25:21.177422+0000","last_undegraded":"2024-08-29T15:25:21.177422+0000","last_fullsized":"2024-08-29T15:25:21.177422+0000","mapping_epoch":234,"log_start":"181'5397","ondisk_log_start":"181'5397","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237699072,"num_objects":3627,"num_object_clones":0,"num_object_copies":10881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12085,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":207,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893505+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:25:07.893505+0000","last_peered":"2024-08-29T15:25:07.893505+0000","last_clean":"2024-08-29T15:25:07.893505+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:25:07.893505+0000","last_undegraded":"2024-08-29T15:25:07.893505+0000","last_fullsized":"2024-08-29T15:25:07.893505+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895403+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:25:07.895403+0000","last_peered":"2024-08-29T15:25:07.895403+0000","last_clean":"2024-08-29T15:25:07.895403+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:25:07.895403+0000","last_undegraded":"2024-08-29T15:25:07.895403+0000","last_fullsized":"2024-08-29T15:25:07.895403+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":114,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893516+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:25:07.893516+0000","last_peered":"2024-08-29T15:25:07.893516+0000","last_clean":"2024-08-29T15:25:07.893516+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:25:07.893516+0000","last_undegraded":"2024-08-29T15:25:07.893516+0000","last_fullsized":"2024-08-29T15:25:07.893516+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"269'12142","reported_seq":12368,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:19.004769+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:25:19.004769+0000","last_peered":"2024-08-29T15:25:19.004769+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:25:19.004769+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"179'3942","ondisk_log_start":"179'3942","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8200,"log_dups_size":0,"ondisk_log_size":8200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241434624,"num_objects":3684,"num_object_clones":0,"num_object_copies":11052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12142,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1261}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":139,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893741+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:25:07.893741+0000","last_peered":"2024-08-29T15:25:07.893741+0000","last_clean":"2024-08-29T15:25:07.893741+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:25:07.893741+0000","last_undegraded":"2024-08-29T15:25:07.893741+0000","last_fullsized":"2024-08-29T15:25:07.893741+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":831,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.921186+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:25:15.921186+0000","last_peered":"2024-08-29T15:25:15.921186+0000","last_clean":"2024-08-29T15:25:15.921186+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:25:15.921186+0000","last_undegraded":"2024-08-29T15:25:15.921186+0000","last_fullsized":"2024-08-29T15:25:15.921186+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":408,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917751+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:25:15.917751+0000","last_peered":"2024-08-29T15:25:15.917751+0000","last_clean":"2024-08-29T15:25:15.917751+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:25:15.917751+0000","last_undegraded":"2024-08-29T15:25:15.917751+0000","last_fullsized":"2024-08-29T15:25:15.917751+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"269'12090","reported_seq":12871,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:20.654608+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:25:20.654608+0000","last_peered":"2024-08-29T15:25:20.654608+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:25:20.654608+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"180'5080","ondisk_log_start":"180'5080","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234487808,"num_objects":3578,"num_object_clones":0,"num_object_copies":10734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":3553,"num_objects_unfound":0,"num_objects_dirty":3578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12090,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":1427}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":413,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.921086+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:25:15.921086+0000","last_peered":"2024-08-29T15:25:15.921086+0000","last_clean":"2024-08-29T15:25:15.921086+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:25:15.921086+0000","last_undegraded":"2024-08-29T15:25:15.921086+0000","last_fullsized":"2024-08-29T15:25:15.921086+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":696,"reported_epoch":268,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:07.902472+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:25:07.902472+0000","last_peered":"2024-08-29T15:25:07.902472+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:25:07.902472+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:25:07.902472+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":404,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917636+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:25:15.917636+0000","last_peered":"2024-08-29T15:25:15.917636+0000","last_clean":"2024-08-29T15:25:15.917636+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:25:15.917636+0000","last_undegraded":"2024-08-29T15:25:15.917636+0000","last_fullsized":"2024-08-29T15:25:15.917636+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"269'11915","reported_seq":19069,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:19.665845+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:25:19.665845+0000","last_peered":"2024-08-29T15:25:19.665845+0000","last_clean":"2024-08-29T15:25:19.665845+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:25:19.665845+0000","last_undegraded":"2024-08-29T15:25:19.665845+0000","last_fullsized":"2024-08-29T15:25:19.665845+0000","mapping_epoch":234,"log_start":"179'3999","ondisk_log_start":"179'3999","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7916,"log_dups_size":0,"ondisk_log_size":7916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230227968,"num_objects":3513,"num_object_clones":0,"num_object_copies":10539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11915,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":398,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.916224+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:25:15.916224+0000","last_peered":"2024-08-29T15:25:15.916224+0000","last_clean":"2024-08-29T15:25:15.916224+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:25:15.916224+0000","last_undegraded":"2024-08-29T15:25:15.916224+0000","last_fullsized":"2024-08-29T15:25:15.916224+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1021,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:15.918792+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:25:15.918792+0000","last_peered":"2024-08-29T15:25:15.918792+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:25:15.918792+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":396,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.914841+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:25:15.914841+0000","last_peered":"2024-08-29T15:25:15.914841+0000","last_clean":"2024-08-29T15:25:15.914841+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:25:15.914841+0000","last_undegraded":"2024-08-29T15:25:15.914841+0000","last_fullsized":"2024-08-29T15:25:15.914841+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"269'11993","reported_seq":18303,"reported_epoch":269,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:18.321120+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:25:18.321120+0000","last_peered":"2024-08-29T15:25:18.321120+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:25:18.321120+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"179'4412","ondisk_log_start":"179'4412","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7581,"log_dups_size":0,"ondisk_log_size":7581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237043712,"num_objects":3617,"num_object_clones":0,"num_object_copies":10851,"num_objects_missing_on_primary":4651,"num_objects_missing":4651,"num_objects_degraded":2313,"num_objects_misplaced":3602,"num_objects_unfound":0,"num_objects_dirty":3617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11993,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3070,"num_bytes_recovered":201064448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":1231},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":394,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917794+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:25:15.917794+0000","last_peered":"2024-08-29T15:25:15.917794+0000","last_clean":"2024-08-29T15:25:15.917794+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:25:15.917794+0000","last_undegraded":"2024-08-29T15:25:15.917794+0000","last_fullsized":"2024-08-29T15:25:15.917794+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":120,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.891292+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:25:07.891292+0000","last_peered":"2024-08-29T15:25:07.891292+0000","last_clean":"2024-08-29T15:25:07.891292+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:25:07.891292+0000","last_undegraded":"2024-08-29T15:25:07.891292+0000","last_fullsized":"2024-08-29T15:25:07.891292+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895212+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:25:07.895212+0000","last_peered":"2024-08-29T15:25:07.895212+0000","last_clean":"2024-08-29T15:25:07.895212+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:25:07.895212+0000","last_undegraded":"2024-08-29T15:25:07.895212+0000","last_fullsized":"2024-08-29T15:25:07.895212+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895523+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:25:07.895523+0000","last_peered":"2024-08-29T15:25:07.895523+0000","last_clean":"2024-08-29T15:25:07.895523+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:25:07.895523+0000","last_undegraded":"2024-08-29T15:25:07.895523+0000","last_fullsized":"2024-08-29T15:25:07.895523+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":46,"reported_epoch":268,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:07.895805+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:25:07.895805+0000","last_peered":"2024-08-29T15:25:07.895805+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:25:07.895805+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:25:07.895805+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":402,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917608+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:25:15.917608+0000","last_peered":"2024-08-29T15:25:15.917608+0000","last_clean":"2024-08-29T15:25:15.917608+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:25:15.917608+0000","last_undegraded":"2024-08-29T15:25:15.917608+0000","last_fullsized":"2024-08-29T15:25:15.917608+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":152,"reported_epoch":269,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:15.916296+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:25:15.916296+0000","last_peered":"2024-08-29T15:25:15.916296+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:25:15.916296+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:25:15.916296+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":137,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.893747+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:25:07.893747+0000","last_peered":"2024-08-29T15:25:07.893747+0000","last_clean":"2024-08-29T15:25:07.893747+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:25:07.893747+0000","last_undegraded":"2024-08-29T15:25:07.893747+0000","last_fullsized":"2024-08-29T15:25:07.893747+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":362,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.914971+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:25:15.914971+0000","last_peered":"2024-08-29T15:25:15.914971+0000","last_clean":"2024-08-29T15:25:15.914971+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:25:15.914971+0000","last_undegraded":"2024-08-29T15:25:15.914971+0000","last_fullsized":"2024-08-29T15:25:15.914971+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":415,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918260+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:25:15.918260+0000","last_peered":"2024-08-29T15:25:15.918260+0000","last_clean":"2024-08-29T15:25:15.918260+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:25:15.918260+0000","last_undegraded":"2024-08-29T15:25:15.918260+0000","last_fullsized":"2024-08-29T15:25:15.918260+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":414,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.914768+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:25:15.914768+0000","last_peered":"2024-08-29T15:25:15.914768+0000","last_clean":"2024-08-29T15:25:15.914768+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:25:15.914768+0000","last_undegraded":"2024-08-29T15:25:15.914768+0000","last_fullsized":"2024-08-29T15:25:15.914768+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:13.316871+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:25:13.316871+0000","last_peered":"2024-08-29T15:25:13.316871+0000","last_clean":"2024-08-29T15:25:13.316871+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:25:13.316871+0000","last_undegraded":"2024-08-29T15:25:13.316871+0000","last_fullsized":"2024-08-29T15:25:13.316871+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":270,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918596+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:25:15.918596+0000","last_peered":"2024-08-29T15:25:15.918596+0000","last_clean":"2024-08-29T15:25:15.918596+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:25:15.918596+0000","last_undegraded":"2024-08-29T15:25:15.918596+0000","last_fullsized":"2024-08-29T15:25:15.918596+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":391,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918054+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:25:15.918054+0000","last_peered":"2024-08-29T15:25:15.918054+0000","last_clean":"2024-08-29T15:25:15.918054+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:25:15.918054+0000","last_undegraded":"2024-08-29T15:25:15.918054+0000","last_fullsized":"2024-08-29T15:25:15.918054+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":394,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.920644+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:25:15.920644+0000","last_peered":"2024-08-29T15:25:15.920644+0000","last_clean":"2024-08-29T15:25:15.920644+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:25:15.920644+0000","last_undegraded":"2024-08-29T15:25:15.920644+0000","last_fullsized":"2024-08-29T15:25:15.920644+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":398,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.916162+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:25:15.916162+0000","last_peered":"2024-08-29T15:25:15.916162+0000","last_clean":"2024-08-29T15:25:15.916162+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:25:15.916162+0000","last_undegraded":"2024-08-29T15:25:15.916162+0000","last_fullsized":"2024-08-29T15:25:15.916162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895277+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:25:07.895277+0000","last_peered":"2024-08-29T15:25:07.895277+0000","last_clean":"2024-08-29T15:25:07.895277+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:25:07.895277+0000","last_undegraded":"2024-08-29T15:25:07.895277+0000","last_fullsized":"2024-08-29T15:25:07.895277+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":418,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.921286+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:25:15.921286+0000","last_peered":"2024-08-29T15:25:15.921286+0000","last_clean":"2024-08-29T15:25:15.921286+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:25:15.921286+0000","last_undegraded":"2024-08-29T15:25:15.921286+0000","last_fullsized":"2024-08-29T15:25:15.921286+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":382,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.917544+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:25:15.917544+0000","last_peered":"2024-08-29T15:25:15.917544+0000","last_clean":"2024-08-29T15:25:15.917544+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:25:15.917544+0000","last_undegraded":"2024-08-29T15:25:15.917544+0000","last_fullsized":"2024-08-29T15:25:15.917544+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895314+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:25:07.895314+0000","last_peered":"2024-08-29T15:25:07.895314+0000","last_clean":"2024-08-29T15:25:07.895314+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:25:07.895314+0000","last_undegraded":"2024-08-29T15:25:07.895314+0000","last_fullsized":"2024-08-29T15:25:07.895314+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":161,"reported_epoch":268,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:11.184329+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:25:11.184329+0000","last_peered":"2024-08-29T15:25:11.184329+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:25:11.184329+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:25:11.184329+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895349+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:25:07.895349+0000","last_peered":"2024-08-29T15:25:07.895349+0000","last_clean":"2024-08-29T15:25:07.895349+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:25:07.895349+0000","last_undegraded":"2024-08-29T15:25:07.895349+0000","last_fullsized":"2024-08-29T15:25:07.895349+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":167,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:11.184439+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:25:11.184439+0000","last_peered":"2024-08-29T15:25:11.184439+0000","last_clean":"2024-08-29T15:25:11.184439+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:25:11.184439+0000","last_undegraded":"2024-08-29T15:25:11.184439+0000","last_fullsized":"2024-08-29T15:25:11.184439+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":429,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.915185+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:25:15.915185+0000","last_peered":"2024-08-29T15:25:15.915185+0000","last_clean":"2024-08-29T15:25:15.915185+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:25:15.915185+0000","last_undegraded":"2024-08-29T15:25:15.915185+0000","last_fullsized":"2024-08-29T15:25:15.915185+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":431,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.915171+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:25:15.915171+0000","last_peered":"2024-08-29T15:25:15.915171+0000","last_clean":"2024-08-29T15:25:15.915171+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:25:15.915171+0000","last_undegraded":"2024-08-29T15:25:15.915171+0000","last_fullsized":"2024-08-29T15:25:15.915171+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":48,"reported_epoch":268,"state":"active+clean","last_fresh":"2024-08-29T15:25:07.895029+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:25:07.895029+0000","last_peered":"2024-08-29T15:25:07.895029+0000","last_clean":"2024-08-29T15:25:07.895029+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:25:07.895029+0000","last_undegraded":"2024-08-29T15:25:07.895029+0000","last_fullsized":"2024-08-29T15:25:07.895029+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":438,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.918178+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:25:15.918178+0000","last_peered":"2024-08-29T15:25:15.918178+0000","last_clean":"2024-08-29T15:25:15.918178+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:25:15.918178+0000","last_undegraded":"2024-08-29T15:25:15.918178+0000","last_fullsized":"2024-08-29T15:25:15.918178+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3814916120,"num_objects":58212,"num_object_clones":0,"num_object_copies":174636,"num_objects_missing_on_primary":4651,"num_objects_missing":4651,"num_objects_degraded":29234,"num_objects_misplaced":10840,"num_objects_unfound":0,"num_objects_dirty":58212,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":192638,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17536,"num_bytes_recovered":1029242880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11521306624,"data_stored":11521294408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124215,"ondisk_log_size":124215,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234368,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2202472,"kb_used_data":1874400,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":91542680,"statfs":{"total":95995035648,"available":93739704320,"internally_reserved":0,"allocated":1919385600,"data_stored":1917831707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":335935947},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495999,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":250020,"kb_used_data":66332,"kb_used_omap":1,"kb_used_meta":183678,"kb_avail":93495132,"statfs":{"total":95995035648,"available":95739015168,"internally_reserved":0,"allocated":67923968,"data_stored":66373875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188086727},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426586,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1244696,"kb_used_data":1171536,"kb_used_omap":8,"kb_used_meta":73143,"kb_avail":92500456,"statfs":{"total":95995035648,"available":94720466944,"internally_reserved":0,"allocated":1199652864,"data_stored":1198110237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":74899338},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019278,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1495624,"kb_used_data":1171648,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":92249528,"statfs":{"total":95995035648,"available":94463516672,"internally_reserved":0,"allocated":1199767552,"data_stored":1198240708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435463,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2636564,"kb_used_data":2325900,"kb_used_omap":4,"kb_used_meta":310651,"kb_avail":91108588,"statfs":{"total":95995035648,"available":93295194112,"internally_reserved":0,"allocated":2381721600,"data_stored":2380182913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":318106889},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903315,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2289752,"kb_used_data":2149712,"kb_used_omap":10,"kb_used_meta":140021,"kb_avail":91455400,"statfs":{"total":95995035648,"available":93650329600,"internally_reserved":0,"allocated":2201305088,"data_stored":2199766337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":143382032},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762516,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1871956,"kb_used_data":1597324,"kb_used_omap":7,"kb_used_meta":274616,"kb_avail":91873196,"statfs":{"total":95995035648,"available":94078152704,"internally_reserved":0,"allocated":1635659776,"data_stored":1634119849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":281207138},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926042,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267996,"kb_used_data":951700,"kb_used_omap":7,"kb_used_meta":316280,"kb_avail":92477156,"statfs":{"total":95995035648,"available":94696607744,"internally_reserved":0,"allocated":974540800,"data_stored":972993652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":323870863},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":964759552,"data_stored":964755480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1623134208,"data_stored":1623130136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2196045824,"data_stored":2196045824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2376138752,"data_stored":2376138752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1194196992,"data_stored":1194196992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1192497152,"data_stored":1192493080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":63569920,"data_stored":63569920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1910964224,"data_stored":1910964224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:25:26.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:26.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:26.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:26.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:26.204 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:25 smithi027 ceph-mon[98970]: pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 29263/174249 objects degraded (16.794%); 10784/174249 objects misplaced (6.189%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:26.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:25 smithi027 ceph-mon[93921]: pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 29263/174249 objects degraded (16.794%); 10784/174249 objects misplaced (6.189%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:26.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:26.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:26.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:25 smithi171 ceph-mon[90031]: pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 171 op/s; 29263/174249 objects degraded (16.794%); 10784/174249 objects misplaced (6.189%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:26.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:26.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:26.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:26.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:26.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:26.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:26.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:26.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:26.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:25:26.658 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:26.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:26.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:26.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:26.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:26.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:26.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:25:26.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:26.926 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:26.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:26.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:26.933 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:25:27.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:27.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:27.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:27.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:27.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:27.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3441032954' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:27.189 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:26 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29263/174249 objects degraded (16.794%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:27.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3441032954' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:27.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:26 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29263/174249 objects degraded (16.794%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3441032954' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:27.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:26 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29263/174249 objects degraded (16.794%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:27.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:27.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:27.366 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:27.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:27.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:27.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:27.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:27.611 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:27.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:27.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:27.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:27.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:27.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:27.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:27.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:28.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:28.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[93921]: pgmap v901: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 29212/172401 objects degraded (16.944%); 10698/172401 objects misplaced (6.205%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:25:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3441032954' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[93921]: osdmap e270: 8 total, 8 up, 8 in 2024-08-29T15:25:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:28.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[98970]: pgmap v901: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 29212/172401 objects degraded (16.944%); 10698/172401 objects misplaced (6.205%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:25:28.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:28.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3441032954' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:28.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[98970]: osdmap e270: 8 total, 8 up, 8 in 2024-08-29T15:25:28.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:28.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:27 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:28.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:28.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:28.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:27 smithi171 ceph-mon[90031]: pgmap v901: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 29212/172401 objects degraded (16.944%); 10698/172401 objects misplaced (6.205%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:25:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:27 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3441032954' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:27 smithi171 ceph-mon[90031]: osdmap e270: 8 total, 8 up, 8 in 2024-08-29T15:25:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:27 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:28.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:27 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:28.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:28.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:28.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:28.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:28.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:28.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:28.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:28.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:28.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:28.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:28.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:28.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:28.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:28.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:29.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:29.080 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:29.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:29.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:25:29.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:29.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:29.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:29.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:29.312 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:25:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:25:29] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:25:29.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:29.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:29.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:29.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:29.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:29.560 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:25:29.560 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124404 6.3 6:cbf2ce75:::benchmark_data_smithi027_169445_object124403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e188)", 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 511.52847416899999, 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3120749140000001, 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.561 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124404 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029267+0000", 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029265+0000", 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029301+0000", 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5939000000000003e-05 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029311+0000", 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5869999999999999e-06 2024-08-29T15:25:29.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.029318+0000", 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0829999999999998e-06 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.036882+0000", 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075643280000000004 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.036891+0000", 2024-08-29T15:25:29.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3100000000000001e-06 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029775+0000", 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99288402499999995 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029786+0000", 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1311000000000001e-05 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.313838+0000", 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2840517499999999 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.313991+0000", 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015349299999999999 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.314109+0000", 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011786300000000001 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.320971+0000", 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0068620870000000002 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325526+0000", 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045548979999999999 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337613+0000", 2024-08-29T15:25:29.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012086573999999999 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337664+0000", 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1085999999999998e-05 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.341342+0000", 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036781280000000001 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.569 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.569 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124411 6.5 6:a9772c5a:::benchmark_data_smithi027_169445_object124410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:25:29.569 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:25:29.569 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 511.52048342299997, 2024-08-29T15:25:29.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2782022199999998, 2024-08-29T15:25:29.569 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124411 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037258+0000", 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037257+0000", 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037284+0000", 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6716e-05 2024-08-29T15:25:29.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037290+0000", 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0850000000000002e-06 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037294+0000", 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4610000000000001e-06 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037613+0000", 2024-08-29T15:25:29.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031895900000000001 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.037620+0000", 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7800000000000003e-06 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031231+0000", 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99361049700000004 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031239+0000", 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.473e-06 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.294818+0000", 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2635784000000001 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.294989+0000", 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000171375 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:25:29.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.295176+0000", 2024-08-29T15:25:29.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:29.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000187002 2024-08-29T15:25:29.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303052+0000", 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078760210000000004 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307520+0000", 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044681469999999996 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.312052+0000", 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045315149999999998 2024-08-29T15:25:29.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.312092+0000", 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0621000000000003e-05 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315460+0000", 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033675900000000002 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.578 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124424 6.5 6:a6739d5e:::benchmark_data_smithi027_169445_object124423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 511.49940768800002, 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.268536273, 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124424 2024-08-29T15:25:29.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058333+0000", 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058331+0000", 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058377+0000", 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6134000000000002e-05 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058391+0000", 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3995000000000001e-05 2024-08-29T15:25:29.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058398+0000", 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2060000000000001e-06 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058448+0000", 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9456999999999998e-05 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.058460+0000", 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1622e-05 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031289+0000", 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97282904800000003 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031295+0000", 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8249999999999999e-06 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303104+0000", 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.271809059 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303275+0000", 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017008999999999999 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.303459+0000", 2024-08-29T15:25:29.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018486800000000001 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315479+0000", 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012019724000000001 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315517+0000", 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7370999999999998e-05 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.322771+0000", 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072540670000000003 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.322827+0000", 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6232e-05 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326870+0000", 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040427620000000001 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.586 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124428 6.3 6:cbc20026:::benchmark_data_smithi027_169445_object124427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 511.497251006, 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3095511559999999, 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124428 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060490+0000", 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060489+0000", 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:29.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060581+0000", 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1643000000000003e-05 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060594+0000", 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3222e-05 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060601+0000", 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8449999999999997e-06 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060631+0000", 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0372999999999999e-05 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.060637+0000", 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4740000000000002e-06 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029808+0000", 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96917036400000001 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029815+0000", 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8190000000000003e-06 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321001+0000", 2024-08-29T15:25:29.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.29118635 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321186+0000", 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018538300000000001 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.321355+0000", 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000168238 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337515+0000", 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016160774999999999 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337526+0000", 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1053000000000001e-05 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.366213+0000", 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028686204 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.366281+0000", 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8071999999999995e-05 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370041+0000", 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037604409999999998 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124433 6.3 6:ca9f8931:::benchmark_data_smithi027_169445_object124432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 511.49073009699998, 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3129230789999999, 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124433 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067011+0000", 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067010+0000", 2024-08-29T15:25:29.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067116+0000", 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106204 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067125+0000", 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.285e-06 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067130+0000", 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1660000000000002e-06 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067165+0000", 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5108000000000003e-05 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.067172+0000", 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0650000000000001e-06 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029836+0000", 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96266382100000003 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.029843+0000", 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3280000000000002e-06 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325550+0000", 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2957075179999999 2024-08-29T15:25:29.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325689+0000", 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013860200000000001 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.325852+0000", 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016336399999999999 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.337576+0000", 2024-08-29T15:25:29.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011724399 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347548+0000", 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099717829999999997 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370096+0000", 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0225477 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.370146+0000", 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0419e-05 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.379934+0000", 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0097876270000000001 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124444 6.5 6:af81ef90:::benchmark_data_smithi027_169445_object124443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:25:29.602 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 511.47871964299998, 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.251856976, 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124444 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079021+0000", 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079020+0000", 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079087+0000", 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6513999999999994e-05 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079099+0000", 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2391999999999999e-05 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079104+0000", 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3589999999999999e-06 2024-08-29T15:25:29.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079138+0000", 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4041000000000002e-05 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:58.079145+0000", 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9870000000000002e-06 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031398+0000", 2024-08-29T15:25:29.606 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95225246100000005 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:16:59.031406+0000", 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7789999999999996e-06 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307552+0000", 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.276146349 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307695+0000", 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014288899999999999 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.307892+0000", 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019679800000000001 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315574+0000", 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0076828130000000001 2024-08-29T15:25:29.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.315619+0000", 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5049000000000002e-05 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326936+0000", 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011316794 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.326989+0000", 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2293999999999998e-05 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330878+0000", 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038897950000000001 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124462 6.3 6:cf3b5247:::benchmark_data_smithi027_169445_object124461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2024-08-29T15:25:29.610 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 509.39242850400001, 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.21866085199999999, 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124462 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:25:29.611 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165313+0000", 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165309+0000", 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.612 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165450+0000", 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014043500000000001 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165476+0000", 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5859000000000001e-05 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165489+0000", 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2765999999999999e-05 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165566+0000", 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7019000000000002e-05 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.165580+0000", 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4005999999999999e-05 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330157+0000", 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16457785 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330324+0000", 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016682599999999999 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.330523+0000", 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019885899999999999 2024-08-29T15:25:29.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.341411+0000", 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010887468000000001 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347578+0000", 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0061672610000000003 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.616 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.379991+0000", 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032413320000000002 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.380174+0000", 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018272399999999999 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.383973+0000", 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037995310000000001 2024-08-29T15:25:29.617 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:124473 6.3 6:c5e714a2:::benchmark_data_smithi027_169445_object124472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e191)", 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 509.362203776, 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19243169900000001, 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.618 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 124473 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.621 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195537+0000", 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195531+0000", 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999933 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195597+0000", 2024-08-29T15:25:29.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6513000000000005e-05 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195613+0000", 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5865000000000001e-05 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195620+0000", 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2259999999999999e-06 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195679+0000", 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8695e-05 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.195690+0000", 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1473e-05 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342341+0000", 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.146650369 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342504+0000", 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000162812 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.342677+0000", 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000172874 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347795+0000", 2024-08-29T15:25:29.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051186010000000004 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.347888+0000", 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2949999999999996e-05 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.384025+0000", 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036137057 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.384081+0000", 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5881000000000003e-05 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.387969+0000", 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038879259999999999 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPushReply(6.5 191/190 [PushReplyOp(6:a4839e33:::benchmark_data_smithi027_169445_object147:head)])", 2024-08-29T15:25:29.627 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 508.57795841500001, 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13693580699999999, 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979783+0000", 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979781+0000", 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979790+0000", 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8929999999999994e-06 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979811+0000", 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0508000000000001e-05 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:00.979820+0000", 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2040000000000003e-06 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:01.107955+0000", 2024-08-29T15:25:29.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12813488200000001 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:01.116718+0000", 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087634329999999993 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137594 6.3 6:c9cad5ba:::benchmark_data_smithi027_169445_object12167:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 491.85286270099999, 2024-08-29T15:25:29.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13802763300000001, 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137594 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704878+0000", 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704881+0000", 2024-08-29T15:25:29.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.627e-06 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704886+0000", 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1390000000000003e-06 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.704890+0000", 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1139999999999999e-06 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711314+0000", 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0064242639999999998 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711480+0000", 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016600600000000001 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.711632+0000", 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015182300000000001 2024-08-29T15:25:29.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717197+0000", 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055650309999999998 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732501+0000", 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015303654 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.838889+0000", 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10638834899999999 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.838943+0000", 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4279e-05 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.842906+0000", 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039627280000000004 2024-08-29T15:25:29.637 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137598 6.3 6:cf32642d:::benchmark_data_smithi027_169445_object12171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 491.85047432800002, 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.139080008, 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.638 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137598 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707267+0000", 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707265+0000", 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707271+0000", 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.48e-06 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707279+0000", 2024-08-29T15:25:29.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8459999999999997e-06 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.707285+0000", 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7899999999999996e-06 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717221+0000", 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099359559999999993 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717372+0000", 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015091400000000001 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.717520+0000", 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014827699999999999 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.729379+0000", 2024-08-29T15:25:29.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011858485 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732510+0000", 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003130678 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.842959+0000", 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.110449724 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.843014+0000", 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4672000000000001e-05 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846347+0000", 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033327650000000001 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:137609 6.3 6:c98a9743:::benchmark_data_smithi027_169445_object12182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:25:29.644 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 491.83654116500003, 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12867278300000001, 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 137609 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721200+0000", 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721199+0000", 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721202+0000", 2024-08-29T15:25:29.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0939999999999999e-06 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721209+0000", 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0800000000000002e-06 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721213+0000", 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5600000000000004e-06 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721607+0000", 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00039427400000000001 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721766+0000", 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000158667 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,6", 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.721903+0000", 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013681500000000001 2024-08-29T15:25:29.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.729409+0000", 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075061219999999996 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.732515+0000", 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031063250000000001 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846478+0000", 2024-08-29T15:25:29.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.113962623 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.846534+0000", 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6375000000000001e-05 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:17.849873+0000", 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033383330000000002 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:138635 6.5 6:aff223a2:::benchmark_data_smithi027_169445_object13208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 490.57570347500001, 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14247343300000001, 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 138635 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.651 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982038+0000", 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982036+0000", 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982044+0000", 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0830000000000004e-06 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982598+0000", 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00055435700000000001 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:18.982609+0000", 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0635999999999999e-05 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.003985+0000", 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021376350999999998 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.004101+0000", 2024-08-29T15:25:29.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011584 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083026+0000", 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078925680999999998 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083103+0000", 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6506000000000002e-05 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:25:29.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.083205+0000", 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000101593 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.091336+0000", 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081309610000000008 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.095604+0000", 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042681050000000003 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.120534+0000", 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024930548 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.120652+0000", 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011811099999999999 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.124511+0000", 2024-08-29T15:25:29.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038586829999999999 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:139269 6.5 6:a8585b14:::benchmark_data_smithi027_169445_object13842:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 489.76012455300003, 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.174785774, 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.658 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 139269 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797616+0000", 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797615+0000", 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797621+0000", 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9689999999999999e-06 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797637+0000", 2024-08-29T15:25:29.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6127999999999999e-05 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.797645+0000", 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2130000000000008e-06 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.821075+0000", 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023429189 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:29.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.821189+0000", 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011487799999999999 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.940614+0000", 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.119424749 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.940879+0000", 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00026527700000000001 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.941261+0000", 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00038113399999999999 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968451+0000", 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027190783999999999 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968461+0000", 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1959999999999997e-06 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968492+0000", 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1841999999999998e-05 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.968542+0000", 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9348000000000003e-05 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:19.972402+0000", 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038604500000000001 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:139443 6.5 6:a02b9a8b:::benchmark_data_smithi027_169445_object14016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 489.52056767300002, 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15171564100000001, 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.665 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 139443 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:25:29.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037173+0000", 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037176+0000", 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.078e-06 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037182+0000", 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8540000000000002e-06 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.037187+0000", 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8110000000000002e-06 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.061998+0000", 2024-08-29T15:25:29.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024811600999999999 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.062127+0000", 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012901000000000001 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.100824+0000", 2024-08-29T15:25:29.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038696362999999998 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.100935+0000", 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011161499999999999 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.104035+0000", 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030998649999999998 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.130597+0000", 2024-08-29T15:25:29.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026562129 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.152790+0000", 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022192587999999999 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.184767+0000", 2024-08-29T15:25:29.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031977515999999997 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.184820+0000", 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3084999999999997e-05 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:17:20.188889+0000", 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040686769999999997 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.673 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:142989 6.3 6.e160cdf3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 439.70266844000002, 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98678860400000001, 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 142989 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.674 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855073+0000", 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855071+0000", 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855079+0000", 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5390000000000007e-06 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855093+0000", 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3835000000000001e-05 2024-08-29T15:25:29.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855102+0000", 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9740000000000008e-06 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855146+0000", 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4082000000000002e-05 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:09.855154+0000", 2024-08-29T15:25:29.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.622e-06 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.841849+0000", 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98669542700000001 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.841861+0000", 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1803e-05 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:142999 6.5 6.8bdf2d15 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 439.39800745100001, 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.683494037, 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 142999 2024-08-29T15:25:29.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159734+0000", 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159731+0000", 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159743+0000", 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1389000000000001e-05 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159759+0000", 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6514000000000001e-05 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159769+0000", 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4409999999999999e-06 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159822+0000", 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2806999999999998e-05 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.159834+0000", 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2047e-05 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843215+0000", 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.68338162599999996 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843228+0000", 2024-08-29T15:25:29.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2296999999999999e-05 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:143011 6.5 6.1a21b555 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 439.02467836400001, 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.310191416, 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:25:29.684 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 143011 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533063+0000", 2024-08-29T15:25:29.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533061+0000", 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533069+0000", 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5210000000000002e-06 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533083+0000", 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4115e-05 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533094+0000", 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0427e-05 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533134+0000", 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0201999999999999e-05 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.533144+0000", 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0164e-05 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843246+0000", 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.31010232399999998 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:10.843254+0000", 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7239999999999992e-06 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.688 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44912.0:170085 6.c e254/202 6:30e4c63e:::benchmark_data_smithi027_169445_object44658:head v 254'10546, mlcod=254'10544)", 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 159.13089852300001, 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0096297399999999995, 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:25:29.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426842+0000", 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426849+0000", 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.092e-06 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426872+0000", 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3258e-05 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.426879+0000", 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2429999999999999e-06 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.427777+0000", 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00089768400000000005 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.427805+0000", 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8583000000000001e-05 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.436412+0000", 2024-08-29T15:25:29.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086069579999999996 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:22:50.436472+0000", 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9851e-05 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44912.0:187222 6.c e265/202 6:36b3ea00:::benchmark_data_smithi027_169445_object61795:head v 265'11634, mlcod=265'11632)", 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:24:48.426501+0000", 2024-08-29T15:25:29.693 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 41.131240206999998, 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087887710000000008, 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426501+0000", 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426501+0000", 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426500+0000", 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426509+0000", 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9730000000000003e-06 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426528+0000", 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9786e-05 2024-08-29T15:25:29.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426536+0000", 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4749999999999996e-06 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426580+0000", 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4678999999999997e-05 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.426604+0000", 2024-08-29T15:25:29.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3411e-05 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.435225+0000", 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086208190000000001 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:24:48.435290+0000", 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4888000000000006e-05 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.697 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:25:29.698 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:25:29.698 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:25:29.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:29.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:29.799 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:29.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:29.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:29.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:29.951 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:29.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:29.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:29 smithi171 ceph-mon[90031]: pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29212/172227 objects degraded (16.961%); 10642/172227 objects misplaced (6.179%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:30.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:29 smithi027 ceph-mon[93921]: pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29212/172227 objects degraded (16.961%); 10642/172227 objects misplaced (6.179%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:29 smithi027 ceph-mon[98970]: pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 29212/172227 objects degraded (16.961%); 10642/172227 objects misplaced (6.179%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:30.077 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:30.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:25:30.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:30.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:30.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:30.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:30.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:30.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:30.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:30.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:30.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:25:30.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:30.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:30.803 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:30.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:30.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:30.898 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:30.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:30.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:30.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:30.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:31.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:31.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:31.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:31.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:31.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:31.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:31.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:31.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:31.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:31.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:31.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:31.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:31.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:31.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:31.822 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:31.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:31.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:31.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:31.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:31.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:32.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[98970]: pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 29212/171363 objects degraded (17.047%); 10642/171363 objects misplaced (6.210%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:32.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:25:32.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:25:32.049 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[93921]: pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 29212/171363 objects degraded (17.047%); 10642/171363 objects misplaced (6.210%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:32.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:25:32.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:25:32.050 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:32.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:32.179 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:31 smithi171 ceph-mon[90031]: pgmap v904: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 29212/171363 objects degraded (17.047%); 10642/171363 objects misplaced (6.210%); 1.4 MiB/s, 22 objects/s recovering 2024-08-29T15:25:32.179 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.179 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.180 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:25:32.180 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:25:32.180 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:32.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:32.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:32.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:32.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:32.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:32.434 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:32.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:32.472 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:32.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:32.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:32.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:32.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:32.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:32.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:32.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:32.831 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:25:32.832 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:25:32.832 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:25:32.832 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:25:32.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:32.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:32.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:32.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:33.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29212/171363 objects degraded (17.047%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:33.026 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29212/171363 objects degraded (17.047%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:33.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:33.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:33.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:33.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:25:33.144 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:25:33.145 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:25:33.158 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:25:33.158 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:25:33.180 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:25:33.181 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:25:33.204 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:25:33.204 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:25:33.205 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29212/171363 objects degraded (17.047%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:33.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:33.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:33.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:33.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:33.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:33.383 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:33.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:33.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:33.485 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:33.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:33.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:33.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:33.635 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:33.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:33.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:33.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:25:33.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:33.761 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:33.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:33.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:33.814 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:25:33.829 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:33 smithi171 ceph-mon[90031]: pgmap v905: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 29136/169182 objects degraded (17.222%); 10512/169182 objects misplaced (6.213%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:33.829 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/936526379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:33.829 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2574314945' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:25:33.829 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:33 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:25:33.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:33.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:33.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:33.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:33.944 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:33.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:34.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:34.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[98970]: pgmap v905: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 29136/169182 objects degraded (17.222%); 10512/169182 objects misplaced (6.213%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/936526379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2574314945' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:25:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:25:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[93921]: pgmap v905: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 29136/169182 objects degraded (17.222%); 10512/169182 objects misplaced (6.213%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/936526379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2574314945' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:25:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:33 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:25:34.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:34.139 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:34.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:34.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:34.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:34.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:34.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:34.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:34.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:34.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:34.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:34.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:34.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:34.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:34.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:34.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:34.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:34.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:34.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:34.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:35.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:35.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:35.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:35.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:35.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:35.234 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:34 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:35.234 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:34 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:25:35.235 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:34 smithi171 ceph-mon[90031]: osdmap e271: 8 total, 8 up, 8 in 2024-08-29T15:25:35.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:35.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:35.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:35.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:34 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:35.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:34 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:25:35.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:34 smithi027 ceph-mon[98970]: osdmap e271: 8 total, 8 up, 8 in 2024-08-29T15:25:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:34 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:34 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:25:35.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:34 smithi027 ceph-mon[93921]: osdmap e271: 8 total, 8 up, 8 in 2024-08-29T15:25:35.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:35.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:35.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:35.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:35.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:35.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:35.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:35.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:35.780 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:35.860 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:35 smithi171 ceph-mon[90031]: pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 29136/168717 objects degraded (17.269%); 10452/168717 objects misplaced (6.195%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:35.868 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:25:35.868 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:25:35.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:35.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:35.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:35.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:36.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:36.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:36.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:36.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:36.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:35 smithi027 ceph-mon[93921]: pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 29136/168717 objects degraded (17.269%); 10452/168717 objects misplaced (6.195%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:36.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:35 smithi027 ceph-mon[98970]: pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 29136/168717 objects degraded (17.269%); 10452/168717 objects misplaced (6.195%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:36.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:36.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.286 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:36.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:36.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:36.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:36.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:36.534 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:36.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:36.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:36.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:25:36.748 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:25:36.749 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:25:36.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:36.792 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:36.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:36.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:36.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:36.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:36.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:36.992 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:25:37.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:37.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:37.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:37.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:37.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:37.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:37.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:37.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:37.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:25:37.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:37.420 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:37.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:37.530 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":908,"stamp":"2024-08-29T15:25:36.634847+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3644856317,"num_objects":55572,"num_object_clones":0,"num_object_copies":166716,"num_objects_missing_on_primary":4487,"num_objects_missing":4487,"num_objects_degraded":29088,"num_objects_misplaced":10345,"num_objects_unfound":0,"num_objects_dirty":55572,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":196429,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17991,"num_bytes_recovered":1057885389,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125365,"ondisk_log_size":125365,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12758580,"kb_used_data":10841000,"kb_used_omap":43,"kb_used_meta":1917460,"kb_avail":737202636,"statfs":{"total":767960285184,"available":754895499264,"internally_reserved":0,"allocated":11101184000,"data_stored":11088738958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":1963479532},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,1,2,1,2,0,4,5,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-124190720,"num_objects":-1895,"num_object_clones":0,"num_object_copies":-5685,"num_objects_missing_on_primary":-116,"num_objects_missing":-116,"num_objects_degraded":-124,"num_objects_misplaced":-353,"num_objects_unfound":0,"num_objects_dirty":-1895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1895,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":239,"num_bytes_recovered":15663104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.700555"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":428,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.868120+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:25:33.868120+0000","last_peered":"2024-08-29T15:25:33.868120+0000","last_clean":"2024-08-29T15:25:33.868120+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:25:33.868120+0000","last_undegraded":"2024-08-29T15:25:33.868120+0000","last_fullsized":"2024-08-29T15:25:33.868120+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":49,"reported_epoch":270,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:26.937828+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:25:26.937828+0000","last_peered":"2024-08-29T15:25:26.937828+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:25:26.937828+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:25:26.937828+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":139,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935833+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:25:26.935833+0000","last_peered":"2024-08-29T15:25:26.935833+0000","last_clean":"2024-08-29T15:25:26.935833+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:25:26.935833+0000","last_undegraded":"2024-08-29T15:25:26.935833+0000","last_fullsized":"2024-08-29T15:25:26.935833+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":393,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.812027+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:25:33.812027+0000","last_peered":"2024-08-29T15:25:33.812027+0000","last_clean":"2024-08-29T15:25:33.812027+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:25:33.812027+0000","last_undegraded":"2024-08-29T15:25:33.812027+0000","last_fullsized":"2024-08-29T15:25:33.812027+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":393,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810802+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:25:33.810802+0000","last_peered":"2024-08-29T15:25:33.810802+0000","last_clean":"2024-08-29T15:25:33.810802+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:25:33.810802+0000","last_undegraded":"2024-08-29T15:25:33.810802+0000","last_fullsized":"2024-08-29T15:25:33.810802+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937422+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:25:26.937422+0000","last_peered":"2024-08-29T15:25:26.937422+0000","last_clean":"2024-08-29T15:25:26.937422+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:25:26.937422+0000","last_undegraded":"2024-08-29T15:25:26.937422+0000","last_fullsized":"2024-08-29T15:25:26.937422+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":205,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813213+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:25:33.813213+0000","last_peered":"2024-08-29T15:25:33.813213+0000","last_clean":"2024-08-29T15:25:33.813213+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:25:33.813213+0000","last_undegraded":"2024-08-29T15:25:33.813213+0000","last_fullsized":"2024-08-29T15:25:33.813213+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":139,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.948500+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:25:15.948500+0000","last_peered":"2024-08-29T15:25:15.948500+0000","last_clean":"2024-08-29T15:25:15.948500+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:25:15.948500+0000","last_undegraded":"2024-08-29T15:25:15.948500+0000","last_fullsized":"2024-08-29T15:25:15.948500+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"271'12376","reported_seq":20426,"reported_epoch":271,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:36.081532+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:25:36.081532+0000","last_peered":"2024-08-29T15:25:36.081532+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:25:36.081532+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"177'2300","ondisk_log_start":"177'2300","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231735296,"num_objects":3536,"num_object_clones":0,"num_object_copies":10608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12376,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":611}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":48,"reported_epoch":270,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:26.938207+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:25:26.938207+0000","last_peered":"2024-08-29T15:25:26.938207+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:25:26.938207+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:25:26.938207+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":157,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.934853+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:25:26.934853+0000","last_peered":"2024-08-29T15:25:26.934853+0000","last_clean":"2024-08-29T15:25:26.934853+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:25:26.934853+0000","last_undegraded":"2024-08-29T15:25:26.934853+0000","last_fullsized":"2024-08-29T15:25:26.934853+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"271'12085","reported_seq":12306,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:36.210748+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:25:36.210748+0000","last_peered":"2024-08-29T15:25:36.210748+0000","last_clean":"2024-08-29T15:25:36.210748+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:25:36.210748+0000","last_undegraded":"2024-08-29T15:25:36.210748+0000","last_fullsized":"2024-08-29T15:25:36.210748+0000","mapping_epoch":168,"log_start":"181'5328","ondisk_log_start":"181'5328","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227082240,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12085,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":48,"reported_epoch":270,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:26.940915+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:25:26.940915+0000","last_peered":"2024-08-29T15:25:26.940915+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:25:26.940915+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:25:26.940915+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937358+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:25:26.937358+0000","last_peered":"2024-08-29T15:25:26.937358+0000","last_clean":"2024-08-29T15:25:26.937358+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:25:26.937358+0000","last_undegraded":"2024-08-29T15:25:26.937358+0000","last_fullsized":"2024-08-29T15:25:26.937358+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"271'12273","reported_seq":12464,"reported_epoch":271,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:36.116670+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:25:36.116670+0000","last_peered":"2024-08-29T15:25:36.116670+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:25:36.116670+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"185'5574","ondisk_log_start":"185'5574","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225378304,"num_objects":3439,"num_object_clones":0,"num_object_copies":10317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12273,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":962}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":210,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.811716+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:25:33.811716+0000","last_peered":"2024-08-29T15:25:33.811716+0000","last_clean":"2024-08-29T15:25:33.811716+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:25:33.811716+0000","last_undegraded":"2024-08-29T15:25:33.811716+0000","last_fullsized":"2024-08-29T15:25:33.811716+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":94,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.948873+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:25:15.948873+0000","last_peered":"2024-08-29T15:25:15.948873+0000","last_clean":"2024-08-29T15:25:15.948873+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:25:15.948873+0000","last_undegraded":"2024-08-29T15:25:15.948873+0000","last_fullsized":"2024-08-29T15:25:15.948873+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"270'12273","reported_seq":17714,"reported_epoch":271,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:33.810081+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:25:33.810081+0000","last_peered":"2024-08-29T15:25:33.810081+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:25:33.810081+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"179'4162","ondisk_log_start":"179'4162","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8111,"log_dups_size":0,"ondisk_log_size":8111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229703680,"num_objects":3505,"num_object_clones":0,"num_object_copies":10515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12273,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3505}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":157,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935596+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:25:26.935596+0000","last_peered":"2024-08-29T15:25:26.935596+0000","last_clean":"2024-08-29T15:25:26.935596+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:25:26.935596+0000","last_undegraded":"2024-08-29T15:25:26.935596+0000","last_fullsized":"2024-08-29T15:25:26.935596+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":408,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810533+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:25:33.810533+0000","last_peered":"2024-08-29T15:25:33.810533+0000","last_clean":"2024-08-29T15:25:33.810533+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:25:33.810533+0000","last_undegraded":"2024-08-29T15:25:33.810533+0000","last_fullsized":"2024-08-29T15:25:33.810533+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"271'12264","reported_seq":12422,"reported_epoch":271,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:34.818115+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:25:34.818115+0000","last_peered":"2024-08-29T15:25:34.818115+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:25:34.818115+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:25:34.818115+0000","mapping_epoch":234,"log_start":"179'4333","ondisk_log_start":"179'4333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7931,"log_dups_size":0,"ondisk_log_size":7931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232652800,"num_objects":3550,"num_object_clones":0,"num_object_copies":10650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12264,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":1073},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":402,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.867315+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:25:33.867315+0000","last_peered":"2024-08-29T15:25:33.867315+0000","last_clean":"2024-08-29T15:25:33.867315+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:25:33.867315+0000","last_undegraded":"2024-08-29T15:25:33.867315+0000","last_fullsized":"2024-08-29T15:25:33.867315+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937713+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:25:26.937713+0000","last_peered":"2024-08-29T15:25:26.937713+0000","last_clean":"2024-08-29T15:25:26.937713+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:25:26.937713+0000","last_undegraded":"2024-08-29T15:25:26.937713+0000","last_fullsized":"2024-08-29T15:25:26.937713+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"271'12287","reported_seq":12525,"reported_epoch":271,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:36.194608+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:25:36.194608+0000","last_peered":"2024-08-29T15:25:36.194608+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:25:36.194608+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:25:36.194608+0000","mapping_epoch":234,"log_start":"184'5568","ondisk_log_start":"184'5568","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232980480,"num_objects":3555,"num_object_clones":0,"num_object_copies":10665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12287,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":1090},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":112,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.948748+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:25:15.948748+0000","last_peered":"2024-08-29T15:25:15.948748+0000","last_clean":"2024-08-29T15:25:15.948748+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:25:15.948748+0000","last_undegraded":"2024-08-29T15:25:15.948748+0000","last_fullsized":"2024-08-29T15:25:15.948748+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.938099+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:25:26.938099+0000","last_peered":"2024-08-29T15:25:26.938099+0000","last_clean":"2024-08-29T15:25:26.938099+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:25:26.938099+0000","last_undegraded":"2024-08-29T15:25:26.938099+0000","last_fullsized":"2024-08-29T15:25:26.938099+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"271'12138","reported_seq":12560,"reported_epoch":271,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:25:36.206962+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:25:36.206962+0000","last_peered":"2024-08-29T15:25:36.206962+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:25:36.206962+0000","last_undegraded":"2024-08-29T15:25:36.206962+0000","last_fullsized":"2024-08-29T15:25:36.206962+0000","mapping_epoch":202,"log_start":"183'5467","ondisk_log_start":"183'5467","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6671,"log_dups_size":0,"ondisk_log_size":6671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230686720,"num_objects":3520,"num_object_clones":0,"num_object_copies":10560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3520,"num_objects_unfound":0,"num_objects_dirty":3520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12138,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":3520}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937167+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:25:26.937167+0000","last_peered":"2024-08-29T15:25:26.937167+0000","last_clean":"2024-08-29T15:25:26.937167+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:25:26.937167+0000","last_undegraded":"2024-08-29T15:25:26.937167+0000","last_fullsized":"2024-08-29T15:25:26.937167+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":256,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.940914+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:25:26.940914+0000","last_peered":"2024-08-29T15:25:26.940914+0000","last_clean":"2024-08-29T15:25:26.940914+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:25:26.940914+0000","last_undegraded":"2024-08-29T15:25:26.940914+0000","last_fullsized":"2024-08-29T15:25:26.940914+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"271'12320","reported_seq":12472,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:36.062043+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:25:36.062043+0000","last_peered":"2024-08-29T15:25:36.062043+0000","last_clean":"2024-08-29T15:25:36.062043+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:25:36.062043+0000","last_undegraded":"2024-08-29T15:25:36.062043+0000","last_fullsized":"2024-08-29T15:25:36.062043+0000","mapping_epoch":218,"log_start":"177'2278","ondisk_log_start":"177'2278","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10042,"log_dups_size":0,"ondisk_log_size":10042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225378328,"num_objects":3440,"num_object_clones":0,"num_object_copies":10320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3440,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12320,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":405,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.867290+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:25:33.867290+0000","last_peered":"2024-08-29T15:25:33.867290+0000","last_clean":"2024-08-29T15:25:33.867290+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:25:33.867290+0000","last_undegraded":"2024-08-29T15:25:33.867290+0000","last_fullsized":"2024-08-29T15:25:33.867290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":397,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813223+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:25:33.813223+0000","last_peered":"2024-08-29T15:25:33.813223+0000","last_clean":"2024-08-29T15:25:33.813223+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:25:33.813223+0000","last_undegraded":"2024-08-29T15:25:33.813223+0000","last_fullsized":"2024-08-29T15:25:33.813223+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"271'11749","reported_seq":11949,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:35.701299+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:25:35.701299+0000","last_peered":"2024-08-29T15:25:35.701299+0000","last_clean":"2024-08-29T15:25:35.701299+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:25:35.701299+0000","last_undegraded":"2024-08-29T15:25:35.701299+0000","last_fullsized":"2024-08-29T15:25:35.701299+0000","mapping_epoch":201,"log_start":"176'1700","ondisk_log_start":"176'1700","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10049,"log_dups_size":0,"ondisk_log_size":10049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226689024,"num_objects":3459,"num_object_clones":0,"num_object_copies":10377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11749,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":496,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810360+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:25:33.810360+0000","last_peered":"2024-08-29T15:25:33.810360+0000","last_clean":"2024-08-29T15:25:33.810360+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:25:33.810360+0000","last_undegraded":"2024-08-29T15:25:33.810360+0000","last_fullsized":"2024-08-29T15:25:33.810360+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":741,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935155+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:25:26.935155+0000","last_peered":"2024-08-29T15:25:26.935155+0000","last_clean":"2024-08-29T15:25:26.935155+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:25:26.935155+0000","last_undegraded":"2024-08-29T15:25:26.935155+0000","last_fullsized":"2024-08-29T15:25:26.935155+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":249,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935070+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:25:26.935070+0000","last_peered":"2024-08-29T15:25:26.935070+0000","last_clean":"2024-08-29T15:25:26.935070+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:25:26.935070+0000","last_undegraded":"2024-08-29T15:25:26.935070+0000","last_fullsized":"2024-08-29T15:25:26.935070+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":111,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.956578+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:25:26.956578+0000","last_peered":"2024-08-29T15:25:26.956578+0000","last_clean":"2024-08-29T15:25:26.956578+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:25:26.956578+0000","last_undegraded":"2024-08-29T15:25:26.956578+0000","last_fullsized":"2024-08-29T15:25:26.956578+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"271'12245","reported_seq":12664,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:35.695712+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:25:35.695712+0000","last_peered":"2024-08-29T15:25:35.695712+0000","last_clean":"2024-08-29T15:25:35.695712+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:25:35.695712+0000","last_undegraded":"2024-08-29T15:25:35.695712+0000","last_fullsized":"2024-08-29T15:25:35.695712+0000","mapping_epoch":201,"log_start":"180'5253","ondisk_log_start":"180'5253","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225247232,"num_objects":3437,"num_object_clones":0,"num_object_copies":10311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12245,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":870,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810832+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:25:33.810832+0000","last_peered":"2024-08-29T15:25:33.810832+0000","last_clean":"2024-08-29T15:25:33.810832+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:25:33.810832+0000","last_undegraded":"2024-08-29T15:25:33.810832+0000","last_fullsized":"2024-08-29T15:25:33.810832+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":408,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810459+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:25:33.810459+0000","last_peered":"2024-08-29T15:25:33.810459+0000","last_clean":"2024-08-29T15:25:33.810459+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:25:33.810459+0000","last_undegraded":"2024-08-29T15:25:33.810459+0000","last_fullsized":"2024-08-29T15:25:33.810459+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":416,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.811016+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:25:33.811016+0000","last_peered":"2024-08-29T15:25:33.811016+0000","last_clean":"2024-08-29T15:25:33.811016+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:25:33.811016+0000","last_undegraded":"2024-08-29T15:25:33.811016+0000","last_fullsized":"2024-08-29T15:25:33.811016+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"271'12215","reported_seq":20086,"reported_epoch":271,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:36.441126+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:25:36.441126+0000","last_peered":"2024-08-29T15:25:36.441126+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:25:36.441126+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"184'5530","ondisk_log_start":"184'5530","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227737600,"num_objects":3475,"num_object_clones":0,"num_object_copies":10425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3951,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12215,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3840,"num_bytes_recovered":251658240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1036},{"shards":"3,6","objects":55},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":679,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.867026+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:25:33.867026+0000","last_peered":"2024-08-29T15:25:33.867026+0000","last_clean":"2024-08-29T15:25:33.867026+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:25:33.867026+0000","last_undegraded":"2024-08-29T15:25:33.867026+0000","last_fullsized":"2024-08-29T15:25:33.867026+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":101,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.948334+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:25:15.948334+0000","last_peered":"2024-08-29T15:25:15.948334+0000","last_clean":"2024-08-29T15:25:15.948334+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:25:15.948334+0000","last_undegraded":"2024-08-29T15:25:15.948334+0000","last_fullsized":"2024-08-29T15:25:15.948334+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":443,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813123+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:25:33.813123+0000","last_peered":"2024-08-29T15:25:33.813123+0000","last_clean":"2024-08-29T15:25:33.813123+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:25:33.813123+0000","last_undegraded":"2024-08-29T15:25:33.813123+0000","last_fullsized":"2024-08-29T15:25:33.813123+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"271'12269","reported_seq":12500,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:36.214257+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:25:36.214257+0000","last_peered":"2024-08-29T15:25:36.214257+0000","last_clean":"2024-08-29T15:25:36.214257+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:25:36.214257+0000","last_undegraded":"2024-08-29T15:25:36.214257+0000","last_fullsized":"2024-08-29T15:25:36.214257+0000","mapping_epoch":234,"log_start":"184'5597","ondisk_log_start":"184'5597","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225640448,"num_objects":3443,"num_object_clones":0,"num_object_copies":10329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12269,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":209,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.934984+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:25:26.934984+0000","last_peered":"2024-08-29T15:25:26.934984+0000","last_clean":"2024-08-29T15:25:26.934984+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:25:26.934984+0000","last_undegraded":"2024-08-29T15:25:26.934984+0000","last_fullsized":"2024-08-29T15:25:26.934984+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937711+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:25:26.937711+0000","last_peered":"2024-08-29T15:25:26.937711+0000","last_clean":"2024-08-29T15:25:26.937711+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:25:26.937711+0000","last_undegraded":"2024-08-29T15:25:26.937711+0000","last_fullsized":"2024-08-29T15:25:26.937711+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":116,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935992+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:25:26.935992+0000","last_peered":"2024-08-29T15:25:26.935992+0000","last_clean":"2024-08-29T15:25:26.935992+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:25:26.935992+0000","last_undegraded":"2024-08-29T15:25:26.935992+0000","last_fullsized":"2024-08-29T15:25:26.935992+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"270'12309","reported_seq":12539,"reported_epoch":271,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:33.810080+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:25:33.810080+0000","last_peered":"2024-08-29T15:25:33.810080+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:25:33.810080+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"179'4142","ondisk_log_start":"179'4142","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8167,"log_dups_size":0,"ondisk_log_size":8167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230490112,"num_objects":3517,"num_object_clones":0,"num_object_copies":10551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12309,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":1094}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935864+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:25:26.935864+0000","last_peered":"2024-08-29T15:25:26.935864+0000","last_clean":"2024-08-29T15:25:26.935864+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:25:26.935864+0000","last_undegraded":"2024-08-29T15:25:26.935864+0000","last_fullsized":"2024-08-29T15:25:26.935864+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":835,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.812722+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:25:33.812722+0000","last_peered":"2024-08-29T15:25:33.812722+0000","last_clean":"2024-08-29T15:25:33.812722+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:25:33.812722+0000","last_undegraded":"2024-08-29T15:25:33.812722+0000","last_fullsized":"2024-08-29T15:25:33.812722+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":412,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813350+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:25:33.813350+0000","last_peered":"2024-08-29T15:25:33.813350+0000","last_clean":"2024-08-29T15:25:33.813350+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:25:33.813350+0000","last_undegraded":"2024-08-29T15:25:33.813350+0000","last_fullsized":"2024-08-29T15:25:33.813350+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"271'12249","reported_seq":13034,"reported_epoch":271,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:35.587224+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:25:35.587224+0000","last_peered":"2024-08-29T15:25:35.587224+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:25:35.587224+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"180'5180","ondisk_log_start":"180'5180","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224067584,"num_objects":3419,"num_object_clones":0,"num_object_copies":10257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":3395,"num_objects_unfound":0,"num_objects_dirty":3419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12249,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":1268}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":417,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.812354+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:25:33.812354+0000","last_peered":"2024-08-29T15:25:33.812354+0000","last_clean":"2024-08-29T15:25:33.812354+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:25:33.812354+0000","last_undegraded":"2024-08-29T15:25:33.812354+0000","last_fullsized":"2024-08-29T15:25:33.812354+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":698,"reported_epoch":270,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:26.956966+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:25:26.956966+0000","last_peered":"2024-08-29T15:25:26.956966+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:25:26.956966+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:25:26.956966+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":408,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810448+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:25:33.810448+0000","last_peered":"2024-08-29T15:25:33.810448+0000","last_clean":"2024-08-29T15:25:33.810448+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:25:33.810448+0000","last_undegraded":"2024-08-29T15:25:33.810448+0000","last_fullsized":"2024-08-29T15:25:33.810448+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"271'12089","reported_seq":19245,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:34.787125+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:25:34.787125+0000","last_peered":"2024-08-29T15:25:34.787125+0000","last_clean":"2024-08-29T15:25:34.787125+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:25:34.787125+0000","last_undegraded":"2024-08-29T15:25:34.787125+0000","last_fullsized":"2024-08-29T15:25:34.787125+0000","mapping_epoch":234,"log_start":"179'4099","ondisk_log_start":"179'4099","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7990,"log_dups_size":0,"ondisk_log_size":7990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218824704,"num_objects":3339,"num_object_clones":0,"num_object_copies":10017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12089,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":402,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.871221+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:25:33.871221+0000","last_peered":"2024-08-29T15:25:33.871221+0000","last_clean":"2024-08-29T15:25:33.871221+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:25:33.871221+0000","last_undegraded":"2024-08-29T15:25:33.871221+0000","last_fullsized":"2024-08-29T15:25:33.871221+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1025,"reported_epoch":271,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:33.812626+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:25:33.812626+0000","last_peered":"2024-08-29T15:25:33.812626+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:25:33.812626+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":400,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.867008+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:25:33.867008+0000","last_peered":"2024-08-29T15:25:33.867008+0000","last_clean":"2024-08-29T15:25:33.867008+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:25:33.867008+0000","last_undegraded":"2024-08-29T15:25:33.867008+0000","last_fullsized":"2024-08-29T15:25:33.867008+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"270'12165","reported_seq":18819,"reported_epoch":270,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:33.526302+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:25:33.526302+0000","last_peered":"2024-08-29T15:25:33.526302+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:25:33.526302+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"180'4612","ondisk_log_start":"180'4612","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225771520,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":4479,"num_objects_missing":4479,"num_objects_degraded":2313,"num_objects_misplaced":3430,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12165,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3241,"num_bytes_recovered":212271104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":15},{"shards":"1,2,7","objects":1059},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":398,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810709+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:25:33.810709+0000","last_peered":"2024-08-29T15:25:33.810709+0000","last_clean":"2024-08-29T15:25:33.810709+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:25:33.810709+0000","last_undegraded":"2024-08-29T15:25:33.810709+0000","last_fullsized":"2024-08-29T15:25:33.810709+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":121,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.949035+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:25:15.949035+0000","last_peered":"2024-08-29T15:25:15.949035+0000","last_clean":"2024-08-29T15:25:15.949035+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:25:15.949035+0000","last_undegraded":"2024-08-29T15:25:15.949035+0000","last_fullsized":"2024-08-29T15:25:15.949035+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937231+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:25:26.937231+0000","last_peered":"2024-08-29T15:25:26.937231+0000","last_clean":"2024-08-29T15:25:26.937231+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:25:26.937231+0000","last_undegraded":"2024-08-29T15:25:26.937231+0000","last_fullsized":"2024-08-29T15:25:26.937231+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937890+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:25:26.937890+0000","last_peered":"2024-08-29T15:25:26.937890+0000","last_clean":"2024-08-29T15:25:26.937890+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:25:26.937890+0000","last_undegraded":"2024-08-29T15:25:26.937890+0000","last_fullsized":"2024-08-29T15:25:26.937890+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":48,"reported_epoch":270,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:26.937946+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:25:26.937946+0000","last_peered":"2024-08-29T15:25:26.937946+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:25:26.937946+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:25:26.937946+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":406,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813020+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:25:33.813020+0000","last_peered":"2024-08-29T15:25:33.813020+0000","last_clean":"2024-08-29T15:25:33.813020+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:25:33.813020+0000","last_undegraded":"2024-08-29T15:25:33.813020+0000","last_fullsized":"2024-08-29T15:25:33.813020+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":157,"reported_epoch":271,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:33.867681+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:25:33.867681+0000","last_peered":"2024-08-29T15:25:33.867681+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:25:33.867681+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:25:33.867681+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":139,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935448+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:25:26.935448+0000","last_peered":"2024-08-29T15:25:26.935448+0000","last_clean":"2024-08-29T15:25:26.935448+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:25:26.935448+0000","last_undegraded":"2024-08-29T15:25:26.935448+0000","last_fullsized":"2024-08-29T15:25:26.935448+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":366,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.866893+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:25:33.866893+0000","last_peered":"2024-08-29T15:25:33.866893+0000","last_clean":"2024-08-29T15:25:33.866893+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:25:33.866893+0000","last_undegraded":"2024-08-29T15:25:33.866893+0000","last_fullsized":"2024-08-29T15:25:33.866893+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":419,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.811190+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:25:33.811190+0000","last_peered":"2024-08-29T15:25:33.811190+0000","last_clean":"2024-08-29T15:25:33.811190+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:25:33.811190+0000","last_undegraded":"2024-08-29T15:25:33.811190+0000","last_fullsized":"2024-08-29T15:25:33.811190+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":418,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.866920+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:25:33.866920+0000","last_peered":"2024-08-29T15:25:33.866920+0000","last_clean":"2024-08-29T15:25:33.866920+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:25:33.866920+0000","last_undegraded":"2024-08-29T15:25:33.866920+0000","last_fullsized":"2024-08-29T15:25:33.866920+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":269,"state":"active+clean","last_fresh":"2024-08-29T15:25:15.949106+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:25:15.949106+0000","last_peered":"2024-08-29T15:25:15.949106+0000","last_clean":"2024-08-29T15:25:15.949106+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:25:15.949106+0000","last_undegraded":"2024-08-29T15:25:15.949106+0000","last_fullsized":"2024-08-29T15:25:15.949106+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":274,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810252+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:25:33.810252+0000","last_peered":"2024-08-29T15:25:33.810252+0000","last_clean":"2024-08-29T15:25:33.810252+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:25:33.810252+0000","last_undegraded":"2024-08-29T15:25:33.810252+0000","last_fullsized":"2024-08-29T15:25:33.810252+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":395,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810945+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:25:33.810945+0000","last_peered":"2024-08-29T15:25:33.810945+0000","last_clean":"2024-08-29T15:25:33.810945+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:25:33.810945+0000","last_undegraded":"2024-08-29T15:25:33.810945+0000","last_fullsized":"2024-08-29T15:25:33.810945+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":398,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.812151+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:25:33.812151+0000","last_peered":"2024-08-29T15:25:33.812151+0000","last_clean":"2024-08-29T15:25:33.812151+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:25:33.812151+0000","last_undegraded":"2024-08-29T15:25:33.812151+0000","last_fullsized":"2024-08-29T15:25:33.812151+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":402,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.867546+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:25:33.867546+0000","last_peered":"2024-08-29T15:25:33.867546+0000","last_clean":"2024-08-29T15:25:33.867546+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:25:33.867546+0000","last_undegraded":"2024-08-29T15:25:33.867546+0000","last_fullsized":"2024-08-29T15:25:33.867546+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937273+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:25:26.937273+0000","last_peered":"2024-08-29T15:25:26.937273+0000","last_clean":"2024-08-29T15:25:26.937273+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:25:26.937273+0000","last_undegraded":"2024-08-29T15:25:26.937273+0000","last_fullsized":"2024-08-29T15:25:26.937273+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":422,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.812827+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:25:33.812827+0000","last_peered":"2024-08-29T15:25:33.812827+0000","last_clean":"2024-08-29T15:25:33.812827+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:25:33.812827+0000","last_undegraded":"2024-08-29T15:25:33.812827+0000","last_fullsized":"2024-08-29T15:25:33.812827+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":386,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.810348+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:25:33.810348+0000","last_peered":"2024-08-29T15:25:33.810348+0000","last_clean":"2024-08-29T15:25:33.810348+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:25:33.810348+0000","last_undegraded":"2024-08-29T15:25:33.810348+0000","last_fullsized":"2024-08-29T15:25:33.810348+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937629+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:25:26.937629+0000","last_peered":"2024-08-29T15:25:26.937629+0000","last_clean":"2024-08-29T15:25:26.937629+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:25:26.937629+0000","last_undegraded":"2024-08-29T15:25:26.937629+0000","last_fullsized":"2024-08-29T15:25:26.937629+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":163,"reported_epoch":270,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:26.935330+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:25:26.935330+0000","last_peered":"2024-08-29T15:25:26.935330+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:25:26.935330+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:25:26.935330+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937722+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:25:26.937722+0000","last_peered":"2024-08-29T15:25:26.937722+0000","last_clean":"2024-08-29T15:25:26.937722+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:25:26.937722+0000","last_undegraded":"2024-08-29T15:25:26.937722+0000","last_fullsized":"2024-08-29T15:25:26.937722+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":169,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.935950+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:25:26.935950+0000","last_peered":"2024-08-29T15:25:26.935950+0000","last_clean":"2024-08-29T15:25:26.935950+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:25:26.935950+0000","last_undegraded":"2024-08-29T15:25:26.935950+0000","last_fullsized":"2024-08-29T15:25:26.935950+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":433,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.867063+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:25:33.867063+0000","last_peered":"2024-08-29T15:25:33.867063+0000","last_clean":"2024-08-29T15:25:33.867063+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:25:33.867063+0000","last_undegraded":"2024-08-29T15:25:33.867063+0000","last_fullsized":"2024-08-29T15:25:33.867063+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":435,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.867924+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:25:33.867924+0000","last_peered":"2024-08-29T15:25:33.867924+0000","last_clean":"2024-08-29T15:25:33.867924+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:25:33.867924+0000","last_undegraded":"2024-08-29T15:25:33.867924+0000","last_fullsized":"2024-08-29T15:25:33.867924+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":50,"reported_epoch":270,"state":"active+clean","last_fresh":"2024-08-29T15:25:26.937010+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:25:26.937010+0000","last_peered":"2024-08-29T15:25:26.937010+0000","last_clean":"2024-08-29T15:25:26.937010+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:25:26.937010+0000","last_undegraded":"2024-08-29T15:25:26.937010+0000","last_fullsized":"2024-08-29T15:25:26.937010+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":442,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.811107+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:25:33.811107+0000","last_peered":"2024-08-29T15:25:33.811107+0000","last_clean":"2024-08-29T15:25:33.811107+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:25:33.811107+0000","last_undegraded":"2024-08-29T15:25:33.811107+0000","last_fullsized":"2024-08-29T15:25:33.811107+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3640066072,"num_objects":55544,"num_object_clones":0,"num_object_copies":166632,"num_objects_missing_on_primary":4479,"num_objects_missing":4479,"num_objects_degraded":29059,"num_objects_misplaced":10345,"num_objects_unfound":0,"num_objects_dirty":55544,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":195306,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17882,"num_bytes_recovered":1051918336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11044597760,"data_stored":11044585544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124183,"ondisk_log_size":124183,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234371,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2101640,"kb_used_data":1785856,"kb_used_omap":1,"kb_used_meta":315774,"kb_avail":91643512,"statfs":{"total":95995035648,"available":93842956288,"internally_reserved":0,"allocated":1828716544,"data_stored":1827147355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":323353035},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496002,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":226548,"kb_used_data":55148,"kb_used_omap":1,"kb_used_meta":171390,"kb_avail":93518604,"statfs":{"total":95995035648,"available":95763050496,"internally_reserved":0,"allocated":56471552,"data_stored":54913811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":175503815},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2,1,2,0,4,5,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426589,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195836,"kb_used_data":1118576,"kb_used_omap":8,"kb_used_meta":77239,"kb_avail":92549316,"statfs":{"total":95995035648,"available":94770499584,"internally_reserved":0,"allocated":1145421824,"data_stored":1143863901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":79093642},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019281,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1445164,"kb_used_data":1121184,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":92299988,"statfs":{"total":95995035648,"available":94515187712,"internally_reserved":0,"allocated":1148092416,"data_stored":1146550276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435466,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2522868,"kb_used_data":2220524,"kb_used_omap":4,"kb_used_meta":302331,"kb_avail":91222284,"statfs":{"total":95995035648,"available":93411618816,"internally_reserved":0,"allocated":2273816576,"data_stored":2272262593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":309587209},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903317,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2218088,"kb_used_data":2090528,"kb_used_omap":10,"kb_used_meta":127541,"kb_avail":91527064,"statfs":{"total":95995035648,"available":93723713536,"internally_reserved":0,"allocated":2140700672,"data_stored":2139154273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":130602512},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762519,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1820532,"kb_used_data":1541740,"kb_used_omap":7,"kb_used_meta":278776,"kb_avail":91924620,"statfs":{"total":95995035648,"available":94130810880,"internally_reserved":0,"allocated":1578741760,"data_stored":1577186537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":285466978},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926045,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1227904,"kb_used_data":907444,"kb_used_omap":7,"kb_used_meta":320440,"kb_avail":92517248,"statfs":{"total":95995035648,"available":94737661952,"internally_reserved":0,"allocated":929222656,"data_stored":927660212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":328130703},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":920588288,"data_stored":920584216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1567625216,"data_stored":1567621144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2136276992,"data_stored":2136276992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2267283456,"data_stored":2267283456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1140785152,"data_stored":1140785152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1139740672,"data_stored":1139736600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":51970048,"data_stored":51970048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1820327936,"data_stored":1820327936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:25:37.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:25:37.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:37.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:37.634 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:37.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:37.685 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:37.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:37.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:37.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:37.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:37.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:37.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:38.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:25:38.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:38.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:37 smithi027 ceph-mon[98970]: pgmap v908: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 29088/166716 objects degraded (17.448%); 10345/166716 objects misplaced (6.205%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:38.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:37 smithi027 ceph-mon[98970]: from='client.45863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:38.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29088/166716 objects degraded (17.448%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:38.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:37 smithi027 ceph-mon[93921]: pgmap v908: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 29088/166716 objects degraded (17.448%); 10345/166716 objects misplaced (6.205%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:38.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:37 smithi027 ceph-mon[93921]: from='client.45863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:38.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29088/166716 objects degraded (17.448%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:38.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:38.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:38.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:38.190 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:37 smithi171 ceph-mon[90031]: pgmap v908: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 29088/166716 objects degraded (17.448%); 10345/166716 objects misplaced (6.205%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:38.190 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:37 smithi171 ceph-mon[90031]: from='client.45863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:38.190 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29088/166716 objects degraded (17.448%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:38.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:38.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:38.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:38.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:38.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:38.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:38.539 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:38.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:38.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:38.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:38.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:38.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:38.804 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:25:38.804 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:25:38.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:38.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:38.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:38.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:38.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:38.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:39.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:39.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:39.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:39.223 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:25:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:25:39] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-08-29T15:25:39.325 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:39.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:39.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:39.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:39.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:39.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:39.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:25:39.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.598 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:39.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:39.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:39.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:39.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:39.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:39.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:39.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:39.977 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:39.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:40.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:40.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:40.207 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:40.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:40.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:39 smithi027 ceph-mon[93921]: pgmap v909: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 189 op/s; 29088/166545 objects degraded (17.466%); 10290/166545 objects misplaced (6.179%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:40.249 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:39 smithi027 ceph-mon[98970]: pgmap v909: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 189 op/s; 29088/166545 objects degraded (17.466%); 10290/166545 objects misplaced (6.179%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:39 smithi171 ceph-mon[90031]: pgmap v909: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 189 op/s; 29088/166545 objects degraded (17.466%); 10290/166545 objects misplaced (6.179%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:40.291 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:40.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:40.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:40.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:40.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:40.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:40.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:40.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:40.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:40.735 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:40.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:40.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:40.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:40.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:40.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:40.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:41.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:41.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:41.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:41.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:41.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:41.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:41.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:41.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:41.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:41.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:41.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:41.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:41.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:41.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:41.660 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:41.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:41.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:41.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:41.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:41.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:41.886 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:41.886 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:41.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:41.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:41.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:42.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:42.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:41 smithi027 ceph-mon[98970]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 29088/165741 objects degraded (17.550%); 10290/165741 objects misplaced (6.208%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:41 smithi027 ceph-mon[93921]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 29088/165741 objects degraded (17.550%); 10290/165741 objects misplaced (6.208%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:42.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:42.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:42.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:42.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:41 smithi171 ceph-mon[90031]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 29088/165741 objects degraded (17.550%); 10290/165741 objects misplaced (6.208%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:42.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:42.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:42.424 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:42.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:42.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:42.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:42.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:42.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:42.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:42.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:42.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:42.841 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:25:42.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:42.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:42.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:42.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:42.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29088/165741 objects degraded (17.550%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:42.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3568344211' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:42.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:42 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:42.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29088/165741 objects degraded (17.550%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:42.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3568344211' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:42.988 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:42 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:43.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:43.029 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:25:43.029 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 29015/163533 objects degraded (17.743%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1324,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":272,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":54511,"data_bytes":3575322621,"bytes_used":12836347904,"bytes_avail":755123937280,"bytes_total":767960285184,"degraded_objects":29015,"degraded_total":163533,"degraded_ratio":0.1774259629554891,"misplaced_objects":10155,"misplaced_total":163533,"misplaced_ratio":0.062097558290987148,"recovering_objects_per_sec":23,"recovering_bytes_per_sec":1513453,"recovering_keys_per_sec":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":188},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:25:43.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:43.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:43.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:43.167 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:43.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29088/165741 objects degraded (17.550%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3568344211' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:42 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:25:43.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:43.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:43.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:43.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:43.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:43.543 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:43.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:43.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:43.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:43.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:43.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:43.791 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:43.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:43.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:43.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:43.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:44.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:44.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:44.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:44.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:44.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:44.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:44.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:43 smithi171 ceph-mon[90031]: pgmap v911: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 29015/163533 objects degraded (17.743%); 10155/163533 objects misplaced (6.210%); 1.4 MiB/s, 23 objects/s recovering 2024-08-29T15:25:44.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:43 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:25:44.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:44.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:43 smithi171 ceph-mon[90031]: osdmap e272: 8 total, 8 up, 8 in 2024-08-29T15:25:44.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1957742640' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:25:44.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:44.201 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:44.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:44.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[98970]: pgmap v911: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 29015/163533 objects degraded (17.743%); 10155/163533 objects misplaced (6.210%); 1.4 MiB/s, 23 objects/s recovering 2024-08-29T15:25:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:25:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[98970]: osdmap e272: 8 total, 8 up, 8 in 2024-08-29T15:25:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1957742640' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[93921]: pgmap v911: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 29015/163533 objects degraded (17.743%); 10155/163533 objects misplaced (6.210%); 1.4 MiB/s, 23 objects/s recovering 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[93921]: osdmap e272: 8 total, 8 up, 8 in 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1957742640' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:44.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:44.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:44.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:44.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:44.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:44.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:44.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:44.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:44.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:44.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:44.799 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:44.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:44.874 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:25:44.874 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:25:44.897 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:25:44.897 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:25:44.901 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:44.915 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:25:44.916 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:25:44.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:44.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:44.933 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:25:44.934 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:25:44.950 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:25:44.950 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:25:44.968 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:25:44.968 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:25:44.984 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:25:44.984 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:25:45.001 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:25:45.001 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:25:45.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:45.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:45.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:45.150 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:45.171 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:45.273 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:45.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:45.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:45.418 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:45.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:45.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:45.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:45.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:45.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:45.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:45.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:45.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:45.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:45.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:45.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:45.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:45.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:45.924 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:45 smithi027 ceph-mon[98970]: pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 29015/162900 objects degraded (17.812%); 10092/162900 objects misplaced (6.195%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:45.924 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/990550850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:45.924 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:45 smithi027 ceph-mon[93921]: pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 29015/162900 objects degraded (17.812%); 10092/162900 objects misplaced (6.195%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:45.924 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/990550850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:46.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:46.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:46.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:46.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:46.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:46.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:45 smithi171 ceph-mon[90031]: pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 29015/162900 objects degraded (17.812%); 10092/162900 objects misplaced (6.195%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:25:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/990550850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:46.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:46.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:46.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:46.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:25:46.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:46.433 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:25:46.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:25:46.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:46.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:46.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:46.673 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:46.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:46.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:46.787 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:197562 6.2 e272/234 6:4ed06ef9:::benchmark_data_smithi027_169445_object72135:head v 272'12258, mlcod=272'12256)", 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:25:46.785678+0000", 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00051858199999999996, 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00054442800000000001, 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:25:46.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785678+0000", 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785678+0000", 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785680+0000", 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.519e-06 2024-08-29T15:25:46.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785696+0000", 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5611e-05 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785721+0000", 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5040000000000001e-05 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785732+0000", 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1019000000000001e-05 2024-08-29T15:25:46.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785810+0000", 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8066999999999996e-05 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:46.785858+0000", 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8242000000000003e-05 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:25:46.791 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:25:46.792 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2024-08-29T15:25:46.792 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:25:46.810 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:46.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:46.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:46.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:46.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:47.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:47.059 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:47.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:47.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:47.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:47.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:47.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:47.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:47.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:47.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:47.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:47.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:47.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:47.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:47.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:47.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:47.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:47.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:47.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:47.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:47.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:47.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:47.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:48.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:48.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:25:48.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:48.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:48.146 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:48.146 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:48.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:48.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:47 smithi171 ceph-mon[90031]: pgmap v914: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28902/160644 objects degraded (17.991%); 10035/160644 objects misplaced (6.247%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28902/160644 objects degraded (17.991%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:48.276 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:47 smithi027 ceph-mon[93921]: pgmap v914: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28902/160644 objects degraded (17.991%); 10035/160644 objects misplaced (6.247%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:48.276 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28902/160644 objects degraded (17.991%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:48.277 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:47 smithi027 ceph-mon[98970]: pgmap v914: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28902/160644 objects degraded (17.991%); 10035/160644 objects misplaced (6.247%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:48.277 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28902/160644 objects degraded (17.991%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:48.279 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:48.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:48.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:48.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:48.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:48.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:48.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:48.654 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:48.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:48.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:48.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:48.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:48.926 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:48.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:49.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:49.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:49.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:49.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:49.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.180 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:49.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:49.270 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:25:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:25:49] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-08-29T15:25:49.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:49.304 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:49.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:49.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:49.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:49.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:49.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:25:49.536 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:49.559 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:49.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:49.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.684 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:49.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:49.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:49.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:49.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:49.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:49.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:49.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:49.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:50.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:50.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:50.157 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:49 smithi027 ceph-mon[98970]: pgmap v915: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 28902/160644 objects degraded (17.991%); 10035/160644 objects misplaced (6.247%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:25:50.158 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:49 smithi027 ceph-mon[93921]: pgmap v915: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 28902/160644 objects degraded (17.991%); 10035/160644 objects misplaced (6.247%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:25:50.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:50.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:49 smithi171 ceph-mon[90031]: pgmap v915: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 196 op/s; 28902/160644 objects degraded (17.991%); 10035/160644 objects misplaced (6.247%); 1.2 MiB/s, 18 objects/s recovering 2024-08-29T15:25:50.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:50.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:50.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:50.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:50.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:50.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:50.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:50.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:50.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:50.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:50.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:50.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:50.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:50.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:50.853 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:25:50.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:50.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:50.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:51.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:51.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:51.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:51.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3983699008' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:51.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:50 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:51.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3983699008' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:51.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:50 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:51.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:51.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:51.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3983699008' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:50 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:25:51.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:51.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:51.413 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:51.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:51.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:51.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:51.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:51.684 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:51.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:51.786 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:51.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:51.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:51.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:51.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:51.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:51.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:52.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:52.040 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:52.041 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:52.068 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:52.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:52.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[93921]: pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 28902/159717 objects degraded (18.096%); 9971/159717 objects misplaced (6.243%); 1.6 MiB/s, 24 objects/s recovering 2024-08-29T15:25:52.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:52.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:25:52.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[93921]: osdmap e273: 8 total, 8 up, 8 in 2024-08-29T15:25:52.139 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[98970]: pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 28902/159717 objects degraded (18.096%); 9971/159717 objects misplaced (6.243%); 1.6 MiB/s, 24 objects/s recovering 2024-08-29T15:25:52.139 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:52.139 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:25:52.140 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:51 smithi027 ceph-mon[98970]: osdmap e273: 8 total, 8 up, 8 in 2024-08-29T15:25:52.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:52.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:52.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:51 smithi171 ceph-mon[90031]: pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 28902/159717 objects degraded (18.096%); 9971/159717 objects misplaced (6.243%); 1.6 MiB/s, 24 objects/s recovering 2024-08-29T15:25:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:51 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:25:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:51 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:25:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:51 smithi171 ceph-mon[90031]: osdmap e273: 8 total, 8 up, 8 in 2024-08-29T15:25:52.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:52.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:52.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:52.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:52.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:52.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:52.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:52.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:52.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:52.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:52.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:52.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:52.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:52.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:52.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:52.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:52.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:52.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:53.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:53.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:53.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28902/159717 objects degraded (18.096%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:53.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28902/159717 objects degraded (18.096%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:53.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:53.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:53.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28902/159717 objects degraded (18.096%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:53.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:53.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:53.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:53.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:53.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:53.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:53.562 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:53.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:53.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:53.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:53.812 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:53.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:53.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:53.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:53.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:54.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:25:54.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:54.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:54.166 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:54.189 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:54.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:53 smithi171 ceph-mon[90031]: pgmap v918: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 207 op/s; 28778/157443 objects degraded (18.278%); 9907/157443 objects misplaced (6.292%); 1.6 MiB/s, 24 objects/s recovering 2024-08-29T15:25:54.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:54.311 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:53 smithi027 ceph-mon[93921]: pgmap v918: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 207 op/s; 28778/157443 objects degraded (18.278%); 9907/157443 objects misplaced (6.292%); 1.6 MiB/s, 24 objects/s recovering 2024-08-29T15:25:54.311 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:53 smithi027 ceph-mon[98970]: pgmap v918: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 207 op/s; 28778/157443 objects degraded (18.278%); 9907/157443 objects misplaced (6.292%); 1.6 MiB/s, 24 objects/s recovering 2024-08-29T15:25:54.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:54.391 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:25:54.392 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:25:54.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:54.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:54.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:54.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:54.668 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:54.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:54.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:54.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:54.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:54.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:54.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:54.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:54.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:55.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:55.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:55.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:55.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:55.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:54 smithi171 ceph-mon[90031]: from='client.35973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:55.287 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:54 smithi027 ceph-mon[98970]: from='client.35973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:55.288 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:54 smithi027 ceph-mon[93921]: from='client.35973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:25:55.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:55.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:55.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:55.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:55.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:55.553 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:55.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:55.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:55.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:55.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:55.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:55.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:55.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:55.932 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":918,"stamp":"2024-08-29T15:25:52.639145+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3442284541,"num_objects":52481,"num_object_clones":0,"num_object_copies":157443,"num_objects_missing_on_primary":4305,"num_objects_missing":4305,"num_objects_degraded":28778,"num_objects_misplaced":9907,"num_objects_unfound":0,"num_objects_dirty":52481,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":199520,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18362,"num_bytes_recovered":1082199245,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125156,"ondisk_log_size":125156,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12203932,"kb_used_data":10247704,"kb_used_omap":43,"kb_used_meta":1955988,"kb_avail":737757284,"statfs":{"total":767960285184,"available":755463458816,"internally_reserved":0,"allocated":10493648896,"data_stored":10481119726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2002932204},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-133038080,"num_objects":-2030,"num_object_clones":0,"num_object_copies":-6090,"num_objects_missing_on_primary":-127,"num_objects_missing":-127,"num_objects_degraded":-237,"num_objects_misplaced":-248,"num_objects_unfound":0,"num_objects_dirty":-2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2030,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":245,"num_bytes_recovered":16056320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.803264"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":432,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852993+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:25:50.852993+0000","last_peered":"2024-08-29T15:25:50.852993+0000","last_clean":"2024-08-29T15:25:50.852993+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:25:50.852993+0000","last_undegraded":"2024-08-29T15:25:50.852993+0000","last_fullsized":"2024-08-29T15:25:50.852993+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":50,"reported_epoch":271,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:33.814354+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:25:33.814354+0000","last_peered":"2024-08-29T15:25:33.814354+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:25:33.814354+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:25:33.814354+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":141,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837187+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:25:42.837187+0000","last_peered":"2024-08-29T15:25:42.837187+0000","last_clean":"2024-08-29T15:25:42.837187+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:25:42.837187+0000","last_undegraded":"2024-08-29T15:25:42.837187+0000","last_fullsized":"2024-08-29T15:25:42.837187+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":395,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.861033+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:25:42.861033+0000","last_peered":"2024-08-29T15:25:42.861033+0000","last_clean":"2024-08-29T15:25:42.861033+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:25:42.861033+0000","last_undegraded":"2024-08-29T15:25:42.861033+0000","last_fullsized":"2024-08-29T15:25:42.861033+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":395,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860238+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:25:42.860238+0000","last_peered":"2024-08-29T15:25:42.860238+0000","last_clean":"2024-08-29T15:25:42.860238+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:25:42.860238+0000","last_undegraded":"2024-08-29T15:25:42.860238+0000","last_fullsized":"2024-08-29T15:25:42.860238+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813617+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:25:33.813617+0000","last_peered":"2024-08-29T15:25:33.813617+0000","last_clean":"2024-08-29T15:25:33.813617+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:25:33.813617+0000","last_undegraded":"2024-08-29T15:25:33.813617+0000","last_fullsized":"2024-08-29T15:25:33.813617+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":207,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844836+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:25:42.844836+0000","last_peered":"2024-08-29T15:25:42.844836+0000","last_clean":"2024-08-29T15:25:42.844836+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:25:42.844836+0000","last_undegraded":"2024-08-29T15:25:42.844836+0000","last_fullsized":"2024-08-29T15:25:42.844836+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":141,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.896925+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:25:33.896925+0000","last_peered":"2024-08-29T15:25:33.896925+0000","last_clean":"2024-08-29T15:25:33.896925+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:25:33.896925+0000","last_undegraded":"2024-08-29T15:25:33.896925+0000","last_fullsized":"2024-08-29T15:25:33.896925+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"273'12546","reported_seq":20598,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:51.064371+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:25:51.064371+0000","last_peered":"2024-08-29T15:25:51.064371+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:25:51.064371+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"177'2500","ondisk_log_start":"177'2500","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220594176,"num_objects":3366,"num_object_clones":0,"num_object_copies":10098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12546,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":441}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":49,"reported_epoch":271,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:33.814493+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:25:33.814493+0000","last_peered":"2024-08-29T15:25:33.814493+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:25:33.814493+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:25:33.814493+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837025+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:25:42.837025+0000","last_peered":"2024-08-29T15:25:42.837025+0000","last_clean":"2024-08-29T15:25:42.837025+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:25:42.837025+0000","last_undegraded":"2024-08-29T15:25:42.837025+0000","last_fullsized":"2024-08-29T15:25:42.837025+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"273'12304","reported_seq":12529,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:51.436800+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:25:51.436800+0000","last_peered":"2024-08-29T15:25:51.436800+0000","last_clean":"2024-08-29T15:25:51.436800+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:25:51.436800+0000","last_undegraded":"2024-08-29T15:25:51.436800+0000","last_fullsized":"2024-08-29T15:25:51.436800+0000","mapping_epoch":168,"log_start":"186'5628","ondisk_log_start":"186'5628","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212729856,"num_objects":3246,"num_object_clones":0,"num_object_copies":9738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12304,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":49,"reported_epoch":271,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:33.817633+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:25:33.817633+0000","last_peered":"2024-08-29T15:25:33.817633+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:25:33.817633+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:25:33.817633+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813486+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:25:33.813486+0000","last_peered":"2024-08-29T15:25:33.813486+0000","last_clean":"2024-08-29T15:25:33.813486+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:25:33.813486+0000","last_undegraded":"2024-08-29T15:25:33.813486+0000","last_fullsized":"2024-08-29T15:25:33.813486+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"273'12451","reported_seq":12646,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:51.338269+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:25:51.338269+0000","last_peered":"2024-08-29T15:25:51.338269+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:25:51.338269+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"186'5774","ondisk_log_start":"186'5774","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6677,"log_dups_size":0,"ondisk_log_size":6677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213712896,"num_objects":3261,"num_object_clones":0,"num_object_copies":9783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12451,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":784}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":212,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.863019+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:25:42.863019+0000","last_peered":"2024-08-29T15:25:42.863019+0000","last_clean":"2024-08-29T15:25:42.863019+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:25:42.863019+0000","last_undegraded":"2024-08-29T15:25:42.863019+0000","last_fullsized":"2024-08-29T15:25:42.863019+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":97,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.863207+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:25:42.863207+0000","last_peered":"2024-08-29T15:25:42.863207+0000","last_clean":"2024-08-29T15:25:42.863207+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:25:42.863207+0000","last_undegraded":"2024-08-29T15:25:42.863207+0000","last_fullsized":"2024-08-29T15:25:42.863207+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"272'12446","reported_seq":17889,"reported_epoch":272,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:48.836574+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:25:48.836574+0000","last_peered":"2024-08-29T15:25:48.836574+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:25:48.836574+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"179'4262","ondisk_log_start":"179'4262","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8184,"log_dups_size":0,"ondisk_log_size":8184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218365952,"num_objects":3332,"num_object_clones":0,"num_object_copies":9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12446,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3332}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.838643+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:25:42.838643+0000","last_peered":"2024-08-29T15:25:42.838643+0000","last_clean":"2024-08-29T15:25:42.838643+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:25:42.838643+0000","last_undegraded":"2024-08-29T15:25:42.838643+0000","last_fullsized":"2024-08-29T15:25:42.838643+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":410,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860099+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:25:42.860099+0000","last_peered":"2024-08-29T15:25:42.860099+0000","last_clean":"2024-08-29T15:25:42.860099+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:25:42.860099+0000","last_undegraded":"2024-08-29T15:25:42.860099+0000","last_fullsized":"2024-08-29T15:25:42.860099+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"272'12456","reported_seq":12615,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:49.645840+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:25:49.645840+0000","last_peered":"2024-08-29T15:25:49.645840+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:25:49.645840+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:25:49.645840+0000","mapping_epoch":234,"log_start":"180'4533","ondisk_log_start":"180'4533","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7923,"log_dups_size":0,"ondisk_log_size":7923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220069888,"num_objects":3358,"num_object_clones":0,"num_object_copies":10074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12456,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":881},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":406,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.851319+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:25:50.851319+0000","last_peered":"2024-08-29T15:25:50.851319+0000","last_clean":"2024-08-29T15:25:50.851319+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:25:50.851319+0000","last_undegraded":"2024-08-29T15:25:50.851319+0000","last_fullsized":"2024-08-29T15:25:50.851319+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813933+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:25:33.813933+0000","last_peered":"2024-08-29T15:25:33.813933+0000","last_clean":"2024-08-29T15:25:33.813933+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:25:33.813933+0000","last_undegraded":"2024-08-29T15:25:33.813933+0000","last_fullsized":"2024-08-29T15:25:33.813933+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"273'12488","reported_seq":12730,"reported_epoch":273,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:51.165470+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:25:51.165470+0000","last_peered":"2024-08-29T15:25:51.165470+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:25:51.165470+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:25:51.165470+0000","mapping_epoch":234,"log_start":"186'5768","ondisk_log_start":"186'5768","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219807744,"num_objects":3354,"num_object_clones":0,"num_object_copies":10062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12488,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":889},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":115,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.863132+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:25:42.863132+0000","last_peered":"2024-08-29T15:25:42.863132+0000","last_clean":"2024-08-29T15:25:42.863132+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:25:42.863132+0000","last_undegraded":"2024-08-29T15:25:42.863132+0000","last_fullsized":"2024-08-29T15:25:42.863132+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.814187+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:25:33.814187+0000","last_peered":"2024-08-29T15:25:33.814187+0000","last_clean":"2024-08-29T15:25:33.814187+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:25:33.814187+0000","last_undegraded":"2024-08-29T15:25:33.814187+0000","last_fullsized":"2024-08-29T15:25:33.814187+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"273'12334","reported_seq":12760,"reported_epoch":273,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:25:51.367031+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:25:51.367031+0000","last_peered":"2024-08-29T15:25:51.367031+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:25:51.367031+0000","last_undegraded":"2024-08-29T15:25:51.367031+0000","last_fullsized":"2024-08-29T15:25:51.367031+0000","mapping_epoch":202,"log_start":"185'5667","ondisk_log_start":"185'5667","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6667,"log_dups_size":0,"ondisk_log_size":6667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217841664,"num_objects":3324,"num_object_clones":0,"num_object_copies":9972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3324,"num_objects_unfound":0,"num_objects_dirty":3324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12334,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":3324}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813514+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:25:33.813514+0000","last_peered":"2024-08-29T15:25:33.813514+0000","last_clean":"2024-08-29T15:25:33.813514+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:25:33.813514+0000","last_undegraded":"2024-08-29T15:25:33.813514+0000","last_fullsized":"2024-08-29T15:25:33.813514+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":257,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.817781+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:25:33.817781+0000","last_peered":"2024-08-29T15:25:33.817781+0000","last_clean":"2024-08-29T15:25:33.817781+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:25:33.817781+0000","last_undegraded":"2024-08-29T15:25:33.817781+0000","last_fullsized":"2024-08-29T15:25:33.817781+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"273'12521","reported_seq":12675,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:51.177912+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:25:51.177912+0000","last_peered":"2024-08-29T15:25:51.177912+0000","last_clean":"2024-08-29T15:25:51.177912+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:25:51.177912+0000","last_undegraded":"2024-08-29T15:25:51.177912+0000","last_fullsized":"2024-08-29T15:25:51.177912+0000","mapping_epoch":218,"log_start":"177'2478","ondisk_log_start":"177'2478","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10043,"log_dups_size":0,"ondisk_log_size":10043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212205592,"num_objects":3239,"num_object_clones":0,"num_object_copies":9717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3239,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12521,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":409,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852613+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:25:50.852613+0000","last_peered":"2024-08-29T15:25:50.852613+0000","last_clean":"2024-08-29T15:25:50.852613+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:25:50.852613+0000","last_undegraded":"2024-08-29T15:25:50.852613+0000","last_fullsized":"2024-08-29T15:25:50.852613+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":400,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844961+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:25:42.844961+0000","last_peered":"2024-08-29T15:25:42.844961+0000","last_clean":"2024-08-29T15:25:42.844961+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:25:42.844961+0000","last_undegraded":"2024-08-29T15:25:42.844961+0000","last_fullsized":"2024-08-29T15:25:42.844961+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"273'11944","reported_seq":12146,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:51.107698+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:25:51.107698+0000","last_peered":"2024-08-29T15:25:51.107698+0000","last_clean":"2024-08-29T15:25:51.107698+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:25:51.107698+0000","last_undegraded":"2024-08-29T15:25:51.107698+0000","last_fullsized":"2024-08-29T15:25:51.107698+0000","mapping_epoch":201,"log_start":"176'1900","ondisk_log_start":"176'1900","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213909504,"num_objects":3264,"num_object_clones":0,"num_object_copies":9792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11944,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":498,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.859863+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:25:42.859863+0000","last_peered":"2024-08-29T15:25:42.859863+0000","last_clean":"2024-08-29T15:25:42.859863+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:25:42.859863+0000","last_undegraded":"2024-08-29T15:25:42.859863+0000","last_fullsized":"2024-08-29T15:25:42.859863+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":743,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837384+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:25:42.837384+0000","last_peered":"2024-08-29T15:25:42.837384+0000","last_clean":"2024-08-29T15:25:42.837384+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:25:42.837384+0000","last_undegraded":"2024-08-29T15:25:42.837384+0000","last_fullsized":"2024-08-29T15:25:42.837384+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":251,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.838907+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:25:42.838907+0000","last_peered":"2024-08-29T15:25:42.838907+0000","last_clean":"2024-08-29T15:25:42.838907+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:25:42.838907+0000","last_undegraded":"2024-08-29T15:25:42.838907+0000","last_fullsized":"2024-08-29T15:25:42.838907+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.898116+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:25:33.898116+0000","last_peered":"2024-08-29T15:25:33.898116+0000","last_clean":"2024-08-29T15:25:33.898116+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:25:33.898116+0000","last_undegraded":"2024-08-29T15:25:33.898116+0000","last_fullsized":"2024-08-29T15:25:33.898116+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"272'12439","reported_seq":12861,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.314213+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:25:50.314213+0000","last_peered":"2024-08-29T15:25:50.314213+0000","last_clean":"2024-08-29T15:25:50.314213+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:25:50.314213+0000","last_undegraded":"2024-08-29T15:25:50.314213+0000","last_fullsized":"2024-08-29T15:25:50.314213+0000","mapping_epoch":201,"log_start":"181'5453","ondisk_log_start":"181'5453","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212533248,"num_objects":3243,"num_object_clones":0,"num_object_copies":9729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12439,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":872,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.843797+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:25:42.843797+0000","last_peered":"2024-08-29T15:25:42.843797+0000","last_clean":"2024-08-29T15:25:42.843797+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:25:42.843797+0000","last_undegraded":"2024-08-29T15:25:42.843797+0000","last_fullsized":"2024-08-29T15:25:42.843797+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":410,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.859973+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:25:42.859973+0000","last_peered":"2024-08-29T15:25:42.859973+0000","last_clean":"2024-08-29T15:25:42.859973+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:25:42.859973+0000","last_undegraded":"2024-08-29T15:25:42.859973+0000","last_fullsized":"2024-08-29T15:25:42.859973+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":418,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860438+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:25:42.860438+0000","last_peered":"2024-08-29T15:25:42.860438+0000","last_clean":"2024-08-29T15:25:42.860438+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:25:42.860438+0000","last_undegraded":"2024-08-29T15:25:42.860438+0000","last_fullsized":"2024-08-29T15:25:42.860438+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"273'12403","reported_seq":20652,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:51.402249+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:25:51.402249+0000","last_peered":"2024-08-29T15:25:51.402249+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:25:51.402249+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"186'5730","ondisk_log_start":"186'5730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215416832,"num_objects":3287,"num_object_clones":0,"num_object_copies":9861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3764,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12403,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4027,"num_bytes_recovered":263913472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":849},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":683,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850772+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:25:50.850772+0000","last_peered":"2024-08-29T15:25:50.850772+0000","last_clean":"2024-08-29T15:25:50.850772+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:25:50.850772+0000","last_undegraded":"2024-08-29T15:25:50.850772+0000","last_fullsized":"2024-08-29T15:25:50.850772+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":103,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.896795+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:25:33.896795+0000","last_peered":"2024-08-29T15:25:33.896795+0000","last_clean":"2024-08-29T15:25:33.896795+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:25:33.896795+0000","last_undegraded":"2024-08-29T15:25:33.896795+0000","last_fullsized":"2024-08-29T15:25:33.896795+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":445,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844694+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:25:42.844694+0000","last_peered":"2024-08-29T15:25:42.844694+0000","last_clean":"2024-08-29T15:25:42.844694+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:25:42.844694+0000","last_undegraded":"2024-08-29T15:25:42.844694+0000","last_fullsized":"2024-08-29T15:25:42.844694+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"273'12468","reported_seq":12703,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:51.374884+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:25:51.374884+0000","last_peered":"2024-08-29T15:25:51.374884+0000","last_clean":"2024-08-29T15:25:51.374884+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:25:51.374884+0000","last_undegraded":"2024-08-29T15:25:51.374884+0000","last_fullsized":"2024-08-29T15:25:51.374884+0000","mapping_epoch":234,"log_start":"186'5797","ondisk_log_start":"186'5797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6671,"log_dups_size":0,"ondisk_log_size":6671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212598784,"num_objects":3244,"num_object_clones":0,"num_object_copies":9732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12468,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":211,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.838832+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:25:42.838832+0000","last_peered":"2024-08-29T15:25:42.838832+0000","last_clean":"2024-08-29T15:25:42.838832+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:25:42.838832+0000","last_undegraded":"2024-08-29T15:25:42.838832+0000","last_fullsized":"2024-08-29T15:25:42.838832+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813763+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:25:33.813763+0000","last_peered":"2024-08-29T15:25:33.813763+0000","last_clean":"2024-08-29T15:25:33.813763+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:25:33.813763+0000","last_undegraded":"2024-08-29T15:25:33.813763+0000","last_fullsized":"2024-08-29T15:25:33.813763+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":118,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.838226+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:25:42.838226+0000","last_peered":"2024-08-29T15:25:42.838226+0000","last_clean":"2024-08-29T15:25:42.838226+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:25:42.838226+0000","last_undegraded":"2024-08-29T15:25:42.838226+0000","last_fullsized":"2024-08-29T15:25:42.838226+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"272'12513","reported_seq":12745,"reported_epoch":272,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:48.905359+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:25:48.905359+0000","last_peered":"2024-08-29T15:25:48.905359+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:25:48.905359+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"179'4342","ondisk_log_start":"179'4342","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217120768,"num_objects":3313,"num_object_clones":0,"num_object_copies":9939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12513,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":890}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837115+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:25:42.837115+0000","last_peered":"2024-08-29T15:25:42.837115+0000","last_clean":"2024-08-29T15:25:42.837115+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:25:42.837115+0000","last_undegraded":"2024-08-29T15:25:42.837115+0000","last_fullsized":"2024-08-29T15:25:42.837115+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":837,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844519+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:25:42.844519+0000","last_peered":"2024-08-29T15:25:42.844519+0000","last_clean":"2024-08-29T15:25:42.844519+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:25:42.844519+0000","last_undegraded":"2024-08-29T15:25:42.844519+0000","last_fullsized":"2024-08-29T15:25:42.844519+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":414,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844300+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:25:42.844300+0000","last_peered":"2024-08-29T15:25:42.844300+0000","last_clean":"2024-08-29T15:25:42.844300+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:25:42.844300+0000","last_undegraded":"2024-08-29T15:25:42.844300+0000","last_fullsized":"2024-08-29T15:25:42.844300+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"272'12434","reported_seq":13222,"reported_epoch":272,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:50.333299+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:25:50.333299+0000","last_peered":"2024-08-29T15:25:50.333299+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:25:50.333299+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"180'5380","ondisk_log_start":"180'5380","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211943424,"num_objects":3234,"num_object_clones":0,"num_object_copies":9702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3212,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":3234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12434,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":1083}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":419,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844425+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:25:42.844425+0000","last_peered":"2024-08-29T15:25:42.844425+0000","last_clean":"2024-08-29T15:25:42.844425+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:25:42.844425+0000","last_undegraded":"2024-08-29T15:25:42.844425+0000","last_fullsized":"2024-08-29T15:25:42.844425+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":700,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:42.838350+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:25:42.838350+0000","last_peered":"2024-08-29T15:25:42.838350+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:25:42.838350+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:25:42.838350+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":410,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860850+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:25:42.860850+0000","last_peered":"2024-08-29T15:25:42.860850+0000","last_clean":"2024-08-29T15:25:42.860850+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:25:42.860850+0000","last_undegraded":"2024-08-29T15:25:42.860850+0000","last_fullsized":"2024-08-29T15:25:42.860850+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"272'12301","reported_seq":19458,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:49.618678+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:25:49.618678+0000","last_peered":"2024-08-29T15:25:49.618678+0000","last_clean":"2024-08-29T15:25:49.618678+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:25:49.618678+0000","last_undegraded":"2024-08-29T15:25:49.618678+0000","last_fullsized":"2024-08-29T15:25:49.618678+0000","mapping_epoch":234,"log_start":"179'4399","ondisk_log_start":"179'4399","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7902,"log_dups_size":0,"ondisk_log_size":7902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204931072,"num_objects":3127,"num_object_clones":0,"num_object_copies":9381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12301,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":406,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852439+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:25:50.852439+0000","last_peered":"2024-08-29T15:25:50.852439+0000","last_clean":"2024-08-29T15:25:50.852439+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:25:50.852439+0000","last_undegraded":"2024-08-29T15:25:50.852439+0000","last_fullsized":"2024-08-29T15:25:50.852439+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1027,"reported_epoch":272,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:42.843204+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:25:42.843204+0000","last_peered":"2024-08-29T15:25:42.843204+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:25:42.843204+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":404,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850700+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:25:50.850700+0000","last_peered":"2024-08-29T15:25:50.850700+0000","last_clean":"2024-08-29T15:25:50.850700+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:25:50.850700+0000","last_undegraded":"2024-08-29T15:25:50.850700+0000","last_fullsized":"2024-08-29T15:25:50.850700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"272'12349","reported_seq":19370,"reported_epoch":272,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:48.690548+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:25:48.690548+0000","last_peered":"2024-08-29T15:25:48.690548+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:25:48.690548+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"180'4812","ondisk_log_start":"180'4812","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7537,"log_dups_size":0,"ondisk_log_size":7537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213712896,"num_objects":3261,"num_object_clones":0,"num_object_copies":9783,"num_objects_missing_on_primary":4297,"num_objects_missing":4297,"num_objects_degraded":2313,"num_objects_misplaced":3248,"num_objects_unfound":0,"num_objects_dirty":3261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12349,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3425,"num_bytes_recovered":224329728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":877},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":400,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860159+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:25:42.860159+0000","last_peered":"2024-08-29T15:25:42.860159+0000","last_clean":"2024-08-29T15:25:42.860159+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:25:42.860159+0000","last_undegraded":"2024-08-29T15:25:42.860159+0000","last_fullsized":"2024-08-29T15:25:42.860159+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":124,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:38.629457+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:25:38.629457+0000","last_peered":"2024-08-29T15:25:38.629457+0000","last_clean":"2024-08-29T15:25:38.629457+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:25:38.629457+0000","last_undegraded":"2024-08-29T15:25:38.629457+0000","last_fullsized":"2024-08-29T15:25:38.629457+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813572+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:25:33.813572+0000","last_peered":"2024-08-29T15:25:33.813572+0000","last_clean":"2024-08-29T15:25:33.813572+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:25:33.813572+0000","last_undegraded":"2024-08-29T15:25:33.813572+0000","last_fullsized":"2024-08-29T15:25:33.813572+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813864+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:25:33.813864+0000","last_peered":"2024-08-29T15:25:33.813864+0000","last_clean":"2024-08-29T15:25:33.813864+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:25:33.813864+0000","last_undegraded":"2024-08-29T15:25:33.813864+0000","last_fullsized":"2024-08-29T15:25:33.813864+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":49,"reported_epoch":271,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:33.814376+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:25:33.814376+0000","last_peered":"2024-08-29T15:25:33.814376+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:25:33.814376+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:25:33.814376+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":408,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844170+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:25:42.844170+0000","last_peered":"2024-08-29T15:25:42.844170+0000","last_clean":"2024-08-29T15:25:42.844170+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:25:42.844170+0000","last_undegraded":"2024-08-29T15:25:42.844170+0000","last_fullsized":"2024-08-29T15:25:42.844170+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":161,"reported_epoch":273,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:50.854007+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:25:50.854007+0000","last_peered":"2024-08-29T15:25:50.854007+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:25:50.854007+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:25:50.854007+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":141,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837771+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:25:42.837771+0000","last_peered":"2024-08-29T15:25:42.837771+0000","last_clean":"2024-08-29T15:25:42.837771+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:25:42.837771+0000","last_undegraded":"2024-08-29T15:25:42.837771+0000","last_fullsized":"2024-08-29T15:25:42.837771+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":370,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850515+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:25:50.850515+0000","last_peered":"2024-08-29T15:25:50.850515+0000","last_clean":"2024-08-29T15:25:50.850515+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:25:50.850515+0000","last_undegraded":"2024-08-29T15:25:50.850515+0000","last_fullsized":"2024-08-29T15:25:50.850515+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":421,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860613+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:25:42.860613+0000","last_peered":"2024-08-29T15:25:42.860613+0000","last_clean":"2024-08-29T15:25:42.860613+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:25:42.860613+0000","last_undegraded":"2024-08-29T15:25:42.860613+0000","last_fullsized":"2024-08-29T15:25:42.860613+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":422,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850615+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:25:50.850615+0000","last_peered":"2024-08-29T15:25:50.850615+0000","last_clean":"2024-08-29T15:25:50.850615+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:25:50.850615+0000","last_undegraded":"2024-08-29T15:25:50.850615+0000","last_fullsized":"2024-08-29T15:25:50.850615+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":112,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.897719+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:25:33.897719+0000","last_peered":"2024-08-29T15:25:33.897719+0000","last_clean":"2024-08-29T15:25:33.897719+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:25:33.897719+0000","last_undegraded":"2024-08-29T15:25:33.897719+0000","last_fullsized":"2024-08-29T15:25:33.897719+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":276,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.859748+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:25:42.859748+0000","last_peered":"2024-08-29T15:25:42.859748+0000","last_clean":"2024-08-29T15:25:42.859748+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:25:42.859748+0000","last_undegraded":"2024-08-29T15:25:42.859748+0000","last_fullsized":"2024-08-29T15:25:42.859748+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":397,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860366+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:25:42.860366+0000","last_peered":"2024-08-29T15:25:42.860366+0000","last_clean":"2024-08-29T15:25:42.860366+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:25:42.860366+0000","last_undegraded":"2024-08-29T15:25:42.860366+0000","last_fullsized":"2024-08-29T15:25:42.860366+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":400,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844135+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:25:42.844135+0000","last_peered":"2024-08-29T15:25:42.844135+0000","last_clean":"2024-08-29T15:25:42.844135+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:25:42.844135+0000","last_undegraded":"2024-08-29T15:25:42.844135+0000","last_fullsized":"2024-08-29T15:25:42.844135+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":406,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852731+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:25:50.852731+0000","last_peered":"2024-08-29T15:25:50.852731+0000","last_clean":"2024-08-29T15:25:50.852731+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:25:50.852731+0000","last_undegraded":"2024-08-29T15:25:50.852731+0000","last_fullsized":"2024-08-29T15:25:50.852731+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813725+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:25:33.813725+0000","last_peered":"2024-08-29T15:25:33.813725+0000","last_clean":"2024-08-29T15:25:33.813725+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:25:33.813725+0000","last_undegraded":"2024-08-29T15:25:33.813725+0000","last_fullsized":"2024-08-29T15:25:33.813725+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":424,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.844613+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:25:42.844613+0000","last_peered":"2024-08-29T15:25:42.844613+0000","last_clean":"2024-08-29T15:25:42.844613+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:25:42.844613+0000","last_undegraded":"2024-08-29T15:25:42.844613+0000","last_fullsized":"2024-08-29T15:25:42.844613+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":388,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.859989+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:25:42.859989+0000","last_peered":"2024-08-29T15:25:42.859989+0000","last_clean":"2024-08-29T15:25:42.859989+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:25:42.859989+0000","last_undegraded":"2024-08-29T15:25:42.859989+0000","last_fullsized":"2024-08-29T15:25:42.859989+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813663+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:25:33.813663+0000","last_peered":"2024-08-29T15:25:33.813663+0000","last_clean":"2024-08-29T15:25:33.813663+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:25:33.813663+0000","last_undegraded":"2024-08-29T15:25:33.813663+0000","last_fullsized":"2024-08-29T15:25:33.813663+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":165,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:42.838380+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:25:42.838380+0000","last_peered":"2024-08-29T15:25:42.838380+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:25:42.838380+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:25:42.838380+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813805+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:25:33.813805+0000","last_peered":"2024-08-29T15:25:33.813805+0000","last_clean":"2024-08-29T15:25:33.813805+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:25:33.813805+0000","last_undegraded":"2024-08-29T15:25:33.813805+0000","last_fullsized":"2024-08-29T15:25:33.813805+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":171,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837612+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:25:42.837612+0000","last_peered":"2024-08-29T15:25:42.837612+0000","last_clean":"2024-08-29T15:25:42.837612+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:25:42.837612+0000","last_undegraded":"2024-08-29T15:25:42.837612+0000","last_fullsized":"2024-08-29T15:25:42.837612+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":437,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.851629+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:25:50.851629+0000","last_peered":"2024-08-29T15:25:50.851629+0000","last_clean":"2024-08-29T15:25:50.851629+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:25:50.851629+0000","last_undegraded":"2024-08-29T15:25:50.851629+0000","last_fullsized":"2024-08-29T15:25:50.851629+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":439,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.857878+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:25:50.857878+0000","last_peered":"2024-08-29T15:25:50.857878+0000","last_clean":"2024-08-29T15:25:50.857878+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:25:50.857878+0000","last_undegraded":"2024-08-29T15:25:50.857878+0000","last_fullsized":"2024-08-29T15:25:50.857878+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":51,"reported_epoch":271,"state":"active+clean","last_fresh":"2024-08-29T15:25:33.813370+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:25:33.813370+0000","last_peered":"2024-08-29T15:25:33.813370+0000","last_clean":"2024-08-29T15:25:33.813370+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:25:33.813370+0000","last_undegraded":"2024-08-29T15:25:33.813370+0000","last_fullsized":"2024-08-29T15:25:33.813370+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":444,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.860529+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:25:42.860529+0000","last_peered":"2024-08-29T15:25:42.860529+0000","last_clean":"2024-08-29T15:25:42.860529+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:25:42.860529+0000","last_undegraded":"2024-08-29T15:25:42.860529+0000","last_fullsized":"2024-08-29T15:25:42.860529+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3437494296,"num_objects":52453,"num_object_clones":0,"num_object_copies":157359,"num_objects_missing_on_primary":4297,"num_objects_missing":4297,"num_objects_degraded":28749,"num_objects_misplaced":9907,"num_objects_unfound":0,"num_objects_dirty":52453,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":198397,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18253,"num_bytes_recovered":1076232192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10433933312,"data_stored":10433921096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123974,"ondisk_log_size":123974,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234374,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2013336,"kb_used_data":1689360,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":91731816,"statfs":{"total":95995035648,"available":93933379584,"internally_reserved":0,"allocated":1729904640,"data_stored":1728327803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":331741643},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496005,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":238292,"kb_used_data":44748,"kb_used_omap":1,"kb_used_meta":193534,"kb_avail":93506860,"statfs":{"total":95995035648,"available":95751024640,"internally_reserved":0,"allocated":45821952,"data_stored":44248915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":198179271},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426592,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1139656,"kb_used_data":1058304,"kb_used_omap":8,"kb_used_meta":81335,"kb_avail":92605496,"statfs":{"total":95995035648,"available":94828027904,"internally_reserved":0,"allocated":1083703296,"data_stored":1082137725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":83287946},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019284,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1385528,"kb_used_data":1057456,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":92359624,"statfs":{"total":95995035648,"available":94576254976,"internally_reserved":0,"allocated":1082834944,"data_stored":1081285156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":335935942},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435469,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2406276,"kb_used_data":2095612,"kb_used_omap":4,"kb_used_meta":310651,"kb_avail":91338876,"statfs":{"total":95995035648,"available":93531009024,"internally_reserved":0,"allocated":2145906688,"data_stored":2144345057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":318106889},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903321,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2096156,"kb_used_data":1964304,"kb_used_omap":10,"kb_used_meta":131701,"kb_avail":91648996,"statfs":{"total":95995035648,"available":93848571904,"internally_reserved":0,"allocated":2011447296,"data_stored":2009877953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":134862352},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762522,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1742916,"kb_used_data":1476604,"kb_used_omap":7,"kb_used_meta":266296,"kb_avail":92002236,"statfs":{"total":95995035648,"available":94210289664,"internally_reserved":0,"allocated":1512042496,"data_stored":1510479625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":272687458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926048,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181772,"kb_used_data":861316,"kb_used_omap":7,"kb_used_meta":320440,"kb_avail":92563380,"statfs":{"total":95995035648,"available":94784901120,"internally_reserved":0,"allocated":881987584,"data_stored":880417492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":328130703},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":873336832,"data_stored":873332760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1504186368,"data_stored":1504182296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2005991424,"data_stored":2005991424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2136342528,"data_stored":2136342528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1075314688,"data_stored":1075314688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1076563968,"data_stored":1076559896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":41025536,"data_stored":41025536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1721171968,"data_stored":1721171968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:25:55.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:55.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:55.983 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:25:56.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:56.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:56.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:56.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:55 smithi171 ceph-mon[90031]: pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 28778/156891 objects degraded (18.343%); 9840/156891 objects misplaced (6.272%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:56.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:55 smithi027 ceph-mon[98970]: pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 28778/156891 objects degraded (18.343%); 9840/156891 objects misplaced (6.272%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:55 smithi027 ceph-mon[93921]: pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 200 op/s; 28778/156891 objects degraded (18.343%); 9840/156891 objects misplaced (6.272%); 1.5 MiB/s, 24 objects/s recovering 2024-08-29T15:25:56.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:56.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:56.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:56.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:56.573 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:56.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:56.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:25:56.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:25:56.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:56.848 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:56.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:25:56.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:56.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:56.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:57.076 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:57.097 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:57.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:57.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:57.222 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:57.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:57.346 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:57.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:25:57.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:25:57.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:57.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:57.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:57.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:57.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:57.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:57.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:57.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:57.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:57.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:58.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:25:58.104 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:58.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:25:58.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:58.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:57 smithi027 ceph-mon[98970]: pgmap v920: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/154560 objects degraded (18.533%); 9768/154560 objects misplaced (6.320%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:25:58.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28645/154560 objects degraded (18.533%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:58.210 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:57 smithi027 ceph-mon[93921]: pgmap v920: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/154560 objects degraded (18.533%); 9768/154560 objects misplaced (6.320%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:25:58.210 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28645/154560 objects degraded (18.533%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:58.227 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:57 smithi171 ceph-mon[90031]: pgmap v920: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/154560 objects degraded (18.533%); 9768/154560 objects misplaced (6.320%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:25:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28645/154560 objects degraded (18.533%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:25:58.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:58.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:58.455 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:58.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:58.579 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:58.601 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:58.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:58.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:58.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:58.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:58.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:58.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:58.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:58.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:25:58.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:25:58.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:58.977 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:58.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:25:59.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:59.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:59.193 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:25:59] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-08-29T15:25:59.207 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:59.229 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:59.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:25:59.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:59.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:25:59.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:25:59.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:25:59.549 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:25:59.549 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:25:59.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:25:59.564 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:25:59.584 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:25:59.584 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:25:59.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:59.603 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:25:59.604 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:25:59.608 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:59.624 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:25:59.624 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:25:59.637 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:25:59.637 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:25:59.637 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:25:59.637 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:25:59.637 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:25:59.642 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:25:59.642 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:25:59.659 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:25:59.659 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:25:59.675 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:25:59.675 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:25:59.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:59.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:59.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[98970]: pgmap v921: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/154560 objects degraded (18.533%); 9768/154560 objects misplaced (6.320%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:25:59.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:59.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2708071040' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[93921]: pgmap v921: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/154560 objects degraded (18.533%); 9768/154560 objects misplaced (6.320%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:25:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:25:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:25:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:25:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2708071040' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:25:59.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:25:59.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:25:59.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:25:59.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:25:59.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:25:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:25:59.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:59 smithi171 ceph-mon[90031]: pgmap v921: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/154560 objects degraded (18.533%); 9768/154560 objects misplaced (6.320%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:26:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:25:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2708071040' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:00.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:00.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:00.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:00.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:00.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:00.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:00.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:00.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:26:00.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:00.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:00.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:00.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:00.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:00.704 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:00.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:00.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:00.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:00.851 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:00.953 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:00.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:00.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:00.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:01.078 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:01.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:01.204 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:01.226 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:01.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:01.349 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:01.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:01.475 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:01.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:01.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:01.601 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:01.624 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:01.624 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:01.699 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:26:01.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:01.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:01.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:01 smithi027 ceph-mon[93921]: pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/153627 objects degraded (18.646%); 9701/153627 objects misplaced (6.315%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:26:01.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2683147212' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:01.747 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:01 smithi027 ceph-mon[98970]: pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/153627 objects degraded (18.646%); 9701/153627 objects misplaced (6.315%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:26:01.747 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2683147212' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:01.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:01.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:01.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:01.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:01.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:01.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:01 smithi171 ceph-mon[90031]: pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 202 op/s; 28645/153627 objects degraded (18.646%); 9701/153627 objects misplaced (6.315%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:26:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2683147212' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:02.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:02.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:02.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:02.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:02.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:02.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:02.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:02.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:02.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:02.476 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:02.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:02.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:02.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:02.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:02.722 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:02.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:02.846 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:02.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2683147212' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[98970]: osdmap e274: 8 total, 8 up, 8 in 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28645/153627 objects degraded (18.646%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2683147212' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[93921]: osdmap e274: 8 total, 8 up, 8 in 2024-08-29T15:26:02.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28645/153627 objects degraded (18.646%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:02.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:02.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:02 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2683147212' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:02 smithi171 ceph-mon[90031]: osdmap e274: 8 total, 8 up, 8 in 2024-08-29T15:26:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28645/153627 objects degraded (18.646%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:03.078 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:03.105 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:03.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:03.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:03.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:03.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:03.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:03.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:03.359 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:03.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:03.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:03.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:03.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:03.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:26:03.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:03.620 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:03.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:03.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:03.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:03.758 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:03.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:03.778 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:03 smithi027 ceph-mon[98970]: pgmap v924: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 28544/151791 objects degraded (18.805%); 9642/151791 objects misplaced (6.352%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:03.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:03 smithi027 ceph-mon[93921]: pgmap v924: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 28544/151791 objects degraded (18.805%); 9642/151791 objects misplaced (6.352%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:03.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:03.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:03.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:03.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:03 smithi171 ceph-mon[90031]: pgmap v924: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 188 op/s; 28544/151791 objects degraded (18.805%); 9642/151791 objects misplaced (6.352%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:04.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:04.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:04.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:04.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:04.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:04.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:04.272 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:04.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:04.374 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:04.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:04.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:04.489 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:26:04.490 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:26:04.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:04.524 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:04.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:04.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:04.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:04.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:04.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:04.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:04.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:04.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:04.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:05.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:05.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:05.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:05.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:05.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:05.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:05.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:05.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:05.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:05.415 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:05.435 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":924,"stamp":"2024-08-29T15:26:02.641875+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3318814717,"num_objects":50597,"num_object_clones":0,"num_object_copies":151791,"num_objects_missing_on_primary":4171,"num_objects_missing":4171,"num_objects_degraded":28544,"num_objects_misplaced":9642,"num_objects_unfound":0,"num_objects_dirty":50597,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":201404,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18616,"num_bytes_recovered":1098845389,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125340,"ondisk_log_size":125340,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11859036,"kb_used_data":9894552,"kb_used_omap":43,"kb_used_meta":1964372,"kb_avail":738102180,"statfs":{"total":767960285184,"available":755816632320,"internally_reserved":0,"allocated":10132021248,"data_stored":10119430894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2011517420},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,1,0,5,6,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-123469824,"num_objects":-1884,"num_object_clones":0,"num_object_copies":-5652,"num_objects_missing_on_primary":-134,"num_objects_missing":-134,"num_objects_degraded":-234,"num_objects_misplaced":-265,"num_objects_unfound":0,"num_objects_dirty":-1884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1884,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":254,"num_bytes_recovered":16646144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002730"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":432,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852993+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:25:50.852993+0000","last_peered":"2024-08-29T15:25:50.852993+0000","last_clean":"2024-08-29T15:25:50.852993+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:25:50.852993+0000","last_undegraded":"2024-08-29T15:25:50.852993+0000","last_fullsized":"2024-08-29T15:25:50.852993+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":51,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:42.840885+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:25:42.840885+0000","last_peered":"2024-08-29T15:25:42.840885+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:25:42.840885+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:25:42.840885+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":141,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837187+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:25:42.837187+0000","last_peered":"2024-08-29T15:25:42.837187+0000","last_clean":"2024-08-29T15:25:42.837187+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:25:42.837187+0000","last_undegraded":"2024-08-29T15:25:42.837187+0000","last_fullsized":"2024-08-29T15:25:42.837187+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":398,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864187+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:25:50.864187+0000","last_peered":"2024-08-29T15:25:50.864187+0000","last_clean":"2024-08-29T15:25:50.864187+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:25:50.864187+0000","last_undegraded":"2024-08-29T15:25:50.864187+0000","last_fullsized":"2024-08-29T15:25:50.864187+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":397,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864215+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:25:50.864215+0000","last_peered":"2024-08-29T15:25:50.864215+0000","last_clean":"2024-08-29T15:25:50.864215+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:25:50.864215+0000","last_undegraded":"2024-08-29T15:25:50.864215+0000","last_fullsized":"2024-08-29T15:25:50.864215+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840633+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:25:42.840633+0000","last_peered":"2024-08-29T15:25:42.840633+0000","last_clean":"2024-08-29T15:25:42.840633+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:25:42.840633+0000","last_undegraded":"2024-08-29T15:25:42.840633+0000","last_fullsized":"2024-08-29T15:25:42.840633+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":209,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.854098+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:25:50.854098+0000","last_peered":"2024-08-29T15:25:50.854098+0000","last_clean":"2024-08-29T15:25:50.854098+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:25:50.854098+0000","last_undegraded":"2024-08-29T15:25:50.854098+0000","last_fullsized":"2024-08-29T15:25:50.854098+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":142,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.863130+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:25:42.863130+0000","last_peered":"2024-08-29T15:25:42.863130+0000","last_clean":"2024-08-29T15:25:42.863130+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:25:42.863130+0000","last_undegraded":"2024-08-29T15:25:42.863130+0000","last_fullsized":"2024-08-29T15:25:42.863130+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"273'12666","reported_seq":20718,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:00.822049+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:26:00.822049+0000","last_peered":"2024-08-29T15:26:00.822049+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:26:00.822049+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"177'2600","ondisk_log_start":"177'2600","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212729856,"num_objects":3246,"num_object_clones":0,"num_object_copies":9738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12666,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":482},{"shards":"0,5","objects":321}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":50,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:42.840876+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:25:42.840876+0000","last_peered":"2024-08-29T15:25:42.840876+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:25:42.840876+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:25:42.840876+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837025+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:25:42.837025+0000","last_peered":"2024-08-29T15:25:42.837025+0000","last_clean":"2024-08-29T15:25:42.837025+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:25:42.837025+0000","last_undegraded":"2024-08-29T15:25:42.837025+0000","last_fullsized":"2024-08-29T15:25:42.837025+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"273'12404","reported_seq":12629,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.414744+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:26:01.414744+0000","last_peered":"2024-08-29T15:26:01.414744+0000","last_clean":"2024-08-29T15:26:01.414744+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:26:01.414744+0000","last_undegraded":"2024-08-29T15:26:01.414744+0000","last_fullsized":"2024-08-29T15:26:01.414744+0000","mapping_epoch":168,"log_start":"186'5728","ondisk_log_start":"186'5728","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206176256,"num_objects":3146,"num_object_clones":0,"num_object_copies":9438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12404,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":50,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:42.843772+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:25:42.843772+0000","last_peered":"2024-08-29T15:25:42.843772+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:25:42.843772+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:25:42.843772+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840507+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:25:42.840507+0000","last_peered":"2024-08-29T15:25:42.840507+0000","last_clean":"2024-08-29T15:25:42.840507+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:25:42.840507+0000","last_undegraded":"2024-08-29T15:25:42.840507+0000","last_fullsized":"2024-08-29T15:25:42.840507+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"273'12552","reported_seq":12747,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:01.341627+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:26:01.341627+0000","last_peered":"2024-08-29T15:26:01.341627+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:26:01.341627+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"186'5874","ondisk_log_start":"186'5874","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207093760,"num_objects":3160,"num_object_clones":0,"num_object_copies":9480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12552,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":683}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":214,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.867426+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:25:50.867426+0000","last_peered":"2024-08-29T15:25:50.867426+0000","last_clean":"2024-08-29T15:25:50.867426+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:25:50.867426+0000","last_undegraded":"2024-08-29T15:25:50.867426+0000","last_fullsized":"2024-08-29T15:25:50.867426+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:48.716636+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:25:48.716636+0000","last_peered":"2024-08-29T15:25:48.716636+0000","last_clean":"2024-08-29T15:25:48.716636+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:25:48.716636+0000","last_undegraded":"2024-08-29T15:25:48.716636+0000","last_fullsized":"2024-08-29T15:25:48.716636+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"273'12577","reported_seq":18022,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:58.480874+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:25:58.480874+0000","last_peered":"2024-08-29T15:25:58.480874+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:25:58.480874+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"180'4462","ondisk_log_start":"180'4462","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8115,"log_dups_size":0,"ondisk_log_size":8115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209780736,"num_objects":3201,"num_object_clones":0,"num_object_copies":9603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12577,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3201}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.838643+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:25:42.838643+0000","last_peered":"2024-08-29T15:25:42.838643+0000","last_clean":"2024-08-29T15:25:42.838643+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:25:42.838643+0000","last_undegraded":"2024-08-29T15:25:42.838643+0000","last_fullsized":"2024-08-29T15:25:42.838643+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":412,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864125+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:25:50.864125+0000","last_peered":"2024-08-29T15:25:50.864125+0000","last_clean":"2024-08-29T15:25:50.864125+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:25:50.864125+0000","last_undegraded":"2024-08-29T15:25:50.864125+0000","last_fullsized":"2024-08-29T15:25:50.864125+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"273'12579","reported_seq":12739,"reported_epoch":273,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:59.414189+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:25:59.414189+0000","last_peered":"2024-08-29T15:25:59.414189+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:25:59.414189+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:25:59.414189+0000","mapping_epoch":234,"log_start":"180'4633","ondisk_log_start":"180'4633","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212008960,"num_objects":3235,"num_object_clones":0,"num_object_copies":9705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12579,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":758},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":406,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.851319+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:25:50.851319+0000","last_peered":"2024-08-29T15:25:50.851319+0000","last_clean":"2024-08-29T15:25:50.851319+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:25:50.851319+0000","last_undegraded":"2024-08-29T15:25:50.851319+0000","last_fullsized":"2024-08-29T15:25:50.851319+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.841021+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:25:42.841021+0000","last_peered":"2024-08-29T15:25:42.841021+0000","last_clean":"2024-08-29T15:25:42.841021+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:25:42.841021+0000","last_undegraded":"2024-08-29T15:25:42.841021+0000","last_fullsized":"2024-08-29T15:25:42.841021+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"273'12609","reported_seq":12851,"reported_epoch":273,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:01.316992+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:26:01.316992+0000","last_peered":"2024-08-29T15:26:01.316992+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:26:01.316992+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:26:01.316992+0000","mapping_epoch":234,"log_start":"186'5868","ondisk_log_start":"186'5868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6741,"log_dups_size":0,"ondisk_log_size":6741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211877888,"num_objects":3233,"num_object_clones":0,"num_object_copies":9699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12609,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":768},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":116,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:48.716618+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:25:48.716618+0000","last_peered":"2024-08-29T15:25:48.716618+0000","last_clean":"2024-08-29T15:25:48.716618+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:25:48.716618+0000","last_undegraded":"2024-08-29T15:25:48.716618+0000","last_fullsized":"2024-08-29T15:25:48.716618+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.841112+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:25:42.841112+0000","last_peered":"2024-08-29T15:25:42.841112+0000","last_clean":"2024-08-29T15:25:42.841112+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:25:42.841112+0000","last_undegraded":"2024-08-29T15:25:42.841112+0000","last_fullsized":"2024-08-29T15:25:42.841112+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"273'12465","reported_seq":12891,"reported_epoch":273,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:26:01.291002+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:26:01.291002+0000","last_peered":"2024-08-29T15:26:01.291002+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:26:01.291002+0000","last_undegraded":"2024-08-29T15:26:01.291002+0000","last_fullsized":"2024-08-29T15:26:01.291002+0000","mapping_epoch":202,"log_start":"186'5767","ondisk_log_start":"186'5767","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209256448,"num_objects":3193,"num_object_clones":0,"num_object_copies":9579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3193,"num_objects_unfound":0,"num_objects_dirty":3193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12465,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":3193}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840554+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:25:42.840554+0000","last_peered":"2024-08-29T15:25:42.840554+0000","last_clean":"2024-08-29T15:25:42.840554+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:25:42.840554+0000","last_undegraded":"2024-08-29T15:25:42.840554+0000","last_fullsized":"2024-08-29T15:25:42.840554+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":258,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.843860+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:25:42.843860+0000","last_peered":"2024-08-29T15:25:42.843860+0000","last_clean":"2024-08-29T15:25:42.843860+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:25:42.843860+0000","last_undegraded":"2024-08-29T15:25:42.843860+0000","last_fullsized":"2024-08-29T15:25:42.843860+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"273'12647","reported_seq":12801,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.131538+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:26:01.131538+0000","last_peered":"2024-08-29T15:26:01.131538+0000","last_clean":"2024-08-29T15:26:01.131538+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:26:01.131538+0000","last_undegraded":"2024-08-29T15:26:01.131538+0000","last_fullsized":"2024-08-29T15:26:01.131538+0000","mapping_epoch":218,"log_start":"177'2578","ondisk_log_start":"177'2578","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10069,"log_dups_size":0,"ondisk_log_size":10069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203948056,"num_objects":3113,"num_object_clones":0,"num_object_copies":9339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3113,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12647,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":409,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852613+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:25:50.852613+0000","last_peered":"2024-08-29T15:25:50.852613+0000","last_clean":"2024-08-29T15:25:50.852613+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:25:50.852613+0000","last_undegraded":"2024-08-29T15:25:50.852613+0000","last_fullsized":"2024-08-29T15:25:50.852613+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":402,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.854178+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:25:50.854178+0000","last_peered":"2024-08-29T15:25:50.854178+0000","last_clean":"2024-08-29T15:25:50.854178+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:25:50.854178+0000","last_undegraded":"2024-08-29T15:25:50.854178+0000","last_fullsized":"2024-08-29T15:25:50.854178+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"273'12045","reported_seq":12247,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:26:00.716965+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:26:00.716965+0000","last_peered":"2024-08-29T15:26:00.716965+0000","last_clean":"2024-08-29T15:26:00.716965+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:26:00.716965+0000","last_undegraded":"2024-08-29T15:26:00.716965+0000","last_fullsized":"2024-08-29T15:26:00.716965+0000","mapping_epoch":201,"log_start":"176'2000","ondisk_log_start":"176'2000","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207290368,"num_objects":3163,"num_object_clones":0,"num_object_copies":9489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12045,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":500,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.863901+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:25:50.863901+0000","last_peered":"2024-08-29T15:25:50.863901+0000","last_clean":"2024-08-29T15:25:50.863901+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:25:50.863901+0000","last_undegraded":"2024-08-29T15:25:50.863901+0000","last_fullsized":"2024-08-29T15:25:50.863901+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":744,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850582+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:25:50.850582+0000","last_peered":"2024-08-29T15:25:50.850582+0000","last_clean":"2024-08-29T15:25:50.850582+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:25:50.850582+0000","last_undegraded":"2024-08-29T15:25:50.850582+0000","last_fullsized":"2024-08-29T15:25:50.850582+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":252,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.851664+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:25:50.851664+0000","last_peered":"2024-08-29T15:25:50.851664+0000","last_clean":"2024-08-29T15:25:50.851664+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:25:50.851664+0000","last_undegraded":"2024-08-29T15:25:50.851664+0000","last_fullsized":"2024-08-29T15:25:50.851664+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":114,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.863477+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:25:42.863477+0000","last_peered":"2024-08-29T15:25:42.863477+0000","last_clean":"2024-08-29T15:25:42.863477+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:25:42.863477+0000","last_undegraded":"2024-08-29T15:25:42.863477+0000","last_fullsized":"2024-08-29T15:25:42.863477+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"273'12559","reported_seq":12983,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:26:00.682861+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:26:00.682861+0000","last_peered":"2024-08-29T15:26:00.682861+0000","last_clean":"2024-08-29T15:26:00.682861+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:26:00.682861+0000","last_undegraded":"2024-08-29T15:26:00.682861+0000","last_fullsized":"2024-08-29T15:26:00.682861+0000","mapping_epoch":201,"log_start":"183'5553","ondisk_log_start":"183'5553","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204668928,"num_objects":3123,"num_object_clones":0,"num_object_copies":9369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12559,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":874,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852758+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:25:50.852758+0000","last_peered":"2024-08-29T15:25:50.852758+0000","last_clean":"2024-08-29T15:25:50.852758+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:25:50.852758+0000","last_undegraded":"2024-08-29T15:25:50.852758+0000","last_fullsized":"2024-08-29T15:25:50.852758+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":412,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.863814+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:25:50.863814+0000","last_peered":"2024-08-29T15:25:50.863814+0000","last_clean":"2024-08-29T15:25:50.863814+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:25:50.863814+0000","last_undegraded":"2024-08-29T15:25:50.863814+0000","last_fullsized":"2024-08-29T15:25:50.863814+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":420,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864451+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:25:50.864451+0000","last_peered":"2024-08-29T15:25:50.864451+0000","last_clean":"2024-08-29T15:25:50.864451+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:25:50.864451+0000","last_undegraded":"2024-08-29T15:25:50.864451+0000","last_fullsized":"2024-08-29T15:25:50.864451+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"273'12524","reported_seq":21015,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:01.355477+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:26:01.355477+0000","last_peered":"2024-08-29T15:26:01.355477+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:26:01.355477+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"186'5830","ondisk_log_start":"186'5830","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207486976,"num_objects":3166,"num_object_clones":0,"num_object_copies":9498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3643,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12524,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4148,"num_bytes_recovered":271843328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":728},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":683,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850772+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:25:50.850772+0000","last_peered":"2024-08-29T15:25:50.850772+0000","last_clean":"2024-08-29T15:25:50.850772+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:25:50.850772+0000","last_undegraded":"2024-08-29T15:25:50.850772+0000","last_fullsized":"2024-08-29T15:25:50.850772+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":104,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.862883+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:25:42.862883+0000","last_peered":"2024-08-29T15:25:42.862883+0000","last_clean":"2024-08-29T15:25:42.862883+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:25:42.862883+0000","last_undegraded":"2024-08-29T15:25:42.862883+0000","last_fullsized":"2024-08-29T15:25:42.862883+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":447,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.853964+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:25:50.853964+0000","last_peered":"2024-08-29T15:25:50.853964+0000","last_clean":"2024-08-29T15:25:50.853964+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:25:50.853964+0000","last_undegraded":"2024-08-29T15:25:50.853964+0000","last_fullsized":"2024-08-29T15:25:50.853964+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"273'12594","reported_seq":12829,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.026159+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:26:01.026159+0000","last_peered":"2024-08-29T15:26:01.026159+0000","last_clean":"2024-08-29T15:26:01.026159+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:26:01.026159+0000","last_undegraded":"2024-08-29T15:26:01.026159+0000","last_fullsized":"2024-08-29T15:26:01.026159+0000","mapping_epoch":234,"log_start":"186'5897","ondisk_log_start":"186'5897","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204341248,"num_objects":3118,"num_object_clones":0,"num_object_copies":9354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12594,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":212,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.851596+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:25:50.851596+0000","last_peered":"2024-08-29T15:25:50.851596+0000","last_clean":"2024-08-29T15:25:50.851596+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:25:50.851596+0000","last_undegraded":"2024-08-29T15:25:50.851596+0000","last_fullsized":"2024-08-29T15:25:50.851596+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840728+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:25:42.840728+0000","last_peered":"2024-08-29T15:25:42.840728+0000","last_clean":"2024-08-29T15:25:42.840728+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:25:42.840728+0000","last_undegraded":"2024-08-29T15:25:42.840728+0000","last_fullsized":"2024-08-29T15:25:42.840728+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":119,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850795+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:25:50.850795+0000","last_peered":"2024-08-29T15:25:50.850795+0000","last_clean":"2024-08-29T15:25:50.850795+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:25:50.850795+0000","last_undegraded":"2024-08-29T15:25:50.850795+0000","last_fullsized":"2024-08-29T15:25:50.850795+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"273'12633","reported_seq":12867,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:58.994061+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:25:58.994061+0000","last_peered":"2024-08-29T15:25:58.994061+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:25:58.994061+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"180'4442","ondisk_log_start":"180'4442","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8191,"log_dups_size":0,"ondisk_log_size":8191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209256448,"num_objects":3193,"num_object_clones":0,"num_object_copies":9579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12633,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":770}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":144,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.851012+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:25:50.851012+0000","last_peered":"2024-08-29T15:25:50.851012+0000","last_clean":"2024-08-29T15:25:50.851012+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:25:50.851012+0000","last_undegraded":"2024-08-29T15:25:50.851012+0000","last_fullsized":"2024-08-29T15:25:50.851012+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":839,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.853753+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:25:50.853753+0000","last_peered":"2024-08-29T15:25:50.853753+0000","last_clean":"2024-08-29T15:25:50.853753+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:25:50.853753+0000","last_undegraded":"2024-08-29T15:25:50.853753+0000","last_fullsized":"2024-08-29T15:25:50.853753+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":416,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.853206+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:25:50.853206+0000","last_peered":"2024-08-29T15:25:50.853206+0000","last_clean":"2024-08-29T15:25:50.853206+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:25:50.853206+0000","last_undegraded":"2024-08-29T15:25:50.853206+0000","last_fullsized":"2024-08-29T15:25:50.853206+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"273'12548","reported_seq":13338,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:00.667705+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:26:00.667705+0000","last_peered":"2024-08-29T15:26:00.667705+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:26:00.667705+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"183'5480","ondisk_log_start":"183'5480","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204472320,"num_objects":3120,"num_object_clones":0,"num_object_copies":9360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3099,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":3120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12548,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":969}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":421,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.853663+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:25:50.853663+0000","last_peered":"2024-08-29T15:25:50.853663+0000","last_clean":"2024-08-29T15:25:50.853663+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:25:50.853663+0000","last_undegraded":"2024-08-29T15:25:50.853663+0000","last_fullsized":"2024-08-29T15:25:50.853663+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":701,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:44.974771+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:25:44.974771+0000","last_peered":"2024-08-29T15:25:44.974771+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:25:44.974771+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:25:44.974771+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":412,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864032+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:25:50.864032+0000","last_peered":"2024-08-29T15:25:50.864032+0000","last_clean":"2024-08-29T15:25:50.864032+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:25:50.864032+0000","last_undegraded":"2024-08-29T15:25:50.864032+0000","last_fullsized":"2024-08-29T15:25:50.864032+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"273'12398","reported_seq":19556,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:59.571088+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:25:59.571088+0000","last_peered":"2024-08-29T15:25:59.571088+0000","last_clean":"2024-08-29T15:25:59.571088+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:25:59.571088+0000","last_undegraded":"2024-08-29T15:25:59.571088+0000","last_fullsized":"2024-08-29T15:25:59.571088+0000","mapping_epoch":234,"log_start":"180'4499","ondisk_log_start":"180'4499","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7899,"log_dups_size":0,"ondisk_log_size":7899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12398,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":406,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852439+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:25:50.852439+0000","last_peered":"2024-08-29T15:25:50.852439+0000","last_clean":"2024-08-29T15:25:50.852439+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:25:50.852439+0000","last_undegraded":"2024-08-29T15:25:50.852439+0000","last_fullsized":"2024-08-29T15:25:50.852439+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1029,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:50.852685+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:25:50.852685+0000","last_peered":"2024-08-29T15:25:50.852685+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:25:50.852685+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":404,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850700+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:25:50.850700+0000","last_peered":"2024-08-29T15:25:50.850700+0000","last_clean":"2024-08-29T15:25:50.850700+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:25:50.850700+0000","last_undegraded":"2024-08-29T15:25:50.850700+0000","last_fullsized":"2024-08-29T15:25:50.850700+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"273'12481","reported_seq":19771,"reported_epoch":273,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:25:58.839919+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:25:58.839919+0000","last_peered":"2024-08-29T15:25:58.839919+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:25:58.839919+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"180'4912","ondisk_log_start":"180'4912","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205062144,"num_objects":3129,"num_object_clones":0,"num_object_copies":9387,"num_objects_missing_on_primary":4163,"num_objects_missing":4163,"num_objects_degraded":2313,"num_objects_misplaced":3114,"num_objects_unfound":0,"num_objects_dirty":3129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12481,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3558,"num_bytes_recovered":233046016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":743},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":402,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864093+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:25:50.864093+0000","last_peered":"2024-08-29T15:25:50.864093+0000","last_clean":"2024-08-29T15:25:50.864093+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:25:50.864093+0000","last_undegraded":"2024-08-29T15:25:50.864093+0000","last_fullsized":"2024-08-29T15:25:50.864093+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":125,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.863570+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:25:42.863570+0000","last_peered":"2024-08-29T15:25:42.863570+0000","last_clean":"2024-08-29T15:25:42.863570+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:25:42.863570+0000","last_undegraded":"2024-08-29T15:25:42.863570+0000","last_fullsized":"2024-08-29T15:25:42.863570+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840577+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:25:42.840577+0000","last_peered":"2024-08-29T15:25:42.840577+0000","last_clean":"2024-08-29T15:25:42.840577+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:25:42.840577+0000","last_undegraded":"2024-08-29T15:25:42.840577+0000","last_fullsized":"2024-08-29T15:25:42.840577+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840828+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:25:42.840828+0000","last_peered":"2024-08-29T15:25:42.840828+0000","last_clean":"2024-08-29T15:25:42.840828+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:25:42.840828+0000","last_undegraded":"2024-08-29T15:25:42.840828+0000","last_fullsized":"2024-08-29T15:25:42.840828+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":50,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:42.840898+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:25:42.840898+0000","last_peered":"2024-08-29T15:25:42.840898+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:25:42.840898+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:25:42.840898+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":410,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.853202+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:25:50.853202+0000","last_peered":"2024-08-29T15:25:50.853202+0000","last_clean":"2024-08-29T15:25:50.853202+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:25:50.853202+0000","last_undegraded":"2024-08-29T15:25:50.853202+0000","last_fullsized":"2024-08-29T15:25:50.853202+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":161,"reported_epoch":273,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:50.854007+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:25:50.854007+0000","last_peered":"2024-08-29T15:25:50.854007+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:25:50.854007+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:25:50.854007+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":141,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837771+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:25:42.837771+0000","last_peered":"2024-08-29T15:25:42.837771+0000","last_clean":"2024-08-29T15:25:42.837771+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:25:42.837771+0000","last_undegraded":"2024-08-29T15:25:42.837771+0000","last_fullsized":"2024-08-29T15:25:42.837771+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":370,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850515+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:25:50.850515+0000","last_peered":"2024-08-29T15:25:50.850515+0000","last_clean":"2024-08-29T15:25:50.850515+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:25:50.850515+0000","last_undegraded":"2024-08-29T15:25:50.850515+0000","last_fullsized":"2024-08-29T15:25:50.850515+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":423,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864643+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:25:50.864643+0000","last_peered":"2024-08-29T15:25:50.864643+0000","last_clean":"2024-08-29T15:25:50.864643+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:25:50.864643+0000","last_undegraded":"2024-08-29T15:25:50.864643+0000","last_fullsized":"2024-08-29T15:25:50.864643+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":422,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.850615+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:25:50.850615+0000","last_peered":"2024-08-29T15:25:50.850615+0000","last_clean":"2024-08-29T15:25:50.850615+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:25:50.850615+0000","last_undegraded":"2024-08-29T15:25:50.850615+0000","last_fullsized":"2024-08-29T15:25:50.850615+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.863640+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:25:42.863640+0000","last_peered":"2024-08-29T15:25:42.863640+0000","last_clean":"2024-08-29T15:25:42.863640+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:25:42.863640+0000","last_undegraded":"2024-08-29T15:25:42.863640+0000","last_fullsized":"2024-08-29T15:25:42.863640+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":278,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.863772+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:25:50.863772+0000","last_peered":"2024-08-29T15:25:50.863772+0000","last_clean":"2024-08-29T15:25:50.863772+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:25:50.863772+0000","last_undegraded":"2024-08-29T15:25:50.863772+0000","last_fullsized":"2024-08-29T15:25:50.863772+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":399,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864368+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:25:50.864368+0000","last_peered":"2024-08-29T15:25:50.864368+0000","last_clean":"2024-08-29T15:25:50.864368+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:25:50.864368+0000","last_undegraded":"2024-08-29T15:25:50.864368+0000","last_fullsized":"2024-08-29T15:25:50.864368+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":402,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.856663+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:25:50.856663+0000","last_peered":"2024-08-29T15:25:50.856663+0000","last_clean":"2024-08-29T15:25:50.856663+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:25:50.856663+0000","last_undegraded":"2024-08-29T15:25:50.856663+0000","last_fullsized":"2024-08-29T15:25:50.856663+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":406,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.852731+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:25:50.852731+0000","last_peered":"2024-08-29T15:25:50.852731+0000","last_clean":"2024-08-29T15:25:50.852731+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:25:50.852731+0000","last_undegraded":"2024-08-29T15:25:50.852731+0000","last_fullsized":"2024-08-29T15:25:50.852731+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840688+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:25:42.840688+0000","last_peered":"2024-08-29T15:25:42.840688+0000","last_clean":"2024-08-29T15:25:42.840688+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:25:42.840688+0000","last_undegraded":"2024-08-29T15:25:42.840688+0000","last_fullsized":"2024-08-29T15:25:42.840688+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":426,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.853859+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:25:50.853859+0000","last_peered":"2024-08-29T15:25:50.853859+0000","last_clean":"2024-08-29T15:25:50.853859+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:25:50.853859+0000","last_undegraded":"2024-08-29T15:25:50.853859+0000","last_fullsized":"2024-08-29T15:25:50.853859+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":390,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864010+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:25:50.864010+0000","last_peered":"2024-08-29T15:25:50.864010+0000","last_clean":"2024-08-29T15:25:50.864010+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:25:50.864010+0000","last_undegraded":"2024-08-29T15:25:50.864010+0000","last_fullsized":"2024-08-29T15:25:50.864010+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840625+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:25:42.840625+0000","last_peered":"2024-08-29T15:25:42.840625+0000","last_clean":"2024-08-29T15:25:42.840625+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:25:42.840625+0000","last_undegraded":"2024-08-29T15:25:42.840625+0000","last_fullsized":"2024-08-29T15:25:42.840625+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":165,"reported_epoch":272,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:25:42.838380+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:25:42.838380+0000","last_peered":"2024-08-29T15:25:42.838380+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:25:42.838380+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:25:42.838380+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840681+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:25:42.840681+0000","last_peered":"2024-08-29T15:25:42.840681+0000","last_clean":"2024-08-29T15:25:42.840681+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:25:42.840681+0000","last_undegraded":"2024-08-29T15:25:42.840681+0000","last_fullsized":"2024-08-29T15:25:42.840681+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":171,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.837612+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:25:42.837612+0000","last_peered":"2024-08-29T15:25:42.837612+0000","last_clean":"2024-08-29T15:25:42.837612+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:25:42.837612+0000","last_undegraded":"2024-08-29T15:25:42.837612+0000","last_fullsized":"2024-08-29T15:25:42.837612+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":438,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.851671+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:25:50.851671+0000","last_peered":"2024-08-29T15:25:50.851671+0000","last_clean":"2024-08-29T15:25:50.851671+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:25:50.851671+0000","last_undegraded":"2024-08-29T15:25:50.851671+0000","last_fullsized":"2024-08-29T15:25:50.851671+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":439,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.857878+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:25:50.857878+0000","last_peered":"2024-08-29T15:25:50.857878+0000","last_clean":"2024-08-29T15:25:50.857878+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:25:50.857878+0000","last_undegraded":"2024-08-29T15:25:50.857878+0000","last_fullsized":"2024-08-29T15:25:50.857878+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":52,"reported_epoch":272,"state":"active+clean","last_fresh":"2024-08-29T15:25:42.840452+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:25:42.840452+0000","last_peered":"2024-08-29T15:25:42.840452+0000","last_clean":"2024-08-29T15:25:42.840452+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:25:42.840452+0000","last_undegraded":"2024-08-29T15:25:42.840452+0000","last_fullsized":"2024-08-29T15:25:42.840452+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":446,"reported_epoch":273,"state":"active+clean","last_fresh":"2024-08-29T15:25:50.864547+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:25:50.864547+0000","last_peered":"2024-08-29T15:25:50.864547+0000","last_clean":"2024-08-29T15:25:50.864547+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:25:50.864547+0000","last_undegraded":"2024-08-29T15:25:50.864547+0000","last_fullsized":"2024-08-29T15:25:50.864547+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3314024472,"num_objects":50569,"num_object_clones":0,"num_object_copies":151707,"num_objects_missing_on_primary":4163,"num_objects_missing":4163,"num_objects_degraded":28515,"num_objects_misplaced":9642,"num_objects_unfound":0,"num_objects_dirty":50569,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":200281,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18507,"num_bytes_recovered":1092878336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10073944064,"data_stored":10073931848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124158,"ondisk_log_size":124158,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234376,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1954728,"kb_used_data":1626656,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":91790424,"statfs":{"total":95995035648,"available":93993394176,"internally_reserved":0,"allocated":1665695744,"data_stored":1664111259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":335935947},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496007,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":233892,"kb_used_data":36252,"kb_used_omap":1,"kb_used_meta":197630,"kb_avail":93511260,"statfs":{"total":95995035648,"available":95755530240,"internally_reserved":0,"allocated":37122048,"data_stored":35541363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":202373575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,1,0,5,6,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426594,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1100120,"kb_used_data":1018768,"kb_used_omap":8,"kb_used_meta":81335,"kb_avail":92645032,"statfs":{"total":95995035648,"available":94868512768,"internally_reserved":0,"allocated":1043218432,"data_stored":1041645213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":83287946},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019286,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335112,"kb_used_data":1019328,"kb_used_omap":1,"kb_used_meta":315774,"kb_avail":92410040,"statfs":{"total":95995035648,"available":94627880960,"internally_reserved":0,"allocated":1043791872,"data_stored":1042234436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":323353030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435471,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2335188,"kb_used_data":2020364,"kb_used_omap":4,"kb_used_meta":314811,"kb_avail":91409964,"statfs":{"total":95995035648,"available":93603803136,"internally_reserved":0,"allocated":2068852736,"data_stored":2067283457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":322366729},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903323,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2038764,"kb_used_data":1902880,"kb_used_omap":10,"kb_used_meta":135861,"kb_avail":91706388,"statfs":{"total":95995035648,"available":93907341312,"internally_reserved":0,"allocated":1948549120,"data_stored":1946972129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":139122192},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762524,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1708052,"kb_used_data":1441740,"kb_used_omap":7,"kb_used_meta":266296,"kb_avail":92037100,"statfs":{"total":95995035648,"available":94245990400,"internally_reserved":0,"allocated":1476341760,"data_stored":1474771241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":272687458},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926050,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1153180,"kb_used_data":828564,"kb_used_omap":7,"kb_used_meta":324600,"kb_avail":92591972,"statfs":{"total":95995035648,"available":94814179328,"internally_reserved":0,"allocated":848449536,"data_stored":846871796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":332390543},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":839454720,"data_stored":839450648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1464995840,"data_stored":1464991768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1944322048,"data_stored":1944322048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2061172736,"data_stored":2061172736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1037500416,"data_stored":1037500416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1037701120,"data_stored":1037697048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":32833536,"data_stored":32833536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1655963648,"data_stored":1655963648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:26:05.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:26:05.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:05.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:05.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:05.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:05.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:05.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:26:05.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:05.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:05.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:05.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:05.909 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:05.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:05.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:05.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:05 smithi171 ceph-mon[90031]: from='client.45911 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:05 smithi171 ceph-mon[90031]: pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 28544/151107 objects degraded (18.890%); 9576/151107 objects misplaced (6.337%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:06.033 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:05 smithi027 ceph-mon[98970]: from='client.45911 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:06.033 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:05 smithi027 ceph-mon[98970]: pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 28544/151107 objects degraded (18.890%); 9576/151107 objects misplaced (6.337%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:06.034 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:05 smithi027 ceph-mon[93921]: from='client.45911 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:06.034 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:05 smithi027 ceph-mon[93921]: pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 28544/151107 objects degraded (18.890%); 9576/151107 objects misplaced (6.337%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:06.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:06.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:06.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:06.185 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:06.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:06.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:06.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:06.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:06.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:06.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:06.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:06.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:06.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:06.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:06.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:06.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:06.854 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:06.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:06.936 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:06.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:06.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:07.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:07.060 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:07.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:07.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:07.185 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:07.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:07.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:07.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:07.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:07.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:07.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:07.435 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:07.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:07.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:07.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:07.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:07.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:07.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:07.687 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:26:07.758 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:26:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:07.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:07.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:07.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:07.883 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:07.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:07.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:07.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:08.007 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:07 smithi171 ceph-mon[90031]: pgmap v926: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 28413/148515 objects degraded (19.131%); 9497/148515 objects misplaced (6.395%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28413/148515 objects degraded (19.131%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:07 smithi027 ceph-mon[98970]: pgmap v926: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 28413/148515 objects degraded (19.131%); 9497/148515 objects misplaced (6.395%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28413/148515 objects degraded (19.131%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:08.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:07 smithi027 ceph-mon[93921]: pgmap v926: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 28413/148515 objects degraded (19.131%); 9497/148515 objects misplaced (6.395%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:08.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28413/148515 objects degraded (19.131%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:08.109 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:08.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:08.237 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:08.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:08.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:08.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:08.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:08.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:08.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:08.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:08.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:08.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:08.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:08.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:08.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:08.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:08.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:08.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:08.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:09.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:09.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:09.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:09.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:09.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:09.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:26:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:26:09] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-08-29T15:26:09.367 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:09.389 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:09.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:09.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:09.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:09.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:09.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:09.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:09.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:26:09.767 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:09.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:09.895 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:09.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:09.997 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:10.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:09 smithi171 ceph-mon[90031]: pgmap v927: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 28413/148515 objects degraded (19.131%); 9497/148515 objects misplaced (6.395%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:10.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:10.040 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:09 smithi027 ceph-mon[98970]: pgmap v927: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 28413/148515 objects degraded (19.131%); 9497/148515 objects misplaced (6.395%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:10.041 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:09 smithi027 ceph-mon[93921]: pgmap v927: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 201 op/s; 28413/148515 objects degraded (19.131%); 9497/148515 objects misplaced (6.395%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:10.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:10.147 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:10.249 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:10.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:10.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:10.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:10.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:10.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:10.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:10.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:10.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:10.721 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:26:10.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:10.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:10.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:10.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:10.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:10.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:10.967 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:10.998 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:11.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:11.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3882853249' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:11.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:10 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:11.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:11.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:11.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3882853249' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:11.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:10 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:11.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3882853249' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:11.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:10 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:11.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:11.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:11.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:11.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:11.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.280 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:11.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:11.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:11.404 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:11.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:11.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:11.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:11.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:11.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:11.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:11.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:11.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[98970]: pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 28413/147531 objects degraded (19.259%); 9428/147531 objects misplaced (6.391%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:26:11.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[98970]: osdmap e275: 8 total, 8 up, 8 in 2024-08-29T15:26:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[93921]: pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 28413/147531 objects degraded (19.259%); 9428/147531 objects misplaced (6.391%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:26:11.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:11 smithi027 ceph-mon[93921]: osdmap e275: 8 total, 8 up, 8 in 2024-08-29T15:26:11.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:11.911 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:11.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:12.013 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:11 smithi171 ceph-mon[90031]: pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 28413/147531 objects degraded (19.259%); 9428/147531 objects misplaced (6.391%); 1.6 MiB/s, 25 objects/s recovering 2024-08-29T15:26:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:11 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:26:12.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:11 smithi171 ceph-mon[90031]: osdmap e275: 8 total, 8 up, 8 in 2024-08-29T15:26:12.035 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:12.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:12.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:12.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:12.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:12.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:12.285 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:12.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:12.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:12.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:12.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:12.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:12.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:12.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:12.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:12.766 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:12.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:12.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28413/147531 objects degraded (19.259%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:12.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28413/147531 objects degraded (19.259%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:12.891 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:12.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:12.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:13.016 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28413/147531 objects degraded (19.259%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:13.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:13.139 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:13.140 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:26:13.141 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:26:13.158 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:26:13.159 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:26:13.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:13.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:13.176 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:26:13.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:26:13.193 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:26:13.193 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:26:13.211 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:26:13.211 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:26:13.229 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:26:13.229 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:26:13.247 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:26:13.247 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:26:13.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:13.269 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:26:13.270 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:26:13.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:13.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:13.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:13.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:13.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:13.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:26:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:13.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:13.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:13.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:13.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:13.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:13.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:13.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:13.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:13.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:13 smithi171 ceph-mon[90031]: pgmap v930: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 28291/145245 objects degraded (19.478%); 9351/145245 objects misplaced (6.438%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:26:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/722906199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:14.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:14.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:14.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:13 smithi027 ceph-mon[93921]: pgmap v930: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 28291/145245 objects degraded (19.478%); 9351/145245 objects misplaced (6.438%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:26:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/722906199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:13 smithi027 ceph-mon[98970]: pgmap v930: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 28291/145245 objects degraded (19.478%); 9351/145245 objects misplaced (6.438%); 1.7 MiB/s, 26 objects/s recovering 2024-08-29T15:26:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/722906199' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:14.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:14.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:14.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:14.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:14.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:14.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:14.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:14.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:14.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:14.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:14.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:14.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:14.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:14.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:14.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:14.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:14.816 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:14.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:14.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:14.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:14.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:15.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:15.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:15.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:15.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:15.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:15.293 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:15.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:15.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:15.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:15.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:15.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:15.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:15.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:15.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:15.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:15.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:15 smithi027 ceph-mon[98970]: pgmap v931: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 217 op/s; 28291/144591 objects degraded (19.566%); 9281/144591 objects misplaced (6.419%); 1.7 MiB/s, 27 objects/s recovering 2024-08-29T15:26:15.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:15 smithi027 ceph-mon[93921]: pgmap v931: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 217 op/s; 28291/144591 objects degraded (19.566%); 9281/144591 objects misplaced (6.419%); 1.7 MiB/s, 27 objects/s recovering 2024-08-29T15:26:15.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:15.895 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:26:15.895 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 28291/144591 objects degraded (19.566%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1357,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":275,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":48197,"data_bytes":3161528317,"bytes_used":11679277056,"bytes_avail":756281008128,"bytes_total":767960285184,"degraded_objects":28291,"degraded_total":144591,"degraded_ratio":0.19566224730446569,"misplaced_objects":9281,"misplaced_total":144591,"misplaced_ratio":0.064187950840647073,"recovering_objects_per_sec":27,"recovering_bytes_per_sec":1782033,"recovering_keys_per_sec":0,"num_objects_recovered":272,"num_bytes_recovered":17825792,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":217},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:26:15.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:15.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:15.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:15.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:15 smithi171 ceph-mon[90031]: pgmap v931: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 217 op/s; 28291/144591 objects degraded (19.566%); 9281/144591 objects misplaced (6.419%); 1.7 MiB/s, 27 objects/s recovering 2024-08-29T15:26:16.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:16.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:16.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:16.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:16.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:16.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:16.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:16.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:16.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:16.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:16.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:16.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:16.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:16.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:16.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:16.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:16.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:16.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:16.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:16.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:17.005 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:16 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/788044364' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:26:17.005 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:16 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/788044364' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:26:17.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:16 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/788044364' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:26:17.027 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:26:17.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:26:17.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:17.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:17.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:17.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:17.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:26:17.254 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:26:17.254 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:26:17.254 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:26:17.254 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:26:17.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:17.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:17.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:17.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:17.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:17.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:17.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:17.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:17.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:17.812 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:17.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:17.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:17.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:17 smithi027 ceph-mon[93921]: pgmap v932: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 28151/141882 objects degraded (19.841%); 9195/141882 objects misplaced (6.481%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:17.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28151/141882 objects degraded (19.841%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:17.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:17 smithi027 ceph-mon[98970]: pgmap v932: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 28151/141882 objects degraded (19.841%); 9195/141882 objects misplaced (6.481%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:17.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28151/141882 objects degraded (19.841%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:17.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:17.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:17.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:17 smithi171 ceph-mon[90031]: pgmap v932: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 28151/141882 objects degraded (19.841%); 9195/141882 objects misplaced (6.481%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28151/141882 objects degraded (19.841%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:18.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:18.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:18.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:18.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:18.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:18.337 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:18.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:18.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:18.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:18.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:18.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:18.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:18.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:18.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:18.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:18.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:18.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:18.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:18.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:18.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:19.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:26:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:26:19] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.51.0" 2024-08-29T15:26:19.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:19.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:19.192 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:19.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:19.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:19.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:19.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:19.362 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:19.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:19.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:19.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:19.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:19.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:19.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:19.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:19.855 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:19.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:19.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:19.982 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:19.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:19.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:19 smithi171 ceph-mon[90031]: pgmap v933: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 28151/141882 objects degraded (19.841%); 9195/141882 objects misplaced (6.481%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:19 smithi027 ceph-mon[98970]: pgmap v933: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 28151/141882 objects degraded (19.841%); 9195/141882 objects misplaced (6.481%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:19 smithi027 ceph-mon[93921]: pgmap v933: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 221 op/s; 28151/141882 objects degraded (19.841%); 9195/141882 objects misplaced (6.481%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:20.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:20.107 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:20.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:20.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:20.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:20.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:20.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:20.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:20.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:20.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:20.458 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:20.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:20.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:20.581 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:20.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:20.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:20.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:26:20.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:20.745 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:20.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:20.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:20.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:20.749 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:26:20.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:20.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:20.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:20.967 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:20 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2100079067' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:20.967 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:20 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2100079067' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:20.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:21.007 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:20 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2100079067' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:21.109 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:21.135 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:21.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:21.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:21.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:21.366 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:21.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:21.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:21.515 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:21.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:21.641 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:21.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:21.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:21.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:21.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:21.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:21.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:21.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:21 smithi171 ceph-mon[90031]: pgmap v934: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 28151/140772 objects degraded (19.998%); 9121/140772 objects misplaced (6.479%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:21 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:26:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2100079067' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:21 smithi171 ceph-mon[90031]: osdmap e276: 8 total, 8 up, 8 in 2024-08-29T15:26:22.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:22.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[93921]: pgmap v934: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 28151/140772 objects degraded (19.998%); 9121/140772 objects misplaced (6.479%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:22.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:26:22.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2100079067' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:22.062 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[93921]: osdmap e276: 8 total, 8 up, 8 in 2024-08-29T15:26:22.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[98970]: pgmap v934: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 28151/140772 objects degraded (19.998%); 9121/140772 objects misplaced (6.479%); 1.8 MiB/s, 28 objects/s recovering 2024-08-29T15:26:22.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:26:22.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2100079067' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:22.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:21 smithi027 ceph-mon[98970]: osdmap e276: 8 total, 8 up, 8 in 2024-08-29T15:26:22.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:22.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:22.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:22.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:22.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:22.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:22.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:22.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:22.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:22.520 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:22.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:22.622 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:22.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:22.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:22.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28151/140772 objects degraded (19.998%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:22.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28151/140772 objects degraded (19.998%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:22.875 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:22.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:22.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:23.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28151/140772 objects degraded (19.998%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:23.023 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:23.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:23.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:23.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:23.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:23.275 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:23.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:23.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:23.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:23.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:23.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:23.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:23.651 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:23.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:23.775 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:23.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:23.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:23.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:23.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:23.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:23 smithi027 ceph-mon[98970]: pgmap v936: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27992/138225 objects degraded (20.251%); 9041/138225 objects misplaced (6.541%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:23.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:23 smithi027 ceph-mon[93921]: pgmap v936: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27992/138225 objects degraded (20.251%); 9041/138225 objects misplaced (6.541%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:23.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:26:24.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:23 smithi171 ceph-mon[90031]: pgmap v936: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27992/138225 objects degraded (20.251%); 9041/138225 objects misplaced (6.541%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:24.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:24.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:24.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:24.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:24.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:24.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:24.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:24.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:24.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:24.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:24.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:24.533 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:24.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:24.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:24.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:24.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:24.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:24.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:24.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:24.912 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:24.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:24.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:24.938 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:26:24.938 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:26:24.938 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:26:24.938 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:26:24.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:26:24.940 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:26:24.940 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:26:25.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4201259504' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:25.036 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4201259504' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4201259504' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:25.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:25.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:25.199 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:25.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:25.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:25.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:25.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:25.519 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:25.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:25.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:25.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:25.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:25.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:25.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:25.792 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:25.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:25 smithi027 ceph-mon[98970]: pgmap v937: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 27992/137532 objects degraded (20.353%); 8968/137532 objects misplaced (6.521%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:25.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:25 smithi027 ceph-mon[93921]: pgmap v937: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 27992/137532 objects degraded (20.353%); 8968/137532 objects misplaced (6.521%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:25.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:25.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:25.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:25 smithi171 ceph-mon[90031]: pgmap v937: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 235 op/s; 27992/137532 objects degraded (20.353%); 8968/137532 objects misplaced (6.521%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:26.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:26.039 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:26.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:26.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:26.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:26.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:26.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:26.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:26.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:26.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:26:26.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:26.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:26.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:26.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:26.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:26.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:26.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:26.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:26.933 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:26.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:27.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:27.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:27.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:27.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:27.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:27.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:27.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:27.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:27.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:27.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:27.559 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:27.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:27.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:27.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:27.810 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:27.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:27.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:27.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:28.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:26:28.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:28.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:27 smithi027 ceph-mon[93921]: from='client.45947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:28.052 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:27 smithi027 ceph-mon[93921]: pgmap v938: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27836/134889 objects degraded (20.636%); 8899/134889 objects misplaced (6.597%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:28.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 27836/134889 objects degraded (20.636%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:28.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:27 smithi027 ceph-mon[98970]: from='client.45947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:28.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:27 smithi027 ceph-mon[98970]: pgmap v938: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27836/134889 objects degraded (20.636%); 8899/134889 objects misplaced (6.597%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:28.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 27836/134889 objects degraded (20.636%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:28.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:28.087 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":937,"stamp":"2024-08-29T15:26:24.648213+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3007322109,"num_objects":45844,"num_object_clones":0,"num_object_copies":137532,"num_objects_missing_on_primary":3819,"num_objects_missing":3819,"num_objects_degraded":27992,"num_objects_misplaced":8968,"num_objects_unfound":0,"num_objects_dirty":45844,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":206157,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19254,"num_bytes_recovered":1140657357,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125493,"ondisk_log_size":125493,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11020176,"kb_used_data":9053960,"kb_used_omap":43,"kb_used_meta":1966036,"kb_avail":738941040,"statfs":{"total":767960285184,"available":756675624960,"internally_reserved":0,"allocated":9271255040,"data_stored":9258519374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2013221356},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,2,1,1,1,0,2,4,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-154206208,"num_objects":-2353,"num_object_clones":0,"num_object_copies":-7059,"num_objects_missing_on_primary":-147,"num_objects_missing":-147,"num_objects_degraded":-299,"num_objects_misplaced":-313,"num_objects_unfound":0,"num_objects_dirty":-2353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2353,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002874"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":438,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.750769+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:26:20.750769+0000","last_peered":"2024-08-29T15:26:20.750769+0000","last_clean":"2024-08-29T15:26:20.750769+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:26:20.750769+0000","last_undegraded":"2024-08-29T15:26:20.750769+0000","last_fullsized":"2024-08-29T15:26:20.750769+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":53,"reported_epoch":274,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:01.700871+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:26:01.700871+0000","last_peered":"2024-08-29T15:26:01.700871+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:26:01.700871+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:26:01.700871+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":144,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.718859+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:26:10.718859+0000","last_peered":"2024-08-29T15:26:10.718859+0000","last_clean":"2024-08-29T15:26:10.718859+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:26:10.718859+0000","last_undegraded":"2024-08-29T15:26:10.718859+0000","last_fullsized":"2024-08-29T15:26:10.718859+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":404,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764909+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:26:20.764909+0000","last_peered":"2024-08-29T15:26:20.764909+0000","last_clean":"2024-08-29T15:26:20.764909+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:26:20.764909+0000","last_undegraded":"2024-08-29T15:26:20.764909+0000","last_fullsized":"2024-08-29T15:26:20.764909+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":403,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764446+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:26:20.764446+0000","last_peered":"2024-08-29T15:26:20.764446+0000","last_clean":"2024-08-29T15:26:20.764446+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:26:20.764446+0000","last_undegraded":"2024-08-29T15:26:20.764446+0000","last_fullsized":"2024-08-29T15:26:20.764446+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.700678+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:26:01.700678+0000","last_peered":"2024-08-29T15:26:01.700678+0000","last_clean":"2024-08-29T15:26:01.700678+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:26:01.700678+0000","last_undegraded":"2024-08-29T15:26:01.700678+0000","last_fullsized":"2024-08-29T15:26:01.700678+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":213,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.722643+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:26:10.722643+0000","last_peered":"2024-08-29T15:26:10.722643+0000","last_clean":"2024-08-29T15:26:10.722643+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:26:10.722643+0000","last_undegraded":"2024-08-29T15:26:10.722643+0000","last_fullsized":"2024-08-29T15:26:10.722643+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":146,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:13.970182+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:26:13.970182+0000","last_peered":"2024-08-29T15:26:13.970182+0000","last_clean":"2024-08-29T15:26:13.970182+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:26:13.970182+0000","last_undegraded":"2024-08-29T15:26:13.970182+0000","last_fullsized":"2024-08-29T15:26:13.970182+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"276'12966","reported_seq":21022,"reported_epoch":276,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:21.163963+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:26:21.163963+0000","last_peered":"2024-08-29T15:26:21.163963+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:26:21.163963+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"178'2900","ondisk_log_start":"178'2900","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193069056,"num_objects":2946,"num_object_clones":0,"num_object_copies":8838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12966,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":436},{"shards":"0,5","objects":21}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":52,"reported_epoch":274,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:01.701133+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:26:01.701133+0000","last_peered":"2024-08-29T15:26:01.701133+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:26:01.701133+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:26:01.701133+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.746664+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:26:20.746664+0000","last_peered":"2024-08-29T15:26:20.746664+0000","last_clean":"2024-08-29T15:26:20.746664+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:26:20.746664+0000","last_undegraded":"2024-08-29T15:26:20.746664+0000","last_fullsized":"2024-08-29T15:26:20.746664+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"276'12664","reported_seq":12895,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:21.287395+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:26:21.287395+0000","last_peered":"2024-08-29T15:26:21.287395+0000","last_clean":"2024-08-29T15:26:21.287395+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:26:21.287395+0000","last_undegraded":"2024-08-29T15:26:21.287395+0000","last_fullsized":"2024-08-29T15:26:21.287395+0000","mapping_epoch":168,"log_start":"186'5928","ondisk_log_start":"186'5928","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189136896,"num_objects":2886,"num_object_clones":0,"num_object_copies":8658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12664,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":52,"reported_epoch":274,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:01.701009+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:26:01.701009+0000","last_peered":"2024-08-29T15:26:01.701009+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:26:01.701009+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:26:01.701009+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":55,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.718733+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:26:10.718733+0000","last_peered":"2024-08-29T15:26:10.718733+0000","last_clean":"2024-08-29T15:26:10.718733+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:26:10.718733+0000","last_undegraded":"2024-08-29T15:26:10.718733+0000","last_fullsized":"2024-08-29T15:26:10.718733+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"276'12817","reported_seq":13018,"reported_epoch":276,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:21.363194+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:26:21.363194+0000","last_peered":"2024-08-29T15:26:21.363194+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:26:21.363194+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"186'6074","ondisk_log_start":"186'6074","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189726720,"num_objects":2895,"num_object_clones":0,"num_object_copies":8685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12817,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":418}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":220,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.765390+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:26:20.765390+0000","last_peered":"2024-08-29T15:26:20.765390+0000","last_clean":"2024-08-29T15:26:20.765390+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:26:20.765390+0000","last_undegraded":"2024-08-29T15:26:20.765390+0000","last_fullsized":"2024-08-29T15:26:20.765390+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.780588+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:26:10.780588+0000","last_peered":"2024-08-29T15:26:10.780588+0000","last_clean":"2024-08-29T15:26:10.780588+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:26:10.780588+0000","last_undegraded":"2024-08-29T15:26:10.780588+0000","last_fullsized":"2024-08-29T15:26:10.780588+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"276'12949","reported_seq":18401,"reported_epoch":276,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:23.199855+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:26:23.199855+0000","last_peered":"2024-08-29T15:26:23.199855+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:26:23.199855+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"180'4762","ondisk_log_start":"180'4762","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185401344,"num_objects":2829,"num_object_clones":0,"num_object_copies":8487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12949,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2829}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":162,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.721592+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:26:10.721592+0000","last_peered":"2024-08-29T15:26:10.721592+0000","last_clean":"2024-08-29T15:26:10.721592+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:26:10.721592+0000","last_undegraded":"2024-08-29T15:26:10.721592+0000","last_fullsized":"2024-08-29T15:26:10.721592+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":418,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.765141+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:26:20.765141+0000","last_peered":"2024-08-29T15:26:20.765141+0000","last_clean":"2024-08-29T15:26:20.765141+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:26:20.765141+0000","last_undegraded":"2024-08-29T15:26:20.765141+0000","last_fullsized":"2024-08-29T15:26:20.765141+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"275'12867","reported_seq":13029,"reported_epoch":275,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:19.780625+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:26:19.780625+0000","last_peered":"2024-08-29T15:26:19.780625+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:26:19.780625+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:26:19.780625+0000","mapping_epoch":234,"log_start":"180'4933","ondisk_log_start":"180'4933","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7934,"log_dups_size":0,"ondisk_log_size":7934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193134592,"num_objects":2947,"num_object_clones":0,"num_object_copies":8841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12867,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":470},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":412,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.751215+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:26:20.751215+0000","last_peered":"2024-08-29T15:26:20.751215+0000","last_clean":"2024-08-29T15:26:20.751215+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:26:20.751215+0000","last_undegraded":"2024-08-29T15:26:20.751215+0000","last_fullsized":"2024-08-29T15:26:20.751215+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.700766+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:26:01.700766+0000","last_peered":"2024-08-29T15:26:01.700766+0000","last_clean":"2024-08-29T15:26:01.700766+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:26:01.700766+0000","last_undegraded":"2024-08-29T15:26:01.700766+0000","last_fullsized":"2024-08-29T15:26:01.700766+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"276'12908","reported_seq":13156,"reported_epoch":276,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:21.279499+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:26:21.279499+0000","last_peered":"2024-08-29T15:26:21.279499+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:26:21.279499+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:26:21.279499+0000","mapping_epoch":234,"log_start":"187'6168","ondisk_log_start":"187'6168","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192282624,"num_objects":2934,"num_object_clones":0,"num_object_copies":8802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12908,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":469},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":119,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.780532+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:26:10.780532+0000","last_peered":"2024-08-29T15:26:10.780532+0000","last_clean":"2024-08-29T15:26:10.780532+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:26:10.780532+0000","last_undegraded":"2024-08-29T15:26:10.780532+0000","last_fullsized":"2024-08-29T15:26:10.780532+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.700919+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:26:01.700919+0000","last_peered":"2024-08-29T15:26:01.700919+0000","last_clean":"2024-08-29T15:26:01.700919+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:26:01.700919+0000","last_undegraded":"2024-08-29T15:26:01.700919+0000","last_fullsized":"2024-08-29T15:26:01.700919+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"276'12787","reported_seq":13219,"reported_epoch":276,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:26:21.228789+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:26:21.228789+0000","last_peered":"2024-08-29T15:26:21.228789+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:26:21.228789+0000","last_undegraded":"2024-08-29T15:26:21.228789+0000","last_fullsized":"2024-08-29T15:26:21.228789+0000","mapping_epoch":202,"log_start":"186'6067","ondisk_log_start":"186'6067","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188153856,"num_objects":2871,"num_object_clones":0,"num_object_copies":8613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2871,"num_objects_unfound":0,"num_objects_dirty":2871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12787,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":2871}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":55,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.718830+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:26:10.718830+0000","last_peered":"2024-08-29T15:26:10.718830+0000","last_clean":"2024-08-29T15:26:10.718830+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:26:10.718830+0000","last_undegraded":"2024-08-29T15:26:10.718830+0000","last_fullsized":"2024-08-29T15:26:10.718830+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":260,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.701213+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:26:01.701213+0000","last_peered":"2024-08-29T15:26:01.701213+0000","last_clean":"2024-08-29T15:26:01.701213+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:26:01.701213+0000","last_undegraded":"2024-08-29T15:26:01.701213+0000","last_fullsized":"2024-08-29T15:26:01.701213+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"276'12927","reported_seq":13084,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:21.182407+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:26:21.182407+0000","last_peered":"2024-08-29T15:26:21.182407+0000","last_clean":"2024-08-29T15:26:21.182407+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:26:21.182407+0000","last_undegraded":"2024-08-29T15:26:21.182407+0000","last_fullsized":"2024-08-29T15:26:21.182407+0000","mapping_epoch":218,"log_start":"178'2878","ondisk_log_start":"178'2878","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10049,"log_dups_size":0,"ondisk_log_size":10049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597976,"num_objects":2833,"num_object_clones":0,"num_object_copies":8499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2833,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12927,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":416,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752845+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:26:20.752845+0000","last_peered":"2024-08-29T15:26:20.752845+0000","last_clean":"2024-08-29T15:26:20.752845+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:26:20.752845+0000","last_undegraded":"2024-08-29T15:26:20.752845+0000","last_fullsized":"2024-08-29T15:26:20.752845+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":406,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.722717+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:26:10.722717+0000","last_peered":"2024-08-29T15:26:10.722717+0000","last_clean":"2024-08-29T15:26:10.722717+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:26:10.722717+0000","last_undegraded":"2024-08-29T15:26:10.722717+0000","last_fullsized":"2024-08-29T15:26:10.722717+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"276'12321","reported_seq":12527,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:21.046773+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:26:21.046773+0000","last_peered":"2024-08-29T15:26:21.046773+0000","last_clean":"2024-08-29T15:26:21.046773+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:26:21.046773+0000","last_undegraded":"2024-08-29T15:26:21.046773+0000","last_fullsized":"2024-08-29T15:26:21.046773+0000","mapping_epoch":201,"log_start":"177'2300","ondisk_log_start":"177'2300","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10021,"log_dups_size":0,"ondisk_log_size":10021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189202432,"num_objects":2887,"num_object_clones":0,"num_object_copies":8661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12321,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":507,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.765399+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:26:20.765399+0000","last_peered":"2024-08-29T15:26:20.765399+0000","last_clean":"2024-08-29T15:26:20.765399+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:26:20.765399+0000","last_undegraded":"2024-08-29T15:26:20.765399+0000","last_fullsized":"2024-08-29T15:26:20.765399+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":747,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.719152+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:26:10.719152+0000","last_peered":"2024-08-29T15:26:10.719152+0000","last_clean":"2024-08-29T15:26:10.719152+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:26:10.719152+0000","last_undegraded":"2024-08-29T15:26:10.719152+0000","last_fullsized":"2024-08-29T15:26:10.719152+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":255,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.720078+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:26:10.720078+0000","last_peered":"2024-08-29T15:26:10.720078+0000","last_clean":"2024-08-29T15:26:10.720078+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:26:10.720078+0000","last_undegraded":"2024-08-29T15:26:10.720078+0000","last_fullsized":"2024-08-29T15:26:10.720078+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":117,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.774465+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:26:10.774465+0000","last_peered":"2024-08-29T15:26:10.774465+0000","last_clean":"2024-08-29T15:26:10.774465+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:26:10.774465+0000","last_undegraded":"2024-08-29T15:26:10.774465+0000","last_fullsized":"2024-08-29T15:26:10.774465+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"275'12821","reported_seq":13249,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.674483+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:26:20.674483+0000","last_peered":"2024-08-29T15:26:20.674483+0000","last_clean":"2024-08-29T15:26:20.674483+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:26:20.674483+0000","last_undegraded":"2024-08-29T15:26:20.674483+0000","last_fullsized":"2024-08-29T15:26:20.674483+0000","mapping_epoch":201,"log_start":"186'5753","ondisk_log_start":"186'5753","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187498496,"num_objects":2861,"num_object_clones":0,"num_object_copies":8583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12821,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":879,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.722159+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:26:10.722159+0000","last_peered":"2024-08-29T15:26:10.722159+0000","last_clean":"2024-08-29T15:26:10.722159+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:26:10.722159+0000","last_undegraded":"2024-08-29T15:26:10.722159+0000","last_fullsized":"2024-08-29T15:26:10.722159+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":419,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764413+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:26:20.764413+0000","last_peered":"2024-08-29T15:26:20.764413+0000","last_clean":"2024-08-29T15:26:20.764413+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:26:20.764413+0000","last_undegraded":"2024-08-29T15:26:20.764413+0000","last_fullsized":"2024-08-29T15:26:20.764413+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":426,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764675+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:26:20.764675+0000","last_peered":"2024-08-29T15:26:20.764675+0000","last_clean":"2024-08-29T15:26:20.764675+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:26:20.764675+0000","last_undegraded":"2024-08-29T15:26:20.764675+0000","last_fullsized":"2024-08-29T15:26:20.764675+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"276'12810","reported_seq":21879,"reported_epoch":276,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:21.439583+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:26:21.439583+0000","last_peered":"2024-08-29T15:26:21.439583+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:26:21.439583+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"186'6130","ondisk_log_start":"186'6130","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188743680,"num_objects":2880,"num_object_clones":0,"num_object_copies":8640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3357,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12810,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4434,"num_bytes_recovered":290586624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":442},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":689,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.750380+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:26:20.750380+0000","last_peered":"2024-08-29T15:26:20.750380+0000","last_clean":"2024-08-29T15:26:20.750380+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:26:20.750380+0000","last_undegraded":"2024-08-29T15:26:20.750380+0000","last_fullsized":"2024-08-29T15:26:20.750380+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":108,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.779742+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:26:20.779742+0000","last_peered":"2024-08-29T15:26:20.779742+0000","last_clean":"2024-08-29T15:26:20.779742+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:26:20.779742+0000","last_undegraded":"2024-08-29T15:26:20.779742+0000","last_fullsized":"2024-08-29T15:26:20.779742+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":451,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.722112+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:26:10.722112+0000","last_peered":"2024-08-29T15:26:10.722112+0000","last_clean":"2024-08-29T15:26:10.722112+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:26:10.722112+0000","last_undegraded":"2024-08-29T15:26:10.722112+0000","last_fullsized":"2024-08-29T15:26:10.722112+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"276'12872","reported_seq":13114,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:21.276359+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:26:21.276359+0000","last_peered":"2024-08-29T15:26:21.276359+0000","last_clean":"2024-08-29T15:26:21.276359+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:26:21.276359+0000","last_undegraded":"2024-08-29T15:26:21.276359+0000","last_fullsized":"2024-08-29T15:26:21.276359+0000","mapping_epoch":234,"log_start":"187'6197","ondisk_log_start":"187'6197","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186122240,"num_objects":2840,"num_object_clones":0,"num_object_copies":8520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12872,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":215,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.719948+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:26:10.719948+0000","last_peered":"2024-08-29T15:26:10.719948+0000","last_clean":"2024-08-29T15:26:10.719948+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:26:10.719948+0000","last_undegraded":"2024-08-29T15:26:10.719948+0000","last_fullsized":"2024-08-29T15:26:10.719948+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":55,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:04.831637+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:26:04.831637+0000","last_peered":"2024-08-29T15:26:04.831637+0000","last_clean":"2024-08-29T15:26:04.831637+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:26:04.831637+0000","last_undegraded":"2024-08-29T15:26:04.831637+0000","last_fullsized":"2024-08-29T15:26:04.831637+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":122,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.719429+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:26:10.719429+0000","last_peered":"2024-08-29T15:26:10.719429+0000","last_clean":"2024-08-29T15:26:10.719429+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:26:10.719429+0000","last_undegraded":"2024-08-29T15:26:10.719429+0000","last_fullsized":"2024-08-29T15:26:10.719429+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"276'12994","reported_seq":13234,"reported_epoch":276,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:23.897364+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:26:23.897364+0000","last_peered":"2024-08-29T15:26:23.897364+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:26:23.897364+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"180'4842","ondisk_log_start":"180'4842","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8152,"log_dups_size":0,"ondisk_log_size":8152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597952,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12994,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":409}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":147,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.718792+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:26:10.718792+0000","last_peered":"2024-08-29T15:26:10.718792+0000","last_clean":"2024-08-29T15:26:10.718792+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:26:10.718792+0000","last_undegraded":"2024-08-29T15:26:10.718792+0000","last_fullsized":"2024-08-29T15:26:10.718792+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":843,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.724783+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:26:10.724783+0000","last_peered":"2024-08-29T15:26:10.724783+0000","last_clean":"2024-08-29T15:26:10.724783+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:26:10.724783+0000","last_undegraded":"2024-08-29T15:26:10.724783+0000","last_fullsized":"2024-08-29T15:26:10.724783+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":420,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.722516+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:26:10.722516+0000","last_peered":"2024-08-29T15:26:10.722516+0000","last_clean":"2024-08-29T15:26:10.722516+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:26:10.722516+0000","last_undegraded":"2024-08-29T15:26:10.722516+0000","last_fullsized":"2024-08-29T15:26:10.722516+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"275'12817","reported_seq":13611,"reported_epoch":275,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:20.714423+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:26:20.714423+0000","last_peered":"2024-08-29T15:26:20.714423+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:26:20.714423+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"186'5780","ondisk_log_start":"186'5780","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186843136,"num_objects":2851,"num_object_clones":0,"num_object_copies":8553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2833,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":2851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12817,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":700}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":425,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.724683+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:26:10.724683+0000","last_peered":"2024-08-29T15:26:10.724683+0000","last_clean":"2024-08-29T15:26:10.724683+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:26:10.724683+0000","last_undegraded":"2024-08-29T15:26:10.724683+0000","last_fullsized":"2024-08-29T15:26:10.724683+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":703,"reported_epoch":274,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:01.714861+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:26:01.714861+0000","last_peered":"2024-08-29T15:26:01.714861+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:26:01.714861+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:26:01.714861+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":419,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.763948+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:26:20.763948+0000","last_peered":"2024-08-29T15:26:20.763948+0000","last_clean":"2024-08-29T15:26:20.763948+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:26:20.763948+0000","last_undegraded":"2024-08-29T15:26:20.763948+0000","last_fullsized":"2024-08-29T15:26:20.763948+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"275'12680","reported_seq":19841,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:19.787172+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:26:19.787172+0000","last_peered":"2024-08-29T15:26:19.787172+0000","last_clean":"2024-08-29T15:26:19.787172+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:26:19.787172+0000","last_undegraded":"2024-08-29T15:26:19.787172+0000","last_fullsized":"2024-08-29T15:26:19.787172+0000","mapping_epoch":234,"log_start":"180'4699","ondisk_log_start":"180'4699","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7981,"log_dups_size":0,"ondisk_log_size":7981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180092928,"num_objects":2748,"num_object_clones":0,"num_object_copies":8244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12680,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":413,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.756546+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:26:20.756546+0000","last_peered":"2024-08-29T15:26:20.756546+0000","last_clean":"2024-08-29T15:26:20.756546+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:26:20.756546+0000","last_undegraded":"2024-08-29T15:26:20.756546+0000","last_fullsized":"2024-08-29T15:26:20.756546+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1033,"reported_epoch":275,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:10.722187+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:26:10.722187+0000","last_peered":"2024-08-29T15:26:10.722187+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:26:10.722187+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":410,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752073+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:26:20.752073+0000","last_peered":"2024-08-29T15:26:20.752073+0000","last_clean":"2024-08-29T15:26:20.752073+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:26:20.752073+0000","last_undegraded":"2024-08-29T15:26:20.752073+0000","last_fullsized":"2024-08-29T15:26:20.752073+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"276'12834","reported_seq":20831,"reported_epoch":276,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:24.076891+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:26:24.076891+0000","last_peered":"2024-08-29T15:26:24.076891+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:26:24.076891+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"180'5312","ondisk_log_start":"180'5312","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7522,"log_dups_size":0,"ondisk_log_size":7522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181927936,"num_objects":2776,"num_object_clones":0,"num_object_copies":8328,"num_objects_missing_on_primary":3811,"num_objects_missing":3811,"num_objects_degraded":2313,"num_objects_misplaced":2762,"num_objects_unfound":0,"num_objects_dirty":2776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12834,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3910,"num_bytes_recovered":256114688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":14},{"shards":"1,2,7","objects":391},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":408,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764346+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:26:20.764346+0000","last_peered":"2024-08-29T15:26:20.764346+0000","last_clean":"2024-08-29T15:26:20.764346+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:26:20.764346+0000","last_undegraded":"2024-08-29T15:26:20.764346+0000","last_fullsized":"2024-08-29T15:26:20.764346+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":128,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.774889+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:26:10.774889+0000","last_peered":"2024-08-29T15:26:10.774889+0000","last_clean":"2024-08-29T15:26:10.774889+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:26:10.774889+0000","last_undegraded":"2024-08-29T15:26:10.774889+0000","last_fullsized":"2024-08-29T15:26:10.774889+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:04.831588+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:26:04.831588+0000","last_peered":"2024-08-29T15:26:04.831588+0000","last_clean":"2024-08-29T15:26:04.831588+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:26:04.831588+0000","last_undegraded":"2024-08-29T15:26:04.831588+0000","last_fullsized":"2024-08-29T15:26:04.831588+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.701070+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:26:01.701070+0000","last_peered":"2024-08-29T15:26:01.701070+0000","last_clean":"2024-08-29T15:26:01.701070+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:26:01.701070+0000","last_undegraded":"2024-08-29T15:26:01.701070+0000","last_fullsized":"2024-08-29T15:26:01.701070+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":52,"reported_epoch":274,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:01.700843+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:26:01.700843+0000","last_peered":"2024-08-29T15:26:01.700843+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:26:01.700843+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:26:01.700843+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":414,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.722367+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:26:10.722367+0000","last_peered":"2024-08-29T15:26:10.722367+0000","last_clean":"2024-08-29T15:26:10.722367+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:26:10.722367+0000","last_undegraded":"2024-08-29T15:26:10.722367+0000","last_fullsized":"2024-08-29T15:26:10.722367+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":167,"reported_epoch":276,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:20.756026+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:26:20.756026+0000","last_peered":"2024-08-29T15:26:20.756026+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:26:20.756026+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:26:20.756026+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":144,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.719481+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:26:10.719481+0000","last_peered":"2024-08-29T15:26:10.719481+0000","last_clean":"2024-08-29T15:26:10.719481+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:26:10.719481+0000","last_undegraded":"2024-08-29T15:26:10.719481+0000","last_fullsized":"2024-08-29T15:26:10.719481+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":376,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.750131+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:26:20.750131+0000","last_peered":"2024-08-29T15:26:20.750131+0000","last_clean":"2024-08-29T15:26:20.750131+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:26:20.750131+0000","last_undegraded":"2024-08-29T15:26:20.750131+0000","last_fullsized":"2024-08-29T15:26:20.750131+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":429,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764896+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:26:20.764896+0000","last_peered":"2024-08-29T15:26:20.764896+0000","last_clean":"2024-08-29T15:26:20.764896+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:26:20.764896+0000","last_undegraded":"2024-08-29T15:26:20.764896+0000","last_fullsized":"2024-08-29T15:26:20.764896+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":428,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.751543+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:26:20.751543+0000","last_peered":"2024-08-29T15:26:20.751543+0000","last_clean":"2024-08-29T15:26:20.751543+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:26:20.751543+0000","last_undegraded":"2024-08-29T15:26:20.751543+0000","last_fullsized":"2024-08-29T15:26:20.751543+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":116,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.774899+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:26:10.774899+0000","last_peered":"2024-08-29T15:26:10.774899+0000","last_clean":"2024-08-29T15:26:10.774899+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:26:10.774899+0000","last_undegraded":"2024-08-29T15:26:10.774899+0000","last_fullsized":"2024-08-29T15:26:10.774899+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":285,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.765309+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:26:20.765309+0000","last_peered":"2024-08-29T15:26:20.765309+0000","last_clean":"2024-08-29T15:26:20.765309+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:26:20.765309+0000","last_undegraded":"2024-08-29T15:26:20.765309+0000","last_fullsized":"2024-08-29T15:26:20.765309+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":405,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764591+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:26:20.764591+0000","last_peered":"2024-08-29T15:26:20.764591+0000","last_clean":"2024-08-29T15:26:20.764591+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:26:20.764591+0000","last_undegraded":"2024-08-29T15:26:20.764591+0000","last_fullsized":"2024-08-29T15:26:20.764591+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":406,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.722352+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:26:10.722352+0000","last_peered":"2024-08-29T15:26:10.722352+0000","last_clean":"2024-08-29T15:26:10.722352+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:26:10.722352+0000","last_undegraded":"2024-08-29T15:26:10.722352+0000","last_fullsized":"2024-08-29T15:26:10.722352+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":412,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.755921+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:26:20.755921+0000","last_peered":"2024-08-29T15:26:20.755921+0000","last_clean":"2024-08-29T15:26:20.755921+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:26:20.755921+0000","last_undegraded":"2024-08-29T15:26:20.755921+0000","last_fullsized":"2024-08-29T15:26:20.755921+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.700593+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:26:01.700593+0000","last_peered":"2024-08-29T15:26:01.700593+0000","last_clean":"2024-08-29T15:26:01.700593+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:26:01.700593+0000","last_undegraded":"2024-08-29T15:26:01.700593+0000","last_fullsized":"2024-08-29T15:26:01.700593+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":430,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.724875+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:26:10.724875+0000","last_peered":"2024-08-29T15:26:10.724875+0000","last_clean":"2024-08-29T15:26:10.724875+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:26:10.724875+0000","last_undegraded":"2024-08-29T15:26:10.724875+0000","last_fullsized":"2024-08-29T15:26:10.724875+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":396,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764033+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:26:20.764033+0000","last_peered":"2024-08-29T15:26:20.764033+0000","last_clean":"2024-08-29T15:26:20.764033+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:26:20.764033+0000","last_undegraded":"2024-08-29T15:26:20.764033+0000","last_fullsized":"2024-08-29T15:26:20.764033+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.700720+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:26:01.700720+0000","last_peered":"2024-08-29T15:26:01.700720+0000","last_clean":"2024-08-29T15:26:01.700720+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:26:01.700720+0000","last_undegraded":"2024-08-29T15:26:01.700720+0000","last_fullsized":"2024-08-29T15:26:01.700720+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":168,"reported_epoch":275,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:10.719584+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:26:10.719584+0000","last_peered":"2024-08-29T15:26:10.719584+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:26:10.719584+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:26:10.719584+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.700617+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:26:01.700617+0000","last_peered":"2024-08-29T15:26:01.700617+0000","last_clean":"2024-08-29T15:26:01.700617+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:26:01.700617+0000","last_undegraded":"2024-08-29T15:26:01.700617+0000","last_fullsized":"2024-08-29T15:26:01.700617+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":174,"reported_epoch":275,"state":"active+clean","last_fresh":"2024-08-29T15:26:10.718935+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:26:10.718935+0000","last_peered":"2024-08-29T15:26:10.718935+0000","last_clean":"2024-08-29T15:26:10.718935+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:26:10.718935+0000","last_undegraded":"2024-08-29T15:26:10.718935+0000","last_fullsized":"2024-08-29T15:26:10.718935+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":444,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.750467+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:26:20.750467+0000","last_peered":"2024-08-29T15:26:20.750467+0000","last_clean":"2024-08-29T15:26:20.750467+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:26:20.750467+0000","last_undegraded":"2024-08-29T15:26:20.750467+0000","last_fullsized":"2024-08-29T15:26:20.750467+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":445,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.751512+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:26:20.751512+0000","last_peered":"2024-08-29T15:26:20.751512+0000","last_clean":"2024-08-29T15:26:20.751512+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:26:20.751512+0000","last_undegraded":"2024-08-29T15:26:20.751512+0000","last_fullsized":"2024-08-29T15:26:20.751512+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":54,"reported_epoch":274,"state":"active+clean","last_fresh":"2024-08-29T15:26:01.700478+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:26:01.700478+0000","last_peered":"2024-08-29T15:26:01.700478+0000","last_clean":"2024-08-29T15:26:01.700478+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:26:01.700478+0000","last_undegraded":"2024-08-29T15:26:01.700478+0000","last_fullsized":"2024-08-29T15:26:01.700478+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":453,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.764785+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:26:20.764785+0000","last_peered":"2024-08-29T15:26:20.764785+0000","last_clean":"2024-08-29T15:26:20.764785+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:26:20.764785+0000","last_undegraded":"2024-08-29T15:26:20.764785+0000","last_fullsized":"2024-08-29T15:26:20.764785+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3002531864,"num_objects":45816,"num_object_clones":0,"num_object_copies":137448,"num_objects_missing_on_primary":3811,"num_objects_missing":3811,"num_objects_degraded":27963,"num_objects_misplaced":8968,"num_objects_unfound":0,"num_objects_dirty":45816,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205034,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19145,"num_bytes_recovered":1134690304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9204477952,"data_stored":9204465736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124311,"ondisk_log_size":124311,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234380,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1807112,"kb_used_data":1487232,"kb_used_omap":1,"kb_used_meta":319870,"kb_avail":91938040,"statfs":{"total":95995035648,"available":94144552960,"internally_reserved":0,"allocated":1522925568,"data_stored":1521325787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":327547339},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496012,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":211284,"kb_used_data":17740,"kb_used_omap":1,"kb_used_meta":193534,"kb_avail":93533868,"statfs":{"total":95995035648,"available":95778680832,"internally_reserved":0,"allocated":18165760,"data_stored":16562131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":198179271},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,0,2,4,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426598,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1002872,"kb_used_data":929712,"kb_used_omap":8,"kb_used_meta":73143,"kb_avail":92742280,"statfs":{"total":95995035648,"available":94968094720,"internally_reserved":0,"allocated":952025088,"data_stored":950436573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":74899338},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019290,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1252840,"kb_used_data":932960,"kb_used_omap":1,"kb_used_meta":319870,"kb_avail":92492312,"statfs":{"total":95995035648,"available":94712127488,"internally_reserved":0,"allocated":955351040,"data_stored":953778308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":327547334},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435475,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2149448,"kb_used_data":1842940,"kb_used_omap":4,"kb_used_meta":306491,"kb_avail":91595704,"statfs":{"total":95995035648,"available":93794000896,"internally_reserved":0,"allocated":1887170560,"data_stored":1885578337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":313847049},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903327,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1903756,"kb_used_data":1753984,"kb_used_omap":10,"kb_used_meta":149749,"kb_avail":91841396,"statfs":{"total":95995035648,"available":94045589504,"internally_reserved":0,"allocated":1796079616,"data_stored":1794487329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":153343504},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762528,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1627188,"kb_used_data":1352492,"kb_used_omap":7,"kb_used_meta":274616,"kb_avail":92117964,"statfs":{"total":95995035648,"available":94328795136,"internally_reserved":0,"allocated":1384951808,"data_stored":1383365993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":281207138},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926055,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065676,"kb_used_data":736900,"kb_used_omap":7,"kb_used_meta":328760,"kb_avail":92679476,"statfs":{"total":95995035648,"available":94903783424,"internally_reserved":0,"allocated":754585600,"data_stored":752984916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":336650383},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":745017344,"data_stored":745013272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1377112064,"data_stored":1377107992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1792409600,"data_stored":1792409600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1875378176,"data_stored":1875378176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":945946624,"data_stored":945946624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":943001600,"data_stored":942997528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":13893632,"data_stored":13893632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1511718912,"data_stored":1511718912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:26:28.097 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:28.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:28.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:27 smithi171 ceph-mon[90031]: from='client.45947 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:27 smithi171 ceph-mon[90031]: pgmap v938: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27836/134889 objects degraded (20.636%); 8899/134889 objects misplaced (6.597%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 27836/134889 objects degraded (20.636%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:28.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:28.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:28.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:28.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:28.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:28.568 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:28.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:28.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:28.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:28.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:28.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:28.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:28.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:28.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:29.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:29.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:29.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:29.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:29.191 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:29.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:29.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:29.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:26:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:26:29] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-08-29T15:26:29.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:29.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:29.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:29.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:29.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:29.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:29.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:29.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:29.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:29.914 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:29 smithi027 ceph-mon[93921]: pgmap v939: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27836/134889 objects degraded (20.636%); 8899/134889 objects misplaced (6.597%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:29.914 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:29.914 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:29.915 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:29 smithi027 ceph-mon[98970]: pgmap v939: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27836/134889 objects degraded (20.636%); 8899/134889 objects misplaced (6.597%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:29.915 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:29.915 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:29.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:29.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:29.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:29.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:29 smithi171 ceph-mon[90031]: pgmap v939: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 233 op/s; 27836/134889 objects degraded (20.636%); 8899/134889 objects misplaced (6.597%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:30.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:30.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:30.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:30.169 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:26:30.169 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:26:30.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:30.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:30.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:30.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:30.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:30.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:30.570 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:30.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:30.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:30.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:30.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:30.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:30.719 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:26:30.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:30.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:30.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:30.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:30.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3280007425' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:26:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:26:31.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3280007425' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:26:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:26:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3280007425' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:26:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:26:31.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:31.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:26:31.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:31.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:31.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:31.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:31.425 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:31.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:31.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:31.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:31.572 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:31.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:31.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:31.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:31.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:26:31.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:31.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:31.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:31.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:31.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:31.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:31 smithi171 ceph-mon[90031]: pgmap v940: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 27836/133869 objects degraded (20.793%); 8822/133869 objects misplaced (6.590%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:31 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:26:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:31 smithi171 ceph-mon[90031]: osdmap e277: 8 total, 8 up, 8 in 2024-08-29T15:26:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:26:32.044 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[93921]: pgmap v940: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 27836/133869 objects degraded (20.793%); 8822/133869 objects misplaced (6.590%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:32.044 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[93921]: osdmap e277: 8 total, 8 up, 8 in 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[98970]: pgmap v940: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 230 op/s; 27836/133869 objects degraded (20.793%); 8822/133869 objects misplaced (6.590%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[98970]: osdmap e277: 8 total, 8 up, 8 in 2024-08-29T15:26:32.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:26:32.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:32.075 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:32.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:32.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:32.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:32.328 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:32.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:32.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:32.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:32.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:32.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:32.579 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:32.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:32.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:32.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 27836/133869 objects degraded (20.793%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 27836/133869 objects degraded (20.793%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:32.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:32.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:32.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:32.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:32.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:33.006 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 27836/133869 objects degraded (20.793%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:33.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:33.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:33.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:33.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:33.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:33.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:33.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:33.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:33.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:33.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:33.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:33.492 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:33.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:33.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:33.621 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:33.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:33.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:33.756 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:33.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:33.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:33.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:33.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:33.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:33.935 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:26:33.936 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:26:33.937 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:26:33.937 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:26:33.937 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:26:33.937 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:26:33.937 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:26:33.937 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:26:33.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:34.005 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:33 smithi171 ceph-mon[90031]: pgmap v942: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 27691/131604 objects degraded (21.041%); 8758/131604 objects misplaced (6.655%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:34.005 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4175549019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:34.020 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:33 smithi027 ceph-mon[98970]: pgmap v942: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 27691/131604 objects degraded (21.041%); 8758/131604 objects misplaced (6.655%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4175549019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:33 smithi027 ceph-mon[93921]: pgmap v942: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 220 op/s; 27691/131604 objects degraded (21.041%); 8758/131604 objects misplaced (6.655%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4175549019' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:34.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:34.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:34.154 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:34.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:34.272 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:34.374 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:34.399 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:34.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:34.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:34.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:34.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:34.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:34.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:34.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.778 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:34.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:34.803 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:34.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:34.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:34.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:34.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:35.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:35.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:35.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:35.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:35.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:35.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:35.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:35.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:35.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:35.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:35.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:35.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:35.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:35.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:35.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:35.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:26:35.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:35.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:35.806 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:35 smithi027 ceph-mon[98970]: pgmap v943: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 27691/130980 objects degraded (21.141%); 8682/130980 objects misplaced (6.628%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:35.807 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:35 smithi027 ceph-mon[93921]: pgmap v943: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 27691/130980 objects degraded (21.141%); 8682/130980 objects misplaced (6.628%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:35.888 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:35.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:35.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:36.012 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:35 smithi171 ceph-mon[90031]: pgmap v943: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 218 op/s; 27691/130980 objects degraded (21.141%); 8682/130980 objects misplaced (6.628%); 1.8 MiB/s, 29 objects/s recovering 2024-08-29T15:26:36.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:36.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:36.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:36.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:36.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:36.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:36.425 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:36.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:36.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:36.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:36.652 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:36.673 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:36.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:36.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:26:36.774 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:36.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:36.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:36.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:36.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:37.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:37.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:37.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:37.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:37.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:37.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:37.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:37.312 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:37.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:37.442 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:37 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:37.442 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:37 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:37.442 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 27691/130980 objects degraded (21.141%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:37.442 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:37 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:37.442 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:37 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:37.442 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 27691/130980 objects degraded (21.141%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:37.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:37 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:37 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 27691/130980 objects degraded (21.141%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:37.549 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:37.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:37.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:37.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:37.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:37.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:37.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:37.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:37.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:37.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:38.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:38.089 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:38.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:38.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:38.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:38.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:38 smithi027 ceph-mon[98970]: pgmap v944: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 27542/127947 objects degraded (21.526%); 8598/127947 objects misplaced (6.720%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:38 smithi027 ceph-mon[93921]: pgmap v944: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 27542/127947 objects degraded (21.526%); 8598/127947 objects misplaced (6.720%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:38.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:38.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:38.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:38.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:38.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:38 smithi171 ceph-mon[90031]: pgmap v944: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 27542/127947 objects degraded (21.526%); 8598/127947 objects misplaced (6.720%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:38.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:38.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:38.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:38.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:38.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:38.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:38.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:38.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:38.874 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:26:38.878 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:26:38.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:38.975 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:38.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:38.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:39.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:39.103 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:39.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.228 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:39.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:26:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:26:39] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-08-29T15:26:39.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:39.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:39.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:39.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:39.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:39.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:39.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:39.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:39.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:39.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:39.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:39.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:39.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:39.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:39.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:39.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:40.090 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:40.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:40.117 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:40.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:40.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:40.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:40.242 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:40.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:40.343 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:40.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:40.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:40.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:40 smithi027 ceph-mon[98970]: pgmap v945: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 27542/127947 objects degraded (21.526%); 8598/127947 objects misplaced (6.720%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:40.440 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:40 smithi027 ceph-mon[98970]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:40.440 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:40 smithi027 ceph-mon[93921]: pgmap v945: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 27542/127947 objects degraded (21.526%); 8598/127947 objects misplaced (6.720%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:40.440 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:40 smithi027 ceph-mon[93921]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:40.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:40.497 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:40.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:40 smithi171 ceph-mon[90031]: pgmap v945: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 231 op/s; 27542/127947 objects degraded (21.526%); 8598/127947 objects misplaced (6.720%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:40 smithi171 ceph-mon[90031]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:26:40.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:40.625 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:40.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:40.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:40.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:40.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:40.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:40.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:40.882 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:40.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:40.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:41.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.007 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:41.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:41.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:41.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:41.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:41.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:41.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:41.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:41.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.386 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:41.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:41.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:41.511 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:41.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:41.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:41.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:41.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:41.650 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":945,"stamp":"2024-08-29T15:26:38.652297+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2797934589,"num_objects":42649,"num_object_clones":0,"num_object_copies":127947,"num_objects_missing_on_primary":3666,"num_objects_missing":3666,"num_objects_degraded":27542,"num_objects_misplaced":8598,"num_objects_unfound":0,"num_objects_dirty":42649,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":209352,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19625,"num_bytes_recovered":1164971213,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125488,"ondisk_log_size":125488,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10440648,"kb_used_data":8466504,"kb_used_omap":43,"kb_used_meta":1974036,"kb_avail":739520568,"statfs":{"total":767960285184,"available":757269061632,"internally_reserved":0,"allocated":8669700096,"data_stored":8656872654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2021413356},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-151650304,"num_objects":-2314,"num_object_clones":0,"num_object_copies":-6942,"num_objects_missing_on_primary":-153,"num_objects_missing":-153,"num_objects_degraded":-294,"num_objects_misplaced":-301,"num_objects_unfound":0,"num_objects_dirty":-2314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2314,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":19595264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002862"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":440,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.716461+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:26:30.716461+0000","last_peered":"2024-08-29T15:26:30.716461+0000","last_clean":"2024-08-29T15:26:30.716461+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:26:30.716461+0000","last_undegraded":"2024-08-29T15:26:30.716461+0000","last_fullsized":"2024-08-29T15:26:30.716461+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":55,"reported_epoch":276,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:20.752502+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:26:20.752502+0000","last_peered":"2024-08-29T15:26:20.752502+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:26:20.752502+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:26:20.752502+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":145,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.746926+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:26:20.746926+0000","last_peered":"2024-08-29T15:26:20.746926+0000","last_clean":"2024-08-29T15:26:20.746926+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:26:20.746926+0000","last_undegraded":"2024-08-29T15:26:20.746926+0000","last_fullsized":"2024-08-29T15:26:20.746926+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":406,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.746407+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:26:30.746407+0000","last_peered":"2024-08-29T15:26:30.746407+0000","last_clean":"2024-08-29T15:26:30.746407+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:26:30.746407+0000","last_undegraded":"2024-08-29T15:26:30.746407+0000","last_fullsized":"2024-08-29T15:26:30.746407+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":405,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743662+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:26:30.743662+0000","last_peered":"2024-08-29T15:26:30.743662+0000","last_clean":"2024-08-29T15:26:30.743662+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:26:30.743662+0000","last_undegraded":"2024-08-29T15:26:30.743662+0000","last_fullsized":"2024-08-29T15:26:30.743662+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752399+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:26:20.752399+0000","last_peered":"2024-08-29T15:26:20.752399+0000","last_clean":"2024-08-29T15:26:20.752399+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:26:20.752399+0000","last_undegraded":"2024-08-29T15:26:20.752399+0000","last_fullsized":"2024-08-29T15:26:20.752399+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":217,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.719209+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:26:30.719209+0000","last_peered":"2024-08-29T15:26:30.719209+0000","last_clean":"2024-08-29T15:26:30.719209+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:26:30.719209+0000","last_undegraded":"2024-08-29T15:26:30.719209+0000","last_fullsized":"2024-08-29T15:26:30.719209+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":147,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.779949+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:26:20.779949+0000","last_peered":"2024-08-29T15:26:20.779949+0000","last_clean":"2024-08-29T15:26:20.779949+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:26:20.779949+0000","last_undegraded":"2024-08-29T15:26:20.779949+0000","last_fullsized":"2024-08-29T15:26:20.779949+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"277'13176","reported_seq":21233,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:36.139758+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:26:36.139758+0000","last_peered":"2024-08-29T15:26:36.139758+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:26:36.139758+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"178'3100","ondisk_log_start":"178'3100","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179306496,"num_objects":2736,"num_object_clones":0,"num_object_copies":8208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13176,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":226},{"shards":"0,5","objects":-189}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":55,"reported_epoch":277,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:30.716053+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:26:30.716053+0000","last_peered":"2024-08-29T15:26:30.716053+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:26:30.716053+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:26:30.716053+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:21.192775+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:26:21.192775+0000","last_peered":"2024-08-29T15:26:21.192775+0000","last_clean":"2024-08-29T15:26:21.192775+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:26:21.192775+0000","last_undegraded":"2024-08-29T15:26:21.192775+0000","last_fullsized":"2024-08-29T15:26:21.192775+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"277'12871","reported_seq":13105,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:36.336745+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:26:36.336745+0000","last_peered":"2024-08-29T15:26:36.336745+0000","last_clean":"2024-08-29T15:26:36.336745+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:26:36.336745+0000","last_undegraded":"2024-08-29T15:26:36.336745+0000","last_fullsized":"2024-08-29T15:26:36.336745+0000","mapping_epoch":168,"log_start":"187'6128","ondisk_log_start":"187'6128","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175570944,"num_objects":2679,"num_object_clones":0,"num_object_copies":8037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12871,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":55,"reported_epoch":277,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:30.717869+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:26:30.717869+0000","last_peered":"2024-08-29T15:26:30.717869+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:26:30.717869+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:26:30.717869+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":57,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752264+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:26:20.752264+0000","last_peered":"2024-08-29T15:26:20.752264+0000","last_clean":"2024-08-29T15:26:20.752264+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:26:20.752264+0000","last_undegraded":"2024-08-29T15:26:20.752264+0000","last_fullsized":"2024-08-29T15:26:20.752264+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"277'13010","reported_seq":13213,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:36.348976+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:26:36.348976+0000","last_peered":"2024-08-29T15:26:36.348976+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:26:36.348976+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"187'6274","ondisk_log_start":"187'6274","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177078272,"num_objects":2702,"num_object_clones":0,"num_object_copies":8106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13010,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":225}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":222,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743461+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:26:30.743461+0000","last_peered":"2024-08-29T15:26:30.743461+0000","last_clean":"2024-08-29T15:26:30.743461+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:26:30.743461+0000","last_undegraded":"2024-08-29T15:26:30.743461+0000","last_fullsized":"2024-08-29T15:26:30.743461+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":102,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.780214+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:26:20.780214+0000","last_peered":"2024-08-29T15:26:20.780214+0000","last_clean":"2024-08-29T15:26:20.780214+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:26:20.780214+0000","last_undegraded":"2024-08-29T15:26:20.780214+0000","last_fullsized":"2024-08-29T15:26:20.780214+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"277'13079","reported_seq":18533,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:33.855402+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:26:33.855402+0000","last_peered":"2024-08-29T15:26:33.855402+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:26:33.855402+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"180'4962","ondisk_log_start":"180'4962","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8117,"log_dups_size":0,"ondisk_log_size":8117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176881664,"num_objects":2699,"num_object_clones":0,"num_object_copies":8097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13079,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2699}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":164,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.713770+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:26:30.713770+0000","last_peered":"2024-08-29T15:26:30.713770+0000","last_clean":"2024-08-29T15:26:30.713770+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:26:30.713770+0000","last_undegraded":"2024-08-29T15:26:30.713770+0000","last_fullsized":"2024-08-29T15:26:30.713770+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":420,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743207+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:26:30.743207+0000","last_peered":"2024-08-29T15:26:30.743207+0000","last_clean":"2024-08-29T15:26:30.743207+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:26:30.743207+0000","last_undegraded":"2024-08-29T15:26:30.743207+0000","last_fullsized":"2024-08-29T15:26:30.743207+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"277'13074","reported_seq":13239,"reported_epoch":277,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:34.792881+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:26:34.792881+0000","last_peered":"2024-08-29T15:26:34.792881+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:26:34.792881+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:26:34.792881+0000","mapping_epoch":234,"log_start":"180'5133","ondisk_log_start":"180'5133","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7941,"log_dups_size":0,"ondisk_log_size":7941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179568640,"num_objects":2740,"num_object_clones":0,"num_object_copies":8220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13074,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,2,4","objects":263},{"shards":"1,4","objects":2477}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":414,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.716159+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:26:30.716159+0000","last_peered":"2024-08-29T15:26:30.716159+0000","last_clean":"2024-08-29T15:26:30.716159+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:26:30.716159+0000","last_undegraded":"2024-08-29T15:26:30.716159+0000","last_fullsized":"2024-08-29T15:26:30.716159+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":57,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.714897+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:26:30.714897+0000","last_peered":"2024-08-29T15:26:30.714897+0000","last_clean":"2024-08-29T15:26:30.714897+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:26:30.714897+0000","last_undegraded":"2024-08-29T15:26:30.714897+0000","last_fullsized":"2024-08-29T15:26:30.714897+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"277'13137","reported_seq":13387,"reported_epoch":277,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:36.341934+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:26:36.341934+0000","last_peered":"2024-08-29T15:26:36.341934+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:26:36.341934+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:26:36.341934+0000","mapping_epoch":234,"log_start":"187'6468","ondisk_log_start":"187'6468","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13137,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":240},{"shards":"3,4","objects":2465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":120,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.780113+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:26:20.780113+0000","last_peered":"2024-08-29T15:26:20.780113+0000","last_clean":"2024-08-29T15:26:20.780113+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:26:20.780113+0000","last_undegraded":"2024-08-29T15:26:20.780113+0000","last_fullsized":"2024-08-29T15:26:20.780113+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":57,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.715342+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:26:30.715342+0000","last_peered":"2024-08-29T15:26:30.715342+0000","last_clean":"2024-08-29T15:26:30.715342+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:26:30.715342+0000","last_undegraded":"2024-08-29T15:26:30.715342+0000","last_fullsized":"2024-08-29T15:26:30.715342+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"277'13004","reported_seq":13438,"reported_epoch":277,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:26:36.455050+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:26:36.455050+0000","last_peered":"2024-08-29T15:26:36.455050+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:26:36.455050+0000","last_undegraded":"2024-08-29T15:26:36.455050+0000","last_fullsized":"2024-08-29T15:26:36.455050+0000","mapping_epoch":202,"log_start":"187'6267","ondisk_log_start":"187'6267","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6737,"log_dups_size":0,"ondisk_log_size":6737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173932544,"num_objects":2654,"num_object_clones":0,"num_object_copies":7962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2654,"num_objects_unfound":0,"num_objects_dirty":2654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13004,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":2654}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752259+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:26:20.752259+0000","last_peered":"2024-08-29T15:26:20.752259+0000","last_clean":"2024-08-29T15:26:20.752259+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:26:20.752259+0000","last_undegraded":"2024-08-29T15:26:20.752259+0000","last_fullsized":"2024-08-29T15:26:20.752259+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":262,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.754978+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:26:20.754978+0000","last_peered":"2024-08-29T15:26:20.754978+0000","last_clean":"2024-08-29T15:26:20.754978+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:26:20.754978+0000","last_undegraded":"2024-08-29T15:26:20.754978+0000","last_fullsized":"2024-08-29T15:26:20.754978+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"277'13140","reported_seq":13298,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:35.939798+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:26:35.939798+0000","last_peered":"2024-08-29T15:26:35.939798+0000","last_clean":"2024-08-29T15:26:35.939798+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:26:35.939798+0000","last_undegraded":"2024-08-29T15:26:35.939798+0000","last_fullsized":"2024-08-29T15:26:35.939798+0000","mapping_epoch":218,"log_start":"178'3078","ondisk_log_start":"178'3078","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171638808,"num_objects":2620,"num_object_clones":0,"num_object_copies":7860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2620,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13140,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":418,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.716707+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:26:30.716707+0000","last_peered":"2024-08-29T15:26:30.716707+0000","last_clean":"2024-08-29T15:26:30.716707+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:26:30.716707+0000","last_undegraded":"2024-08-29T15:26:30.716707+0000","last_fullsized":"2024-08-29T15:26:30.716707+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":410,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.719288+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:26:30.719288+0000","last_peered":"2024-08-29T15:26:30.719288+0000","last_clean":"2024-08-29T15:26:30.719288+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:26:30.719288+0000","last_undegraded":"2024-08-29T15:26:30.719288+0000","last_fullsized":"2024-08-29T15:26:30.719288+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"277'12527","reported_seq":12734,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:36.016455+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:26:36.016455+0000","last_peered":"2024-08-29T15:26:36.016455+0000","last_clean":"2024-08-29T15:26:36.016455+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:26:36.016455+0000","last_undegraded":"2024-08-29T15:26:36.016455+0000","last_fullsized":"2024-08-29T15:26:36.016455+0000","mapping_epoch":201,"log_start":"177'2500","ondisk_log_start":"177'2500","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10027,"log_dups_size":0,"ondisk_log_size":10027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175702016,"num_objects":2681,"num_object_clones":0,"num_object_copies":8043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12527,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":509,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743073+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:26:30.743073+0000","last_peered":"2024-08-29T15:26:30.743073+0000","last_clean":"2024-08-29T15:26:30.743073+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:26:30.743073+0000","last_undegraded":"2024-08-29T15:26:30.743073+0000","last_fullsized":"2024-08-29T15:26:30.743073+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":748,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.746967+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:26:20.746967+0000","last_peered":"2024-08-29T15:26:20.746967+0000","last_clean":"2024-08-29T15:26:20.746967+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:26:20.746967+0000","last_undegraded":"2024-08-29T15:26:20.746967+0000","last_fullsized":"2024-08-29T15:26:20.746967+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":256,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.746892+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:26:20.746892+0000","last_peered":"2024-08-29T15:26:20.746892+0000","last_clean":"2024-08-29T15:26:20.746892+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:26:20.746892+0000","last_undegraded":"2024-08-29T15:26:20.746892+0000","last_fullsized":"2024-08-29T15:26:20.746892+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":118,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.780672+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:26:20.780672+0000","last_peered":"2024-08-29T15:26:20.780672+0000","last_clean":"2024-08-29T15:26:20.780672+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:26:20.780672+0000","last_undegraded":"2024-08-29T15:26:20.780672+0000","last_fullsized":"2024-08-29T15:26:20.780672+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"277'13029","reported_seq":13461,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:35.746849+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:26:35.746849+0000","last_peered":"2024-08-29T15:26:35.746849+0000","last_clean":"2024-08-29T15:26:35.746849+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:26:35.746849+0000","last_undegraded":"2024-08-29T15:26:35.746849+0000","last_fullsized":"2024-08-29T15:26:35.746849+0000","mapping_epoch":201,"log_start":"186'5953","ondisk_log_start":"186'5953","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173867008,"num_objects":2653,"num_object_clones":0,"num_object_copies":7959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13029,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":883,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.718569+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:26:30.718569+0000","last_peered":"2024-08-29T15:26:30.718569+0000","last_clean":"2024-08-29T15:26:30.718569+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:26:30.718569+0000","last_undegraded":"2024-08-29T15:26:30.718569+0000","last_fullsized":"2024-08-29T15:26:30.718569+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":421,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743758+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:26:30.743758+0000","last_peered":"2024-08-29T15:26:30.743758+0000","last_clean":"2024-08-29T15:26:30.743758+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:26:30.743758+0000","last_undegraded":"2024-08-29T15:26:30.743758+0000","last_fullsized":"2024-08-29T15:26:30.743758+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":428,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743917+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:26:30.743917+0000","last_peered":"2024-08-29T15:26:30.743917+0000","last_clean":"2024-08-29T15:26:30.743917+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:26:30.743917+0000","last_undegraded":"2024-08-29T15:26:30.743917+0000","last_fullsized":"2024-08-29T15:26:30.743917+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"277'13027","reported_seq":22532,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:36.445307+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:26:36.445307+0000","last_peered":"2024-08-29T15:26:36.445307+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:26:36.445307+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"187'6330","ondisk_log_start":"187'6330","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174522368,"num_objects":2663,"num_object_clones":0,"num_object_copies":7989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3140,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13027,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4651,"num_bytes_recovered":304807936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":225},{"shards":"3,6","objects":54},{"shards":"3,7","objects":2384}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":691,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.715790+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:26:30.715790+0000","last_peered":"2024-08-29T15:26:30.715790+0000","last_clean":"2024-08-29T15:26:30.715790+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:26:30.715790+0000","last_undegraded":"2024-08-29T15:26:30.715790+0000","last_fullsized":"2024-08-29T15:26:30.715790+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":109,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:24.094072+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:26:24.094072+0000","last_peered":"2024-08-29T15:26:24.094072+0000","last_clean":"2024-08-29T15:26:24.094072+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:26:24.094072+0000","last_undegraded":"2024-08-29T15:26:24.094072+0000","last_fullsized":"2024-08-29T15:26:24.094072+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":455,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.719078+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:26:30.719078+0000","last_peered":"2024-08-29T15:26:30.719078+0000","last_clean":"2024-08-29T15:26:30.719078+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:26:30.719078+0000","last_undegraded":"2024-08-29T15:26:30.719078+0000","last_fullsized":"2024-08-29T15:26:30.719078+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"277'13092","reported_seq":13336,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:36.367850+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:26:36.367850+0000","last_peered":"2024-08-29T15:26:36.367850+0000","last_clean":"2024-08-29T15:26:36.367850+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:26:36.367850+0000","last_undegraded":"2024-08-29T15:26:36.367850+0000","last_fullsized":"2024-08-29T15:26:36.367850+0000","mapping_epoch":234,"log_start":"187'6397","ondisk_log_start":"187'6397","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6695,"log_dups_size":0,"ondisk_log_size":6695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171704320,"num_objects":2620,"num_object_clones":0,"num_object_copies":7860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13092,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":216,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.746789+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:26:20.746789+0000","last_peered":"2024-08-29T15:26:20.746789+0000","last_clean":"2024-08-29T15:26:20.746789+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:26:20.746789+0000","last_undegraded":"2024-08-29T15:26:20.746789+0000","last_fullsized":"2024-08-29T15:26:20.746789+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752507+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:26:20.752507+0000","last_peered":"2024-08-29T15:26:20.752507+0000","last_clean":"2024-08-29T15:26:20.752507+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:26:20.752507+0000","last_undegraded":"2024-08-29T15:26:20.752507+0000","last_fullsized":"2024-08-29T15:26:20.752507+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":123,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.746884+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:26:20.746884+0000","last_peered":"2024-08-29T15:26:20.746884+0000","last_clean":"2024-08-29T15:26:20.746884+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:26:20.746884+0000","last_undegraded":"2024-08-29T15:26:20.746884+0000","last_fullsized":"2024-08-29T15:26:20.746884+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"277'13129","reported_seq":13371,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:33.961440+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:26:33.961440+0000","last_peered":"2024-08-29T15:26:33.961440+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:26:33.961440+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"180'4942","ondisk_log_start":"180'4942","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176750592,"num_objects":2697,"num_object_clones":0,"num_object_copies":8091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13129,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":274}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.747185+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:26:20.747185+0000","last_peered":"2024-08-29T15:26:20.747185+0000","last_clean":"2024-08-29T15:26:20.747185+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:26:20.747185+0000","last_undegraded":"2024-08-29T15:26:20.747185+0000","last_fullsized":"2024-08-29T15:26:20.747185+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":848,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.720479+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:26:30.720479+0000","last_peered":"2024-08-29T15:26:30.720479+0000","last_clean":"2024-08-29T15:26:30.720479+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:26:30.720479+0000","last_undegraded":"2024-08-29T15:26:30.720479+0000","last_fullsized":"2024-08-29T15:26:30.720479+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":425,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.719598+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:26:30.719598+0000","last_peered":"2024-08-29T15:26:30.719598+0000","last_clean":"2024-08-29T15:26:30.719598+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:26:30.719598+0000","last_undegraded":"2024-08-29T15:26:30.719598+0000","last_fullsized":"2024-08-29T15:26:30.719598+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"277'13050","reported_seq":13848,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:35.589040+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:26:35.589040+0000","last_peered":"2024-08-29T15:26:35.589040+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:26:35.589040+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"186'5980","ondisk_log_start":"186'5980","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171573248,"num_objects":2618,"num_object_clones":0,"num_object_copies":7854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2600,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":2618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13050,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":467}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":430,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.720399+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:26:30.720399+0000","last_peered":"2024-08-29T15:26:30.720399+0000","last_clean":"2024-08-29T15:26:30.720399+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:26:30.720399+0000","last_undegraded":"2024-08-29T15:26:30.720399+0000","last_fullsized":"2024-08-29T15:26:30.720399+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":705,"reported_epoch":276,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:20.750591+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:26:20.750591+0000","last_peered":"2024-08-29T15:26:20.750591+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:26:20.750591+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:26:20.750591+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":421,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743094+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:26:30.743094+0000","last_peered":"2024-08-29T15:26:30.743094+0000","last_clean":"2024-08-29T15:26:30.743094+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:26:30.743094+0000","last_undegraded":"2024-08-29T15:26:30.743094+0000","last_fullsized":"2024-08-29T15:26:30.743094+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"277'12897","reported_seq":20060,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:34.672018+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:26:34.672018+0000","last_peered":"2024-08-29T15:26:34.672018+0000","last_clean":"2024-08-29T15:26:34.672018+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:26:34.672018+0000","last_undegraded":"2024-08-29T15:26:34.672018+0000","last_fullsized":"2024-08-29T15:26:34.672018+0000","mapping_epoch":234,"log_start":"180'4999","ondisk_log_start":"180'4999","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7898,"log_dups_size":0,"ondisk_log_size":7898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165871616,"num_objects":2531,"num_object_clones":0,"num_object_copies":7593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12897,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":415,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.717719+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:26:30.717719+0000","last_peered":"2024-08-29T15:26:30.717719+0000","last_clean":"2024-08-29T15:26:30.717719+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:26:30.717719+0000","last_undegraded":"2024-08-29T15:26:30.717719+0000","last_fullsized":"2024-08-29T15:26:30.717719+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1037,"reported_epoch":277,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:30.718274+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:26:30.718274+0000","last_peered":"2024-08-29T15:26:30.718274+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:26:30.718274+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":412,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.716581+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:26:30.716581+0000","last_peered":"2024-08-29T15:26:30.716581+0000","last_clean":"2024-08-29T15:26:30.716581+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:26:30.716581+0000","last_undegraded":"2024-08-29T15:26:30.716581+0000","last_fullsized":"2024-08-29T15:26:30.716581+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"277'12987","reported_seq":21292,"reported_epoch":277,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:34.197202+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:26:34.197202+0000","last_peered":"2024-08-29T15:26:34.197202+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:26:34.197202+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"181'5412","ondisk_log_start":"181'5412","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171900928,"num_objects":2623,"num_object_clones":0,"num_object_copies":7869,"num_objects_missing_on_primary":3658,"num_objects_missing":3658,"num_objects_degraded":2313,"num_objects_misplaced":2609,"num_objects_unfound":0,"num_objects_dirty":2623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12987,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4064,"num_bytes_recovered":266207232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":13},{"shards":"1,2,7","objects":238},{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":2313},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":410,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743265+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:26:30.743265+0000","last_peered":"2024-08-29T15:26:30.743265+0000","last_clean":"2024-08-29T15:26:30.743265+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:26:30.743265+0000","last_undegraded":"2024-08-29T15:26:30.743265+0000","last_fullsized":"2024-08-29T15:26:30.743265+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":129,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.780364+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:26:20.780364+0000","last_peered":"2024-08-29T15:26:20.780364+0000","last_clean":"2024-08-29T15:26:20.780364+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:26:20.780364+0000","last_undegraded":"2024-08-29T15:26:20.780364+0000","last_fullsized":"2024-08-29T15:26:20.780364+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752349+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:26:20.752349+0000","last_peered":"2024-08-29T15:26:20.752349+0000","last_clean":"2024-08-29T15:26:20.752349+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:26:20.752349+0000","last_undegraded":"2024-08-29T15:26:20.752349+0000","last_fullsized":"2024-08-29T15:26:20.752349+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":56,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752578+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:26:20.752578+0000","last_peered":"2024-08-29T15:26:20.752578+0000","last_clean":"2024-08-29T15:26:20.752578+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:26:20.752578+0000","last_undegraded":"2024-08-29T15:26:20.752578+0000","last_fullsized":"2024-08-29T15:26:20.752578+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":54,"reported_epoch":276,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:20.752521+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:26:20.752521+0000","last_peered":"2024-08-29T15:26:20.752521+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:26:20.752521+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:26:20.752521+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":419,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.719477+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:26:30.719477+0000","last_peered":"2024-08-29T15:26:30.719477+0000","last_clean":"2024-08-29T15:26:30.719477+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:26:30.719477+0000","last_undegraded":"2024-08-29T15:26:30.719477+0000","last_fullsized":"2024-08-29T15:26:30.719477+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":169,"reported_epoch":277,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:30.717019+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:26:30.717019+0000","last_peered":"2024-08-29T15:26:30.717019+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:26:30.717019+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:26:30.717019+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":145,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.747590+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:26:20.747590+0000","last_peered":"2024-08-29T15:26:20.747590+0000","last_clean":"2024-08-29T15:26:20.747590+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:26:20.747590+0000","last_undegraded":"2024-08-29T15:26:20.747590+0000","last_fullsized":"2024-08-29T15:26:20.747590+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":378,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.715629+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:26:30.715629+0000","last_peered":"2024-08-29T15:26:30.715629+0000","last_clean":"2024-08-29T15:26:30.715629+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:26:30.715629+0000","last_undegraded":"2024-08-29T15:26:30.715629+0000","last_fullsized":"2024-08-29T15:26:30.715629+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":431,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743762+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:26:30.743762+0000","last_peered":"2024-08-29T15:26:30.743762+0000","last_clean":"2024-08-29T15:26:30.743762+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:26:30.743762+0000","last_undegraded":"2024-08-29T15:26:30.743762+0000","last_fullsized":"2024-08-29T15:26:30.743762+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":430,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.716509+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:26:30.716509+0000","last_peered":"2024-08-29T15:26:30.716509+0000","last_clean":"2024-08-29T15:26:30.716509+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:26:30.716509+0000","last_undegraded":"2024-08-29T15:26:30.716509+0000","last_fullsized":"2024-08-29T15:26:30.716509+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":117,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.779696+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:26:20.779696+0000","last_peered":"2024-08-29T15:26:20.779696+0000","last_clean":"2024-08-29T15:26:20.779696+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:26:20.779696+0000","last_undegraded":"2024-08-29T15:26:20.779696+0000","last_fullsized":"2024-08-29T15:26:20.779696+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":287,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.742959+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:26:30.742959+0000","last_peered":"2024-08-29T15:26:30.742959+0000","last_clean":"2024-08-29T15:26:30.742959+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:26:30.742959+0000","last_undegraded":"2024-08-29T15:26:30.742959+0000","last_fullsized":"2024-08-29T15:26:30.742959+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":407,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743830+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:26:30.743830+0000","last_peered":"2024-08-29T15:26:30.743830+0000","last_clean":"2024-08-29T15:26:30.743830+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:26:30.743830+0000","last_undegraded":"2024-08-29T15:26:30.743830+0000","last_fullsized":"2024-08-29T15:26:30.743830+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":410,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.718809+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:26:30.718809+0000","last_peered":"2024-08-29T15:26:30.718809+0000","last_clean":"2024-08-29T15:26:30.718809+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:26:30.718809+0000","last_undegraded":"2024-08-29T15:26:30.718809+0000","last_fullsized":"2024-08-29T15:26:30.718809+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":415,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.716908+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:26:30.716908+0000","last_peered":"2024-08-29T15:26:30.716908+0000","last_clean":"2024-08-29T15:26:30.716908+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:26:30.716908+0000","last_undegraded":"2024-08-29T15:26:30.716908+0000","last_fullsized":"2024-08-29T15:26:30.716908+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":56,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752470+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:26:20.752470+0000","last_peered":"2024-08-29T15:26:20.752470+0000","last_clean":"2024-08-29T15:26:20.752470+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:26:20.752470+0000","last_undegraded":"2024-08-29T15:26:20.752470+0000","last_fullsized":"2024-08-29T15:26:20.752470+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":435,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.720559+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:26:30.720559+0000","last_peered":"2024-08-29T15:26:30.720559+0000","last_clean":"2024-08-29T15:26:30.720559+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:26:30.720559+0000","last_undegraded":"2024-08-29T15:26:30.720559+0000","last_fullsized":"2024-08-29T15:26:30.720559+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":398,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743075+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:26:30.743075+0000","last_peered":"2024-08-29T15:26:30.743075+0000","last_clean":"2024-08-29T15:26:30.743075+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:26:30.743075+0000","last_undegraded":"2024-08-29T15:26:30.743075+0000","last_fullsized":"2024-08-29T15:26:30.743075+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":56,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752308+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:26:20.752308+0000","last_peered":"2024-08-29T15:26:20.752308+0000","last_clean":"2024-08-29T15:26:20.752308+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:26:20.752308+0000","last_undegraded":"2024-08-29T15:26:20.752308+0000","last_fullsized":"2024-08-29T15:26:20.752308+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":169,"reported_epoch":276,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:20.747008+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:26:20.747008+0000","last_peered":"2024-08-29T15:26:20.747008+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:26:20.747008+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:26:20.747008+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752289+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:26:20.752289+0000","last_peered":"2024-08-29T15:26:20.752289+0000","last_clean":"2024-08-29T15:26:20.752289+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:26:20.752289+0000","last_undegraded":"2024-08-29T15:26:20.752289+0000","last_fullsized":"2024-08-29T15:26:20.752289+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":175,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.747269+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:26:20.747269+0000","last_peered":"2024-08-29T15:26:20.747269+0000","last_clean":"2024-08-29T15:26:20.747269+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:26:20.747269+0000","last_undegraded":"2024-08-29T15:26:20.747269+0000","last_fullsized":"2024-08-29T15:26:20.747269+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":446,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.715827+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:26:30.715827+0000","last_peered":"2024-08-29T15:26:30.715827+0000","last_clean":"2024-08-29T15:26:30.715827+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:26:30.715827+0000","last_undegraded":"2024-08-29T15:26:30.715827+0000","last_fullsized":"2024-08-29T15:26:30.715827+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":447,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.716517+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:26:30.716517+0000","last_peered":"2024-08-29T15:26:30.716517+0000","last_clean":"2024-08-29T15:26:30.716517+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:26:30.716517+0000","last_undegraded":"2024-08-29T15:26:30.716517+0000","last_fullsized":"2024-08-29T15:26:30.716517+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":56,"reported_epoch":276,"state":"active+clean","last_fresh":"2024-08-29T15:26:20.752180+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:26:20.752180+0000","last_peered":"2024-08-29T15:26:20.752180+0000","last_clean":"2024-08-29T15:26:20.752180+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:26:20.752180+0000","last_undegraded":"2024-08-29T15:26:20.752180+0000","last_fullsized":"2024-08-29T15:26:20.752180+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":455,"reported_epoch":277,"state":"active+clean","last_fresh":"2024-08-29T15:26:30.743715+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:26:30.743715+0000","last_peered":"2024-08-29T15:26:30.743715+0000","last_clean":"2024-08-29T15:26:30.743715+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:26:30.743715+0000","last_undegraded":"2024-08-29T15:26:30.743715+0000","last_fullsized":"2024-08-29T15:26:30.743715+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2793144344,"num_objects":42621,"num_object_clones":0,"num_object_copies":127863,"num_objects_missing_on_primary":3658,"num_objects_missing":3658,"num_objects_degraded":27513,"num_objects_misplaced":8598,"num_objects_unfound":0,"num_objects_dirty":42621,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":208229,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19516,"num_bytes_recovered":1159004160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8593813504,"data_stored":8593801288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124306,"ondisk_log_size":124306,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234383,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1704552,"kb_used_data":1376480,"kb_used_omap":1,"kb_used_meta":328062,"kb_avail":92040600,"statfs":{"total":95995035648,"available":94249574400,"internally_reserved":0,"allocated":1409515520,"data_stored":1407900443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":335935947},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496014,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":215268,"kb_used_data":17628,"kb_used_omap":1,"kb_used_meta":197630,"kb_avail":93529884,"statfs":{"total":95995035648,"available":95774601216,"internally_reserved":0,"allocated":18051072,"data_stored":16439795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":202373575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426601,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937816,"kb_used_data":860560,"kb_used_omap":8,"kb_used_meta":77239,"kb_avail":92807336,"statfs":{"total":95995035648,"available":95034712064,"internally_reserved":0,"allocated":881213440,"data_stored":879609629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":79093642},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019293,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1188232,"kb_used_data":864256,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":92556920,"statfs":{"total":95995035648,"available":94778286080,"internally_reserved":0,"allocated":884998144,"data_stored":883410116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435478,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2007380,"kb_used_data":1705036,"kb_used_omap":4,"kb_used_meta":302331,"kb_avail":91737772,"statfs":{"total":95995035648,"available":93939478528,"internally_reserved":0,"allocated":1745956864,"data_stored":1744356993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":309587209},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903330,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1806424,"kb_used_data":1652496,"kb_used_omap":10,"kb_used_meta":153909,"kb_avail":91938728,"statfs":{"total":95995035648,"available":94145257472,"internally_reserved":0,"allocated":1692155904,"data_stored":1690555969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":157603344},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762531,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1562580,"kb_used_data":1287948,"kb_used_omap":7,"kb_used_meta":274616,"kb_avail":92182572,"statfs":{"total":95995035648,"available":94394953728,"internally_reserved":0,"allocated":1318858752,"data_stored":1317257641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":281207138},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926057,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1018396,"kb_used_data":702100,"kb_used_omap":7,"kb_used_meta":316280,"kb_avail":92726756,"statfs":{"total":95995035648,"available":94952198144,"internally_reserved":0,"allocated":718950400,"data_stored":717342068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":323870863},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":708907008,"data_stored":708902936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1305874432,"data_stored":1305870360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1679163392,"data_stored":1679163392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1735786496,"data_stored":1735786496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":875429888,"data_stored":875429888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":873992192,"data_stored":873988120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":13893632,"data_stored":13893632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1400766464,"data_stored":1400766464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:26:41.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:26:41.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:41.768 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:41.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:41.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:41.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:41.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:41.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:42.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:42.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:42.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:42.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:42.210 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:26:42.210 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:26:42.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:26:42.210 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:26:42.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140284 6.a 6.98e9132a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 561.03457046300002, 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59515916599999996, 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140284 2024-08-29T15:26:42.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173565+0000", 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173567+0000", 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4729999999999999e-06 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173572+0000", 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7270000000000004e-06 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173590+0000", 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7442999999999999e-05 2024-08-29T15:26:42.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173598+0000", 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2830000000000001e-06 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173700+0000", 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101889 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.173709+0000", 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4749999999999993e-06 2024-08-29T15:26:42.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768713+0000", 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.59500392899999999 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768724+0000", 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1946999999999999e-05 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140312 6.a 6.b6852f8a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 561.02305216399998, 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58367630699999995, 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140312 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185084+0000", 2024-08-29T15:26:42.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5400000000000002e-07 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185087+0000", 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1209999999999998e-06 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185095+0000", 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0530000000000007e-06 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185101+0000", 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0349999999999998e-06 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185153+0000", 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1468999999999998e-05 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.185161+0000", 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9549999999999992e-06 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768751+0000", 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.58359058799999997 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768760+0000", 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6319999999999997e-06 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140319 6.a 6.5ae9c7ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:26:42.220 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 561.01066401100002, 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57131815699999999, 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140319 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:26:42.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197472+0000", 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197473+0000", 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2270000000000001e-06 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197513+0000", 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9731000000000003e-05 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197529+0000", 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6337e-05 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197537+0000", 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1890000000000007e-06 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197704+0000", 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016718599999999999 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.197716+0000", 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1282e-05 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768782+0000", 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57106648599999998 2024-08-29T15:26:42.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768790+0000", 2024-08-29T15:26:42.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7190000000000001e-06 2024-08-29T15:26:42.225 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140323 6.a 6.def6e63a (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 561.00954313099999, 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57023369499999998, 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.226 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140323 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198593+0000", 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7599999999999996e-07 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198597+0000", 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9940000000000002e-06 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198611+0000", 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3522e-05 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198617+0000", 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4880000000000004e-06 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198665+0000", 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8040000000000001e-05 2024-08-29T15:26:42.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.198673+0000", 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1400000000000002e-06 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768811+0000", 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.57013830200000004 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.768826+0000", 2024-08-29T15:26:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5432999999999999e-05 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140331 6.a 6.ecf214ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 560.99463438600003, 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56336548399999997, 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:26:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140331 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213501+0000", 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213503+0000", 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.091e-06 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213510+0000", 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.861e-06 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213531+0000", 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0811000000000001e-05 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213538+0000", 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2869999999999999e-06 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213591+0000", 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2735999999999998e-05 2024-08-29T15:26:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.213598+0000", 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4330000000000001e-06 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776857+0000", 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56325978499999996 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776867+0000", 2024-08-29T15:26:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4800000000000007e-06 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140333 6.a 6.7707feaa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 560.99276051300001, 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56152404199999995, 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:26:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140333 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215375+0000", 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215377+0000", 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7320000000000001e-06 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215385+0000", 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6839999999999996e-06 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215402+0000", 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7489000000000001e-05 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215411+0000", 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7450000000000001e-06 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215492+0000", 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0819e-05 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:26:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.215501+0000", 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6369999999999994e-06 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776893+0000", 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56139153100000005 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.776899+0000", 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4049999999999997e-06 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:140336 6.a 6.ee7969ea (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e196)", 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 560.98969589800004, 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.55869965600000004, 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140336 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218440+0000", 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218442+0000", 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8250000000000001e-06 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218450+0000", 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0690000000000002e-06 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218472+0000", 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2178e-05 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218482+0000", 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0071999999999999e-05 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218555+0000", 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3465000000000002e-05 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.218568+0000", 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2772000000000001e-05 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.777129+0000", 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.55856027900000005 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:17:21.777140+0000", 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0995999999999999e-05 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:183025 6.0 6:08892af3:::benchmark_data_smithi027_169445_object57598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:24:22.771934+0000", 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 139.43620148700001, 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020281896000000001, 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 183025 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.771934+0000", 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.771934+0000", 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.771936+0000", 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0320000000000002e-06 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.771942+0000", 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.733e-06 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.771955+0000", 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3134e-05 2024-08-29T15:26:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.771961+0000", 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0569999999999998e-06 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.773418+0000", 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014570099999999999 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.773597+0000", 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000178767 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.773761+0000", 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016430899999999999 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.788328+0000", 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014567157000000001 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.790493+0000", 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021648570000000001 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.790654+0000", 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016033900000000001 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.790714+0000", 2024-08-29T15:26:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0151000000000001e-05 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:24:22.792216+0000", 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00150235 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:190420 6.0 6:0fe32611:::benchmark_data_smithi027_169445_object64993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:25:07.882203+0000", 2024-08-29T15:26:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.325932898999994, 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018830481, 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190420 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882203+0000", 2024-08-29T15:26:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882203+0000", 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882205+0000", 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0200000000000001e-06 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882212+0000", 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4730000000000003e-06 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882235+0000", 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2815000000000001e-05 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882245+0000", 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0164e-05 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882317+0000", 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2059999999999998e-05 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882456+0000", 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013865200000000001 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.882591+0000", 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013477000000000001 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.890242+0000", 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076509660000000004 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.891943+0000", 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001701057 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.900021+0000", 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080780330000000001 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.900061+0000", 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0289e-05 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.901033+0000", 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097218200000000004 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:190421 6.0 6:0732374a:::benchmark_data_smithi027_169445_object64994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:25:07.883321+0000", 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.324814681999996, 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018999634000000001, 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190421 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.883321+0000", 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.883321+0000", 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.883322+0000", 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.254e-06 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.883329+0000", 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1829999999999999e-06 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.883345+0000", 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6339000000000001e-05 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.883353+0000", 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4810000000000006e-06 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.886234+0000", 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028808179999999998 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.886389+0000", 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015452 2024-08-29T15:26:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.886485+0000", 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6597999999999999e-05 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.890250+0000", 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037651400000000002 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.901085+0000", 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010834139 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.901130+0000", 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.515e-05 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.901189+0000", 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9369e-05 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:07.902321+0000", 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011316430000000001 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:190668 6.0 6:0dc5dced:::benchmark_data_smithi027_169445_object65241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:25:09.125858+0000", 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 93.082277896999997, 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018908231000000001, 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190668 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.125858+0000", 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.125858+0000", 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.125860+0000", 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0729999999999999e-06 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.125866+0000", 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.947e-06 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.125883+0000", 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6660999999999999e-05 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.125890+0000", 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9109999999999993e-06 2024-08-29T15:26:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.127605+0000", 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001714234 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.127776+0000", 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000171297 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.127951+0000", 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017540000000000001 2024-08-29T15:26:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.132285+0000", 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043335240000000001 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.133056+0000", 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077135400000000003 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.143544+0000", 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010487619 2024-08-29T15:26:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.143586+0000", 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2435999999999998e-05 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:09.144766+0000", 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011797750000000001 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:191000 6.9 6:9217cfcc:::benchmark_data_smithi027_169445_object65573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:25:11.176898+0000", 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 91.031238009000006, 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023846948999999999, 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191000 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.176898+0000", 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.176898+0000", 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.176899+0000", 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4899999999999999e-06 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.176905+0000", 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.037e-06 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.176918+0000", 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3178e-05 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.176925+0000", 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4489999999999996e-06 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.176990+0000", 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4809999999999998e-05 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.177090+0000", 2024-08-29T15:26:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100209 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.177180+0000", 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0457000000000005e-05 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.180411+0000", 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032303169999999999 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.180952+0000", 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000541466 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.200466+0000", 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019513714000000001 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.200529+0000", 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3579999999999998e-05 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:11.200745+0000", 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021524200000000001 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:200074 6.9 6:9d373aa2:::benchmark_data_smithi027_169445_object74647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:25:59.234504+0000", 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 42.973631730999998, 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019282798, 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200074 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234504+0000", 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234504+0000", 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234506+0000", 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.99e-06 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234514+0000", 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6389999999999992e-06 2024-08-29T15:26:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234534+0000", 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0327999999999999e-05 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234543+0000", 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6810000000000004e-06 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234600+0000", 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7417000000000003e-05 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234694+0000", 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4124999999999998e-05 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.234797+0000", 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010279 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.238056+0000", 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032593890000000001 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.238148+0000", 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1188999999999997e-05 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.253527+0000", 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015379113 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.253595+0000", 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8441000000000004e-05 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:25:59.253787+0000", 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019169599999999999 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:203604 6.0 6:0a082cb6:::benchmark_data_smithi027_169445_object78177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:26:15.744612+0000", 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 26.463524199999998, 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019283437, 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 203604 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744612+0000", 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744612+0000", 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744614+0000", 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.956e-06 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744620+0000", 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8410000000000002e-06 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744642+0000", 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1339e-05 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744651+0000", 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3449999999999995e-06 2024-08-29T15:26:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744728+0000", 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7414000000000001e-05 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.744944+0000", 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000215835 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.745103+0000", 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015886000000000001 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.749225+0000", 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041220099999999997 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.751863+0000", 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026380359999999999 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.761527+0000", 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096634010000000003 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.761587+0000", 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0254000000000003e-05 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:15.763895+0000", 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023081460000000001 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:203862 6.0 6:0702adf1:::benchmark_data_smithi027_169445_object78435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:26:16.598318+0000", 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 25.609817756999998, 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019158817000000002, 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 203862 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598318+0000", 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598318+0000", 2024-08-29T15:26:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598320+0000", 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1409999999999999e-06 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598330+0000", 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8830000000000001e-06 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598355+0000", 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4800999999999999e-05 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598365+0000", 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7769999999999997e-06 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598443+0000", 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8576000000000001e-05 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598619+0000", 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000175694 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.598782+0000", 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016317799999999999 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.607911+0000", 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0091286400000000004 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.608944+0000", 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010331610000000001 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.615712+0000", 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006768573 2024-08-29T15:26:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.615785+0000", 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2428999999999994e-05 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.617477+0000", 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001691964 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:203864 6.0 6:0ef486d7:::benchmark_data_smithi027_169445_object78437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:26:16.600605+0000", 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 25.607530472000001, 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018844763, 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 203864 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.600605+0000", 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.600605+0000", 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.600608+0000", 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.243e-06 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.600614+0000", 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.669e-06 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.600636+0000", 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1668e-05 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.600645+0000", 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6330000000000008e-06 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.603375+0000", 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027297419999999998 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.603528+0000", 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000152991 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.603680+0000", 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015129999999999999 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.607920+0000", 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042406190000000002 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.609162+0000", 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001242174 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.617536+0000", 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083734499999999993 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.617721+0000", 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018517499999999999 2024-08-29T15:26:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:16.619450+0000", 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001729099 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:208940 6.0 6:08298271:::benchmark_data_smithi027_169445_object83513:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:26:38.054120+0000", 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.154015309, 2024-08-29T15:26:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018382368999999999, 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 208940 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054120+0000", 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054120+0000", 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054122+0000", 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.725e-06 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054129+0000", 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8700000000000003e-06 2024-08-29T15:26:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054143+0000", 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3855e-05 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054151+0000", 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8460000000000008e-06 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054212+0000", 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1121e-05 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054372+0000", 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000160063 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.054534+0000", 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016202999999999999 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.066010+0000", 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011476385 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.066025+0000", 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4596999999999999e-05 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.070605+0000", 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045805100000000003 2024-08-29T15:26:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.070742+0000", 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013647799999999999 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.072503+0000", 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017608890000000001 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:208942 6.0 6:0b5c1a83:::benchmark_data_smithi027_169445_object83515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-08-29T15:26:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:26:38.056251+0000", 2024-08-29T15:26:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.1518851809999999, 2024-08-29T15:26:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018000649000000001, 2024-08-29T15:26:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 208942 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.056251+0000", 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.056251+0000", 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.056252+0000", 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5799999999999999e-06 2024-08-29T15:26:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.056259+0000", 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6969999999999996e-06 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.056273+0000", 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3720000000000001e-05 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.056280+0000", 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6739999999999997e-06 2024-08-29T15:26:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.060801+0000", 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045211970000000002 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.060932+0000", 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013080000000000001 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.061017+0000", 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5004000000000001e-05 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.066031+0000", 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005013422 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.070562+0000", 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045315349999999997 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.072565+0000", 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020028120000000001 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.072746+0000", 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018127400000000001 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.074251+0000", 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0015049340000000001 2024-08-29T15:26:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:209133 6.0 6:0b05f041:::benchmark_data_smithi027_169445_object83706:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:26:38.869055+0000", 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.3390809379999999, 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020997284000000001, 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 209133 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869055+0000", 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869055+0000", 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869057+0000", 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7990000000000001e-06 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869062+0000", 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3709999999999999e-06 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869074+0000", 2024-08-29T15:26:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.24e-05 2024-08-29T15:26:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869082+0000", 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9330000000000001e-06 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869153+0000", 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0618999999999996e-05 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869311+0000", 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015762600000000001 2024-08-29T15:26:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.869465+0000", 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015399899999999999 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.874031+0000", 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045667190000000003 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.879136+0000", 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051051070000000002 2024-08-29T15:26:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.888567+0000", 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094306400000000006 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.888634+0000", 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6851000000000001e-05 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.890052+0000", 2024-08-29T15:26:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00141822 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:209161 6.9 6:97e090cd:::benchmark_data_smithi027_169445_object83734:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:26:38.917053+0000", 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.2910831850000002, 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017980968999999999, 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:26:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 209161 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917053+0000", 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917053+0000", 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917054+0000", 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.632e-06 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917061+0000", 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3249999999999996e-06 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:26:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917076+0000", 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5719999999999999e-05 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917084+0000", 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1329999999999999e-06 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917161+0000", 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7045999999999994e-05 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917291+0000", 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012927500000000001 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.917425+0000", 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013418800000000001 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.927337+0000", 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099122429999999994 2024-08-29T15:26:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.927351+0000", 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3723000000000001e-05 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.934736+0000", 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073855809999999996 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:26:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.934809+0000", 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2478000000000006e-05 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:26:38.935034+0000", 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000224625 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:26:42.333 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:26:42.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:42.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:42.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:42.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:42 smithi027 ceph-mon[98970]: pgmap v946: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 27542/126618 objects degraded (21.752%); 8520/126618 objects misplaced (6.729%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:42.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 27542/126618 objects degraded (21.752%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:42.448 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:42 smithi027 ceph-mon[93921]: pgmap v946: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 27542/126618 objects degraded (21.752%); 8520/126618 objects misplaced (6.729%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:42.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 27542/126618 objects degraded (21.752%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:42.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:42 smithi171 ceph-mon[90031]: pgmap v946: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 241 op/s; 27542/126618 objects degraded (21.752%); 8520/126618 objects misplaced (6.729%); 1.9 MiB/s, 29 objects/s recovering 2024-08-29T15:26:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 27542/126618 objects degraded (21.752%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:42.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:42.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:42.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:42.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:42.709 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:42.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:42.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:42.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:42.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:42.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:42.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:42.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:42.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:43.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:43.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:43.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:43.178 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:26:43.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:43.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:43.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:43.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:43.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:43.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:43.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:43.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/627182241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:43.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/627182241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:43.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/627182241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:26:43.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:43.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:43.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:43.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:43.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:43.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:43.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:43.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:43.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:43.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:43.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:44.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:44.100 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:44.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:44.223 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[98970]: pgmap v947: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 287 op/s; 27352/123573 objects degraded (22.134%); 8422/123573 objects misplaced (6.815%); 2.0 MiB/s, 32 objects/s recovering 2024-08-29T15:26:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/627182241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[98970]: osdmap e278: 8 total, 8 up, 8 in 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[93921]: pgmap v947: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 287 op/s; 27352/123573 objects degraded (22.134%); 8422/123573 objects misplaced (6.815%); 2.0 MiB/s, 32 objects/s recovering 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/627182241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[93921]: osdmap e278: 8 total, 8 up, 8 in 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:44.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:44.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:44.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:44.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:44.472 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:44.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:44 smithi171 ceph-mon[90031]: pgmap v947: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 287 op/s; 27352/123573 objects degraded (22.134%); 8422/123573 objects misplaced (6.815%); 2.0 MiB/s, 32 objects/s recovering 2024-08-29T15:26:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:44 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/627182241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:26:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:44 smithi171 ceph-mon[90031]: osdmap e278: 8 total, 8 up, 8 in 2024-08-29T15:26:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:26:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:44.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:44.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:44.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:44.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:44.724 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:44.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:44.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:44.959 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:44.984 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:44.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:44.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:45.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:45.109 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:45.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:45.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:45.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:45.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:45.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:45.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:45.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:45.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:45.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:45.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:45.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:45.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:45.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:45.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:45.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:45.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:45.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:45.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:26:45.717 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:45.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:45.758 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:45.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:45.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:45.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:45.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:45.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:45.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:45.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:46.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:46.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:46.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:46.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:46.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:46.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:46.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:46.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:46.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:26:46.386 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:46.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:46.436 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:46 smithi027 ceph-mon[93921]: pgmap v949: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 27352/122889 objects degraded (22.257%); 8340/122889 objects misplaced (6.787%); 2.0 MiB/s, 32 objects/s recovering 2024-08-29T15:26:46.437 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:46 smithi027 ceph-mon[98970]: pgmap v949: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 27352/122889 objects degraded (22.257%); 8340/122889 objects misplaced (6.787%); 2.0 MiB/s, 32 objects/s recovering 2024-08-29T15:26:46.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:46.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:46 smithi171 ceph-mon[90031]: pgmap v949: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 27352/122889 objects degraded (22.257%); 8340/122889 objects misplaced (6.787%); 2.0 MiB/s, 32 objects/s recovering 2024-08-29T15:26:46.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:46.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:46.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:46.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:46.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:46.796 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:26:46.796 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:26:46.828 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:26:46.828 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:26:46.855 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:26:46.855 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:26:46.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:46.887 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:26:46.887 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:26:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:46.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:46.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:46.919 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:26:46.919 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:26:46.952 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:26:46.952 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:26:46.976 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:26:46.976 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:26:46.995 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:26:46.995 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:26:47.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:47.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.040 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:47.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:47.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:47.167 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:47.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:47.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:47.290 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3837214773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:47.290 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 27352/122889 objects degraded (22.257%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:47.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3837214773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:47.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 27352/122889 objects degraded (22.257%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:47.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:47.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:47.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3837214773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:26:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 27352/122889 objects degraded (22.257%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:47.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:47.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:47.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:47.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:47.674 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:47.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:47.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:47.799 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:47.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:47.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:47.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:48.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:48.064 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:48.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:48.166 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:48.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:48.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:48.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:48.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:48 smithi027 ceph-mon[93921]: pgmap v950: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 27202/119571 objects degraded (22.750%); 8254/119571 objects misplaced (6.903%); 2.1 MiB/s, 32 objects/s recovering 2024-08-29T15:26:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:48 smithi027 ceph-mon[98970]: pgmap v950: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 27202/119571 objects degraded (22.750%); 8254/119571 objects misplaced (6.903%); 2.1 MiB/s, 32 objects/s recovering 2024-08-29T15:26:48.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:48.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:48 smithi171 ceph-mon[90031]: pgmap v950: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 27202/119571 objects degraded (22.750%); 8254/119571 objects misplaced (6.903%); 2.1 MiB/s, 32 objects/s recovering 2024-08-29T15:26:48.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:48.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:48.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:48.691 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:48.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:48.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:48.920 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:48.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:48.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:49.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:49.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:26:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:26:49] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-08-29T15:26:49.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:49.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:49.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:49.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:49.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:49.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:49.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:49.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:49.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:49.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:49.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:49.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:49.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:49.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:49.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.730 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:49.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:49.820 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:26:49.821 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 27202/119571 objects degraded (22.750%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1391,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":278,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":89,"num_pools":5,"num_objects":39857,"data_bytes":2614958077,"bytes_used":10178281472,"bytes_avail":757782003712,"bytes_total":767960285184,"degraded_objects":27202,"degraded_total":119571,"degraded_ratio":0.22749663379916535,"misplaced_objects":8254,"misplaced_total":119571,"misplaced_ratio":0.069030115998026276,"recovering_objects_per_sec":32,"recovering_bytes_per_sec":2162045,"recovering_keys_per_sec":0,"num_objects_recovered":330,"num_bytes_recovered":21626880,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":279},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:26:49.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:49.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:49.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:49.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:49.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:49.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:49.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:50.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:50.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:50.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:50.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:50.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:50.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:50.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:50.355 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:50.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:50.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:50.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:26:50.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:50.484 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:50.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:50 smithi171 ceph-mon[90031]: pgmap v951: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 27202/119571 objects degraded (22.750%); 8254/119571 objects misplaced (6.903%); 2.1 MiB/s, 32 objects/s recovering 2024-08-29T15:26:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3818464426' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:26:50.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:50 smithi027 ceph-mon[98970]: pgmap v951: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 27202/119571 objects degraded (22.750%); 8254/119571 objects misplaced (6.903%); 2.1 MiB/s, 32 objects/s recovering 2024-08-29T15:26:50.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3818464426' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:26:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:50 smithi027 ceph-mon[93921]: pgmap v951: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 27202/119571 objects degraded (22.750%); 8254/119571 objects misplaced (6.903%); 2.1 MiB/s, 32 objects/s recovering 2024-08-29T15:26:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3818464426' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:26:50.586 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:50.611 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:50.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:50.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:50.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:50.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:50.755 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:26:50.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:26:50.839 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:50.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:50.841 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:50.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:50.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:50.910 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:50.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:51.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:51.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:51.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:51.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:51.242 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:51.268 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:51.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:51.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:51.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:51.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:51.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:51.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:51.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:51.647 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:51.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:51.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:51.876 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:51.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:51.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:52.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:52.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:52.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:52.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:52.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:52.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:52.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:52.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:52.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:52.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:52.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:52.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:52.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:52 smithi027 ceph-mon[98970]: pgmap v952: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 27175/118026 objects degraded (23.025%); 8176/118026 objects misplaced (6.927%); 2.3 MiB/s, 36 objects/s recovering 2024-08-29T15:26:52.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 27175/118026 objects degraded (23.025%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:52.514 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:52 smithi027 ceph-mon[93921]: pgmap v952: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 27175/118026 objects degraded (23.025%); 8176/118026 objects misplaced (6.927%); 2.3 MiB/s, 36 objects/s recovering 2024-08-29T15:26:52.514 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 27175/118026 objects degraded (23.025%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:52 smithi171 ceph-mon[90031]: pgmap v952: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 27175/118026 objects degraded (23.025%); 8176/118026 objects misplaced (6.927%); 2.3 MiB/s, 36 objects/s recovering 2024-08-29T15:26:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 27175/118026 objects degraded (23.025%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:52.530 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:52.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:52.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:52.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:52.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:52.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:52.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:52.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:52.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:52.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:53.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:53.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:53.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:53.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:53.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:53.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:53.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:53.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:53.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:53.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:53.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:53.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:53.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.691 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:53.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:53.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:53.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:53.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:53.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:53.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:53.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:54.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:54.077 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:54.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:54.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:54.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:54.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:54.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:54.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:54.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:54 smithi171 ceph-mon[90031]: pgmap v953: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 26904/114636 objects degraded (23.469%); 8078/114636 objects misplaced (7.047%); 2.7 MiB/s, 42 objects/s recovering 2024-08-29T15:26:54.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:54 smithi027 ceph-mon[98970]: pgmap v953: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 26904/114636 objects degraded (23.469%); 8078/114636 objects misplaced (7.047%); 2.7 MiB/s, 42 objects/s recovering 2024-08-29T15:26:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:54 smithi027 ceph-mon[93921]: pgmap v953: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 26904/114636 objects degraded (23.469%); 8078/114636 objects misplaced (7.047%); 2.7 MiB/s, 42 objects/s recovering 2024-08-29T15:26:54.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:54.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:54.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:54.726 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:54.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:54.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:54.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:54.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:54.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:54.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:54.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:55.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:55.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:55.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:55.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:55.226 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:55.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:55.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:55.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:55.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:55.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:55.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:55.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:55.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:55.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:55.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:55.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:55.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:55.858 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:55.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:55.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:55.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:55.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:55.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:56.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:56.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:56.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:56.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:56.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:56.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:56 smithi027 ceph-mon[98970]: pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 284 op/s; 26819/113784 objects degraded (23.570%); 8004/113784 objects misplaced (7.034%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:26:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:56 smithi027 ceph-mon[93921]: pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 284 op/s; 26819/113784 objects degraded (23.570%); 8004/113784 objects misplaced (7.034%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:26:56.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:56.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.368 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:56.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:56.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:56.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:56 smithi171 ceph-mon[90031]: pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 284 op/s; 26819/113784 objects degraded (23.570%); 8004/113784 objects misplaced (7.034%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:26:56.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:26:56.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:56.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:26:56.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:56.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:56.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:56.866 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:56.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:56.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:56.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:56.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:56.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:57.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:57.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:57.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:57.203 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:26:57.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:57.244 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:57.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:57.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:57.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:57.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:57.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:57.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 26819/113784 objects degraded (23.570%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4117148285' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 26819/113784 objects degraded (23.570%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4117148285' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 26819/113784 objects degraded (23.570%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:26:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4117148285' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:26:57.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:57.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:57.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:57.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:57.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:57.850 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:57.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:57.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:57.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:57.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:57.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:58.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:58.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:58.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:58.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:58.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:58.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:58.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:58.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:58.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:58.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:58.442 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:26:58.464 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[93921]: pgmap v955: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 26478/110484 objects degraded (23.965%); 7925/110484 objects misplaced (7.173%); 3.6 MiB/s, 56 objects/s recovering 2024-08-29T15:26:58.464 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:58.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4117148285' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:26:58.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[93921]: osdmap e279: 8 total, 8 up, 8 in 2024-08-29T15:26:58.465 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[98970]: pgmap v955: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 26478/110484 objects degraded (23.965%); 7925/110484 objects misplaced (7.173%); 3.6 MiB/s, 56 objects/s recovering 2024-08-29T15:26:58.465 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:58.465 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4117148285' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:26:58.465 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:58 smithi027 ceph-mon[98970]: osdmap e279: 8 total, 8 up, 8 in 2024-08-29T15:26:58.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:58.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:58.503 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:58.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:58 smithi171 ceph-mon[90031]: pgmap v955: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 26478/110484 objects degraded (23.965%); 7925/110484 objects misplaced (7.173%); 3.6 MiB/s, 56 objects/s recovering 2024-08-29T15:26:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:58 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:26:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4117148285' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:26:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:58 smithi171 ceph-mon[90031]: osdmap e279: 8 total, 8 up, 8 in 2024-08-29T15:26:58.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:26:58.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:58.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:58.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:26:58.727 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:58.751 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:58.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:58.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:26:58.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:58.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:26:58.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:59.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:59.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:59.098 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:26:59.099 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:26:59.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:59.129 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:59.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:26:59.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:59.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:26:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:26:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:26:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:26:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:26:59] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.51.0" 2024-08-29T15:26:59.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:26:59.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:26:59.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:26:59.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:26:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:26:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:26:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:26:59.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:59.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:59.698 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:26:59.698 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:26:59.718 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:26:59.719 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:26:59.736 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:26:59.736 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:26:59.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:26:59.754 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:26:59.754 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:26:59.766 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:59.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:26:59.774 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:26:59.774 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:26:59.795 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:26:59.795 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:26:59.816 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:26:59.817 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:26:59.837 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:26:59.837 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:26:59.868 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:26:59.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:26:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:26:59.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:26:59.994 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:00.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:00.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:00.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:00.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:00.276 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:00.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:00.343 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:27:00.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:27:00.378 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:00.402 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:00.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:00 smithi171 ceph-mon[90031]: pgmap v957: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 26478/110484 objects degraded (23.965%); 7925/110484 objects misplaced (7.173%); 3.7 MiB/s, 59 objects/s recovering 2024-08-29T15:27:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/171262509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:00.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.530 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:00.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:00.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:00 smithi027 ceph-mon[93921]: pgmap v957: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 26478/110484 objects degraded (23.965%); 7925/110484 objects misplaced (7.173%); 3.7 MiB/s, 59 objects/s recovering 2024-08-29T15:27:00.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/171262509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:00 smithi027 ceph-mon[98970]: pgmap v957: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 302 op/s; 26478/110484 objects degraded (23.965%); 7925/110484 objects misplaced (7.173%); 3.7 MiB/s, 59 objects/s recovering 2024-08-29T15:27:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/171262509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:00.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:00.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:00.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:00.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:00.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:00.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:00.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:00.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:01.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:01.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:01.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:01.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:01.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:01.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:01.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:01.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:01.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:01.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:01.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:01.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:01.681 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:01.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:01.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:01.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:01.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:01.936 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:01.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:02.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:02.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:02.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:02.189 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:02.274 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:02.277 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:02.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:02.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:02.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[93921]: pgmap v958: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 26338/109602 objects degraded (24.031%); 7841/109602 objects misplaced (7.154%); 3.9 MiB/s, 62 objects/s recovering 2024-08-29T15:27:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:27:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:27:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 26338/109602 objects degraded (24.031%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[98970]: pgmap v958: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 26338/109602 objects degraded (24.031%); 7841/109602 objects misplaced (7.154%); 3.9 MiB/s, 62 objects/s recovering 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 26338/109602 objects degraded (24.031%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:02.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:02.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:02.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:02 smithi171 ceph-mon[90031]: pgmap v958: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 26338/109602 objects degraded (24.031%); 7841/109602 objects misplaced (7.154%); 3.9 MiB/s, 62 objects/s recovering 2024-08-29T15:27:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:27:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:27:02.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:02.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 26338/109602 objects degraded (24.031%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:02.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:02.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:02.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:02.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:02.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:02.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:02.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:02.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:02.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:02.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:02.947 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:02.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:03.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:03.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:03.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:03.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:03.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:03.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:03.341 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:03.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:03.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:03.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:03.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:03.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:03.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:03.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:03.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:03.846 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:03.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:03.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:03.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:04.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:04.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:04.207 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:04.220 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:27:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:04.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:04.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:04 smithi027 ceph-mon[98970]: pgmap v959: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 26094/107037 objects degraded (24.378%); 7750/107037 objects misplaced (7.240%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:04.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:04 smithi027 ceph-mon[93921]: pgmap v959: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 26094/107037 objects degraded (24.378%); 7750/107037 objects misplaced (7.240%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:04.319 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:04 smithi171 ceph-mon[90031]: pgmap v959: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 26094/107037 objects degraded (24.378%); 7750/107037 objects misplaced (7.240%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:04.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:04.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:04.465 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:04.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:04.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:04.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:04.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:04.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:04.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:04.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:04.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:04.866 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:04.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:04.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:04.992 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:04.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:05.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:05.122 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:05.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:05.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:05.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:05.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:05.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:05.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:05.373 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:05.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:05.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:05.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:05.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:05.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:05.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:05.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:05.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:05.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:05.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:05.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:05.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:05.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:05.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:06.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:06.019 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:27:06.020 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:27:06.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:06.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:06.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:06.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:06.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:06 smithi171 ceph-mon[90031]: pgmap v960: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 26011/106239 objects degraded (24.483%); 7665/106239 objects misplaced (7.215%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:06.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:06.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:06.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:06 smithi027 ceph-mon[93921]: pgmap v960: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 26011/106239 objects degraded (24.483%); 7665/106239 objects misplaced (7.215%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:06.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:06 smithi027 ceph-mon[98970]: pgmap v960: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 251 op/s; 26011/106239 objects degraded (24.483%); 7665/106239 objects misplaced (7.215%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:06.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:06.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:06.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:06.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:06.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:06.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:06.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:06.751 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":960,"stamp":"2024-08-29T15:27:04.660298+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2323716093,"num_objects":35413,"num_object_clones":0,"num_object_copies":106239,"num_objects_missing_on_primary":3173,"num_objects_missing":3173,"num_objects_degraded":26011,"num_objects_misplaced":7665,"num_objects_unfound":0,"num_objects_dirty":35413,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":216588,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20947,"num_bytes_recovered":1251609805,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125524,"ondisk_log_size":125524,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9234888,"kb_used_data":7245064,"kb_used_omap":43,"kb_used_meta":1989716,"kb_avail":740726328,"statfs":{"total":767960285184,"available":758503759872,"internally_reserved":0,"allocated":7418945536,"data_stored":7405995726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2037469676},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,2,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-164823040,"num_objects":-2515,"num_object_clones":0,"num_object_copies":-7545,"num_objects_missing_on_primary":-168,"num_objects_missing":-168,"num_objects_degraded":-808,"num_objects_misplaced":-339,"num_objects_unfound":0,"num_objects_dirty":-2515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2515,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":648,"num_bytes_recovered":42467328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003243"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":444,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207259+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:26:57.207259+0000","last_peered":"2024-08-29T15:26:57.207259+0000","last_clean":"2024-08-29T15:26:57.207259+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:26:57.207259+0000","last_undegraded":"2024-08-29T15:26:57.207259+0000","last_fullsized":"2024-08-29T15:26:57.207259+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":58,"reported_epoch":278,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:49.845667+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:26:49.845667+0000","last_peered":"2024-08-29T15:26:49.845667+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:26:49.845667+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:26:49.845667+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":147,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.182954+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:26:43.182954+0000","last_peered":"2024-08-29T15:26:43.182954+0000","last_clean":"2024-08-29T15:26:43.182954+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:26:43.182954+0000","last_undegraded":"2024-08-29T15:26:43.182954+0000","last_fullsized":"2024-08-29T15:26:43.182954+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":410,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211884+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:26:57.211884+0000","last_peered":"2024-08-29T15:26:57.211884+0000","last_clean":"2024-08-29T15:26:57.211884+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:26:57.211884+0000","last_undegraded":"2024-08-29T15:26:57.211884+0000","last_fullsized":"2024-08-29T15:26:57.211884+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":410,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211782+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:26:57.211782+0000","last_peered":"2024-08-29T15:26:57.211782+0000","last_clean":"2024-08-29T15:26:57.211782+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:26:57.211782+0000","last_undegraded":"2024-08-29T15:26:57.211782+0000","last_fullsized":"2024-08-29T15:26:57.211782+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:49.845596+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:26:49.845596+0000","last_peered":"2024-08-29T15:26:49.845596+0000","last_clean":"2024-08-29T15:26:49.845596+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:26:49.845596+0000","last_undegraded":"2024-08-29T15:26:49.845596+0000","last_fullsized":"2024-08-29T15:26:49.845596+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":221,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208652+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:26:57.208652+0000","last_peered":"2024-08-29T15:26:57.208652+0000","last_clean":"2024-08-29T15:26:57.208652+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:26:57.208652+0000","last_undegraded":"2024-08-29T15:26:57.208652+0000","last_fullsized":"2024-08-29T15:26:57.208652+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":149,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.204926+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:26:43.204926+0000","last_peered":"2024-08-29T15:26:43.204926+0000","last_clean":"2024-08-29T15:26:43.204926+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:26:43.204926+0000","last_undegraded":"2024-08-29T15:26:43.204926+0000","last_fullsized":"2024-08-29T15:26:43.204926+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"279'13581","reported_seq":21640,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:01.139475+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:27:01.139475+0000","last_peered":"2024-08-29T15:27:01.139475+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:27:01.139475+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"178'3500","ondisk_log_start":"178'3500","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10081,"log_dups_size":0,"ondisk_log_size":10081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152764416,"num_objects":2331,"num_object_clones":0,"num_object_copies":6993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13581,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,5","objects":-594}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":57,"reported_epoch":278,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:43.184871+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:26:43.184871+0000","last_peered":"2024-08-29T15:26:43.184871+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:26:43.184871+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:26:43.184871+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":166,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.182669+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:26:43.182669+0000","last_peered":"2024-08-29T15:26:43.182669+0000","last_clean":"2024-08-29T15:26:43.182669+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:26:43.182669+0000","last_undegraded":"2024-08-29T15:26:43.182669+0000","last_fullsized":"2024-08-29T15:26:43.182669+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"279'13309","reported_seq":13547,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:27:01.396539+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:27:01.396539+0000","last_peered":"2024-08-29T15:27:01.396539+0000","last_clean":"2024-08-29T15:27:01.396539+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:27:01.396539+0000","last_undegraded":"2024-08-29T15:27:01.396539+0000","last_fullsized":"2024-08-29T15:27:01.396539+0000","mapping_epoch":168,"log_start":"187'6628","ondisk_log_start":"187'6628","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146866176,"num_objects":2241,"num_object_clones":0,"num_object_copies":6723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13309,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":57,"reported_epoch":278,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:43.187928+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:26:43.187928+0000","last_peered":"2024-08-29T15:26:43.187928+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:26:43.187928+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:26:43.187928+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184242+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:26:43.184242+0000","last_peered":"2024-08-29T15:26:43.184242+0000","last_clean":"2024-08-29T15:26:43.184242+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:26:43.184242+0000","last_undegraded":"2024-08-29T15:26:43.184242+0000","last_fullsized":"2024-08-29T15:26:43.184242+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"279'13447","reported_seq":13655,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:01.233331+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:27:01.233331+0000","last_peered":"2024-08-29T15:27:01.233331+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:27:01.233331+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"187'6774","ondisk_log_start":"187'6774","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148439040,"num_objects":2265,"num_object_clones":0,"num_object_copies":6795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13447,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-212}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":226,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211532+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:26:57.211532+0000","last_peered":"2024-08-29T15:26:57.211532+0000","last_clean":"2024-08-29T15:26:57.211532+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:26:57.211532+0000","last_undegraded":"2024-08-29T15:26:57.211532+0000","last_fullsized":"2024-08-29T15:26:57.211532+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":104,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.206590+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:26:43.206590+0000","last_peered":"2024-08-29T15:26:43.206590+0000","last_clean":"2024-08-29T15:26:43.206590+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:26:43.206590+0000","last_undegraded":"2024-08-29T15:26:43.206590+0000","last_fullsized":"2024-08-29T15:26:43.206590+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"279'13597","reported_seq":19055,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:03.925438+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:27:03.925438+0000","last_peered":"2024-08-29T15:27:03.925438+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:27:03.925438+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"181'5462","ondisk_log_start":"181'5462","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8135,"log_dups_size":0,"ondisk_log_size":8135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142934016,"num_objects":2181,"num_object_clones":0,"num_object_copies":6543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13597,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2181}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":166,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.183012+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:26:43.183012+0000","last_peered":"2024-08-29T15:26:43.183012+0000","last_clean":"2024-08-29T15:26:43.183012+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:26:43.183012+0000","last_undegraded":"2024-08-29T15:26:43.183012+0000","last_fullsized":"2024-08-29T15:26:43.183012+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":424,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211171+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:26:57.211171+0000","last_peered":"2024-08-29T15:26:57.211171+0000","last_clean":"2024-08-29T15:26:57.211171+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:26:57.211171+0000","last_undegraded":"2024-08-29T15:26:57.211171+0000","last_fullsized":"2024-08-29T15:26:57.211171+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"279'13509","reported_seq":14020,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:59.715303+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:26:59.715303+0000","last_peered":"2024-08-29T15:26:59.715303+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:26:59.715303+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:26:59.715303+0000","mapping_epoch":234,"log_start":"184'5533","ondisk_log_start":"184'5533","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7976,"log_dups_size":0,"ondisk_log_size":7976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151060480,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3488,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13509,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":11272192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":2305}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":419,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.206761+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:26:57.206761+0000","last_peered":"2024-08-29T15:26:57.206761+0000","last_clean":"2024-08-29T15:26:57.206761+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:26:57.206761+0000","last_undegraded":"2024-08-29T15:26:57.206761+0000","last_fullsized":"2024-08-29T15:26:57.206761+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184732+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:26:43.184732+0000","last_peered":"2024-08-29T15:26:43.184732+0000","last_clean":"2024-08-29T15:26:43.184732+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:26:43.184732+0000","last_undegraded":"2024-08-29T15:26:43.184732+0000","last_fullsized":"2024-08-29T15:26:43.184732+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"279'13590","reported_seq":14270,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:01.473953+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:27:01.473953+0000","last_peered":"2024-08-29T15:27:01.473953+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:27:01.473953+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:27:01.473953+0000","mapping_epoch":234,"log_start":"187'6868","ondisk_log_start":"187'6868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147587072,"num_objects":2252,"num_object_clones":0,"num_object_copies":6756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13590,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":13959168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":2252}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.206446+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:26:43.206446+0000","last_peered":"2024-08-29T15:26:43.206446+0000","last_clean":"2024-08-29T15:26:43.206446+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:26:43.206446+0000","last_undegraded":"2024-08-29T15:26:43.206446+0000","last_fullsized":"2024-08-29T15:26:43.206446+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184983+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:26:43.184983+0000","last_peered":"2024-08-29T15:26:43.184983+0000","last_clean":"2024-08-29T15:26:43.184983+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:26:43.184983+0000","last_undegraded":"2024-08-29T15:26:43.184983+0000","last_fullsized":"2024-08-29T15:26:43.184983+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"279'13456","reported_seq":13895,"reported_epoch":279,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:27:01.320159+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:27:01.320159+0000","last_peered":"2024-08-29T15:27:01.320159+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:27:01.320159+0000","last_undegraded":"2024-08-29T15:27:01.320159+0000","last_fullsized":"2024-08-29T15:27:01.320159+0000","mapping_epoch":202,"log_start":"187'6767","ondisk_log_start":"187'6767","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6689,"log_dups_size":0,"ondisk_log_size":6689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144310272,"num_objects":2202,"num_object_clones":0,"num_object_copies":6606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2202,"num_objects_unfound":0,"num_objects_dirty":2202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13456,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":2202}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184274+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:26:43.184274+0000","last_peered":"2024-08-29T15:26:43.184274+0000","last_clean":"2024-08-29T15:26:43.184274+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:26:43.184274+0000","last_undegraded":"2024-08-29T15:26:43.184274+0000","last_fullsized":"2024-08-29T15:26:43.184274+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":264,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.187924+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:26:43.187924+0000","last_peered":"2024-08-29T15:26:43.187924+0000","last_clean":"2024-08-29T15:26:43.187924+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:26:43.187924+0000","last_undegraded":"2024-08-29T15:26:43.187924+0000","last_fullsized":"2024-08-29T15:26:43.187924+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"279'13562","reported_seq":13723,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:27:01.172415+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:27:01.172415+0000","last_peered":"2024-08-29T15:27:01.172415+0000","last_clean":"2024-08-29T15:27:01.172415+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:27:01.172415+0000","last_undegraded":"2024-08-29T15:27:01.172415+0000","last_fullsized":"2024-08-29T15:27:01.172415+0000","mapping_epoch":218,"log_start":"178'3478","ondisk_log_start":"178'3478","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143982616,"num_objects":2198,"num_object_clones":0,"num_object_copies":6594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2198,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13562,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":422,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.206229+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:26:57.206229+0000","last_peered":"2024-08-29T15:26:57.206229+0000","last_clean":"2024-08-29T15:26:57.206229+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:26:57.206229+0000","last_undegraded":"2024-08-29T15:26:57.206229+0000","last_fullsized":"2024-08-29T15:26:57.206229+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":414,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208744+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:26:57.208744+0000","last_peered":"2024-08-29T15:26:57.208744+0000","last_clean":"2024-08-29T15:26:57.208744+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:26:57.208744+0000","last_undegraded":"2024-08-29T15:26:57.208744+0000","last_fullsized":"2024-08-29T15:26:57.208744+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"279'12984","reported_seq":13193,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:27:01.024642+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:27:01.024642+0000","last_peered":"2024-08-29T15:27:01.024642+0000","last_clean":"2024-08-29T15:27:01.024642+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:27:01.024642+0000","last_undegraded":"2024-08-29T15:27:01.024642+0000","last_fullsized":"2024-08-29T15:27:01.024642+0000","mapping_epoch":201,"log_start":"178'2900","ondisk_log_start":"178'2900","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145752064,"num_objects":2224,"num_object_clones":0,"num_object_copies":6672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12984,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":513,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.215074+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:26:57.215074+0000","last_peered":"2024-08-29T15:26:57.215074+0000","last_clean":"2024-08-29T15:26:57.215074+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:26:57.215074+0000","last_undegraded":"2024-08-29T15:26:57.215074+0000","last_fullsized":"2024-08-29T15:26:57.215074+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":750,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.182875+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:26:43.182875+0000","last_peered":"2024-08-29T15:26:43.182875+0000","last_clean":"2024-08-29T15:26:43.182875+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:26:43.182875+0000","last_undegraded":"2024-08-29T15:26:43.182875+0000","last_fullsized":"2024-08-29T15:26:43.182875+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":258,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.182827+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:26:43.182827+0000","last_peered":"2024-08-29T15:26:43.182827+0000","last_clean":"2024-08-29T15:26:43.182827+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:26:43.182827+0000","last_undegraded":"2024-08-29T15:26:43.182827+0000","last_fullsized":"2024-08-29T15:26:43.182827+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":120,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.205917+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:26:43.205917+0000","last_peered":"2024-08-29T15:26:43.205917+0000","last_clean":"2024-08-29T15:26:43.205917+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:26:43.205917+0000","last_undegraded":"2024-08-29T15:26:43.205917+0000","last_fullsized":"2024-08-29T15:26:43.205917+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"279'13507","reported_seq":13943,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:27:00.651720+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:27:00.651720+0000","last_peered":"2024-08-29T15:27:00.651720+0000","last_clean":"2024-08-29T15:27:00.651720+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:27:00.651720+0000","last_undegraded":"2024-08-29T15:27:00.651720+0000","last_fullsized":"2024-08-29T15:27:00.651720+0000","mapping_epoch":201,"log_start":"187'6453","ondisk_log_start":"187'6453","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142540800,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13507,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":887,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208148+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:26:57.208148+0000","last_peered":"2024-08-29T15:26:57.208148+0000","last_clean":"2024-08-29T15:26:57.208148+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:26:57.208148+0000","last_undegraded":"2024-08-29T15:26:57.208148+0000","last_fullsized":"2024-08-29T15:26:57.208148+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":425,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.212520+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:26:57.212520+0000","last_peered":"2024-08-29T15:26:57.212520+0000","last_clean":"2024-08-29T15:26:57.212520+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:26:57.212520+0000","last_undegraded":"2024-08-29T15:26:57.212520+0000","last_fullsized":"2024-08-29T15:26:57.212520+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":433,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.212008+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:26:57.212008+0000","last_peered":"2024-08-29T15:26:57.212008+0000","last_clean":"2024-08-29T15:26:57.212008+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:26:57.212008+0000","last_undegraded":"2024-08-29T15:26:57.212008+0000","last_fullsized":"2024-08-29T15:26:57.212008+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"279'13470","reported_seq":23866,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:01.463544+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:27:01.463544+0000","last_peered":"2024-08-29T15:27:01.463544+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:27:01.463544+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"187'6730","ondisk_log_start":"187'6730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145489920,"num_objects":2220,"num_object_clones":0,"num_object_copies":6660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13470,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5094,"num_bytes_recovered":333840384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"3,6","objects":54},{"shards":"3,7","objects":2166}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":696,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.206285+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:26:57.206285+0000","last_peered":"2024-08-29T15:26:57.206285+0000","last_clean":"2024-08-29T15:26:57.206285+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:26:57.206285+0000","last_undegraded":"2024-08-29T15:26:57.206285+0000","last_fullsized":"2024-08-29T15:26:57.206285+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.204724+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:26:43.204724+0000","last_peered":"2024-08-29T15:26:43.204724+0000","last_clean":"2024-08-29T15:26:43.204724+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:26:43.204724+0000","last_undegraded":"2024-08-29T15:26:43.204724+0000","last_fullsized":"2024-08-29T15:26:43.204724+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":459,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208467+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:26:57.208467+0000","last_peered":"2024-08-29T15:26:57.208467+0000","last_clean":"2024-08-29T15:26:57.208467+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:26:57.208467+0000","last_undegraded":"2024-08-29T15:26:57.208467+0000","last_fullsized":"2024-08-29T15:26:57.208467+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"279'13520","reported_seq":13768,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:27:01.355285+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:27:01.355285+0000","last_peered":"2024-08-29T15:27:01.355285+0000","last_clean":"2024-08-29T15:27:01.355285+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:27:01.355285+0000","last_undegraded":"2024-08-29T15:27:01.355285+0000","last_fullsized":"2024-08-29T15:27:01.355285+0000","mapping_epoch":234,"log_start":"187'6797","ondisk_log_start":"187'6797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6723,"log_dups_size":0,"ondisk_log_size":6723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143654912,"num_objects":2192,"num_object_clones":0,"num_object_copies":6576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13520,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":218,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.182694+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:26:43.182694+0000","last_peered":"2024-08-29T15:26:43.182694+0000","last_clean":"2024-08-29T15:26:43.182694+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:26:43.182694+0000","last_undegraded":"2024-08-29T15:26:43.182694+0000","last_fullsized":"2024-08-29T15:26:43.182694+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184456+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:26:43.184456+0000","last_peered":"2024-08-29T15:26:43.184456+0000","last_clean":"2024-08-29T15:26:43.184456+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:26:43.184456+0000","last_undegraded":"2024-08-29T15:26:43.184456+0000","last_fullsized":"2024-08-29T15:26:43.184456+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":125,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.182801+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:26:43.182801+0000","last_peered":"2024-08-29T15:26:43.182801+0000","last_clean":"2024-08-29T15:26:43.182801+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:26:43.182801+0000","last_undegraded":"2024-08-29T15:26:43.182801+0000","last_fullsized":"2024-08-29T15:26:43.182801+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"279'13666","reported_seq":13912,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:03.937041+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:27:03.937041+0000","last_peered":"2024-08-29T15:27:03.937041+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:27:03.937041+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"183'5542","ondisk_log_start":"183'5542","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8124,"log_dups_size":0,"ondisk_log_size":8124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141557760,"num_objects":2160,"num_object_clones":0,"num_object_copies":6480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13666,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-263}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":150,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.183182+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:26:43.183182+0000","last_peered":"2024-08-29T15:26:43.183182+0000","last_clean":"2024-08-29T15:26:43.183182+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:26:43.183182+0000","last_undegraded":"2024-08-29T15:26:43.183182+0000","last_fullsized":"2024-08-29T15:26:43.183182+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":852,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208973+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:26:57.208973+0000","last_peered":"2024-08-29T15:26:57.208973+0000","last_clean":"2024-08-29T15:26:57.208973+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:26:57.208973+0000","last_undegraded":"2024-08-29T15:26:57.208973+0000","last_fullsized":"2024-08-29T15:26:57.208973+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":429,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208408+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:26:57.208408+0000","last_peered":"2024-08-29T15:26:57.208408+0000","last_clean":"2024-08-29T15:26:57.208408+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:26:57.208408+0000","last_undegraded":"2024-08-29T15:26:57.208408+0000","last_fullsized":"2024-08-29T15:26:57.208408+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"279'13503","reported_seq":14305,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:00.720537+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:27:00.720537+0000","last_peered":"2024-08-29T15:27:00.720537+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:27:00.720537+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"187'6480","ondisk_log_start":"187'6480","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141885440,"num_objects":2165,"num_object_clones":0,"num_object_copies":6495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2152,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":2165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13503,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":12},{"shards":"1,7","objects":14}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":434,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208875+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:26:57.208875+0000","last_peered":"2024-08-29T15:26:57.208875+0000","last_clean":"2024-08-29T15:26:57.208875+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:26:57.208875+0000","last_undegraded":"2024-08-29T15:26:57.208875+0000","last_fullsized":"2024-08-29T15:26:57.208875+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":707,"reported_epoch":278,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:43.201349+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:26:43.201349+0000","last_peered":"2024-08-29T15:26:43.201349+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:26:43.201349+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:26:43.201349+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":425,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211099+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:26:57.211099+0000","last_peered":"2024-08-29T15:26:57.211099+0000","last_clean":"2024-08-29T15:26:57.211099+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:26:57.211099+0000","last_undegraded":"2024-08-29T15:26:57.211099+0000","last_fullsized":"2024-08-29T15:26:57.211099+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"279'13283","reported_seq":20448,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:59.538390+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:26:59.538390+0000","last_peered":"2024-08-29T15:26:59.538390+0000","last_clean":"2024-08-29T15:26:59.538390+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:26:59.538390+0000","last_undegraded":"2024-08-29T15:26:59.538390+0000","last_fullsized":"2024-08-29T15:26:59.538390+0000","mapping_epoch":234,"log_start":"180'5299","ondisk_log_start":"180'5299","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7984,"log_dups_size":0,"ondisk_log_size":7984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140574720,"num_objects":2145,"num_object_clones":0,"num_object_copies":6435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13283,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":419,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208639+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:26:57.208639+0000","last_peered":"2024-08-29T15:26:57.208639+0000","last_clean":"2024-08-29T15:26:57.208639+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:26:57.208639+0000","last_undegraded":"2024-08-29T15:26:57.208639+0000","last_fullsized":"2024-08-29T15:26:57.208639+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1041,"reported_epoch":279,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:26:57.208528+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:26:57.208528+0000","last_peered":"2024-08-29T15:26:57.208528+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:26:57.208528+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":417,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205967+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:26:57.205967+0000","last_peered":"2024-08-29T15:26:57.205967+0000","last_clean":"2024-08-29T15:26:57.205967+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:26:57.205967+0000","last_undegraded":"2024-08-29T15:26:57.205967+0000","last_fullsized":"2024-08-29T15:26:57.205967+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"279'13481","reported_seq":23018,"reported_epoch":279,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:04.472102+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:27:04.472102+0000","last_peered":"2024-08-29T15:27:04.472102+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:27:04.472102+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"186'5912","ondisk_log_start":"186'5912","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139526144,"num_objects":2129,"num_object_clones":0,"num_object_copies":6387,"num_objects_missing_on_primary":3165,"num_objects_missing":3165,"num_objects_degraded":2058,"num_objects_misplaced":2128,"num_objects_unfound":0,"num_objects_dirty":2129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13481,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4558,"num_bytes_recovered":298582016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":70},{"shards":"1,7","objects":2058},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":415,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.210880+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:26:57.210880+0000","last_peered":"2024-08-29T15:26:57.210880+0000","last_clean":"2024-08-29T15:26:57.210880+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:26:57.210880+0000","last_undegraded":"2024-08-29T15:26:57.210880+0000","last_fullsized":"2024-08-29T15:26:57.210880+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":131,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.205963+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:26:43.205963+0000","last_peered":"2024-08-29T15:26:43.205963+0000","last_clean":"2024-08-29T15:26:43.205963+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:26:43.205963+0000","last_undegraded":"2024-08-29T15:26:43.205963+0000","last_fullsized":"2024-08-29T15:26:43.205963+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184314+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:26:43.184314+0000","last_peered":"2024-08-29T15:26:43.184314+0000","last_clean":"2024-08-29T15:26:43.184314+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:26:43.184314+0000","last_undegraded":"2024-08-29T15:26:43.184314+0000","last_fullsized":"2024-08-29T15:26:43.184314+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:49.845743+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:26:49.845743+0000","last_peered":"2024-08-29T15:26:49.845743+0000","last_clean":"2024-08-29T15:26:49.845743+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:26:49.845743+0000","last_undegraded":"2024-08-29T15:26:49.845743+0000","last_fullsized":"2024-08-29T15:26:49.845743+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":57,"reported_epoch":278,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:49.845761+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:26:49.845761+0000","last_peered":"2024-08-29T15:26:49.845761+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:26:49.845761+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:26:49.845761+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":423,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208247+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:26:57.208247+0000","last_peered":"2024-08-29T15:26:57.208247+0000","last_clean":"2024-08-29T15:26:57.208247+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:26:57.208247+0000","last_undegraded":"2024-08-29T15:26:57.208247+0000","last_fullsized":"2024-08-29T15:26:57.208247+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":173,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:57.207737+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:26:57.207737+0000","last_peered":"2024-08-29T15:26:57.207737+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:26:57.207737+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:26:57.207737+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":147,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.183108+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:26:43.183108+0000","last_peered":"2024-08-29T15:26:43.183108+0000","last_clean":"2024-08-29T15:26:43.183108+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:26:43.183108+0000","last_undegraded":"2024-08-29T15:26:43.183108+0000","last_fullsized":"2024-08-29T15:26:43.183108+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":382,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205969+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:26:57.205969+0000","last_peered":"2024-08-29T15:26:57.205969+0000","last_clean":"2024-08-29T15:26:57.205969+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:26:57.205969+0000","last_undegraded":"2024-08-29T15:26:57.205969+0000","last_fullsized":"2024-08-29T15:26:57.205969+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":436,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211523+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:26:57.211523+0000","last_peered":"2024-08-29T15:26:57.211523+0000","last_clean":"2024-08-29T15:26:57.211523+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:26:57.211523+0000","last_undegraded":"2024-08-29T15:26:57.211523+0000","last_fullsized":"2024-08-29T15:26:57.211523+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":434,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205842+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:26:57.205842+0000","last_peered":"2024-08-29T15:26:57.205842+0000","last_clean":"2024-08-29T15:26:57.205842+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:26:57.205842+0000","last_undegraded":"2024-08-29T15:26:57.205842+0000","last_fullsized":"2024-08-29T15:26:57.205842+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.206009+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:26:43.206009+0000","last_peered":"2024-08-29T15:26:43.206009+0000","last_clean":"2024-08-29T15:26:43.206009+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:26:43.206009+0000","last_undegraded":"2024-08-29T15:26:43.206009+0000","last_fullsized":"2024-08-29T15:26:43.206009+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":291,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.214970+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:26:57.214970+0000","last_peered":"2024-08-29T15:26:57.214970+0000","last_clean":"2024-08-29T15:26:57.214970+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:26:57.214970+0000","last_undegraded":"2024-08-29T15:26:57.214970+0000","last_fullsized":"2024-08-29T15:26:57.214970+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":411,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211931+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:26:57.211931+0000","last_peered":"2024-08-29T15:26:57.211931+0000","last_clean":"2024-08-29T15:26:57.211931+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:26:57.211931+0000","last_undegraded":"2024-08-29T15:26:57.211931+0000","last_fullsized":"2024-08-29T15:26:57.211931+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":414,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.210016+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:26:57.210016+0000","last_peered":"2024-08-29T15:26:57.210016+0000","last_clean":"2024-08-29T15:26:57.210016+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:26:57.210016+0000","last_undegraded":"2024-08-29T15:26:57.210016+0000","last_fullsized":"2024-08-29T15:26:57.210016+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":419,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207598+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:26:57.207598+0000","last_peered":"2024-08-29T15:26:57.207598+0000","last_clean":"2024-08-29T15:26:57.207598+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:26:57.207598+0000","last_undegraded":"2024-08-29T15:26:57.207598+0000","last_fullsized":"2024-08-29T15:26:57.207598+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":58,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184748+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:26:43.184748+0000","last_peered":"2024-08-29T15:26:43.184748+0000","last_clean":"2024-08-29T15:26:43.184748+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:26:43.184748+0000","last_undegraded":"2024-08-29T15:26:43.184748+0000","last_fullsized":"2024-08-29T15:26:43.184748+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":439,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.209042+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:26:57.209042+0000","last_peered":"2024-08-29T15:26:57.209042+0000","last_clean":"2024-08-29T15:26:57.209042+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:26:57.209042+0000","last_undegraded":"2024-08-29T15:26:57.209042+0000","last_fullsized":"2024-08-29T15:26:57.209042+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":402,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211010+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:26:57.211010+0000","last_peered":"2024-08-29T15:26:57.211010+0000","last_clean":"2024-08-29T15:26:57.211010+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:26:57.211010+0000","last_undegraded":"2024-08-29T15:26:57.211010+0000","last_fullsized":"2024-08-29T15:26:57.211010+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":58,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184475+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:26:43.184475+0000","last_peered":"2024-08-29T15:26:43.184475+0000","last_clean":"2024-08-29T15:26:43.184475+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:26:43.184475+0000","last_undegraded":"2024-08-29T15:26:43.184475+0000","last_fullsized":"2024-08-29T15:26:43.184475+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":171,"reported_epoch":278,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:43.183173+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:26:43.183173+0000","last_peered":"2024-08-29T15:26:43.183173+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:26:43.183173+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:26:43.183173+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":58,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184819+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:26:43.184819+0000","last_peered":"2024-08-29T15:26:43.184819+0000","last_clean":"2024-08-29T15:26:43.184819+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:26:43.184819+0000","last_undegraded":"2024-08-29T15:26:43.184819+0000","last_fullsized":"2024-08-29T15:26:43.184819+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":177,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.183036+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:26:43.183036+0000","last_peered":"2024-08-29T15:26:43.183036+0000","last_clean":"2024-08-29T15:26:43.183036+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:26:43.183036+0000","last_undegraded":"2024-08-29T15:26:43.183036+0000","last_fullsized":"2024-08-29T15:26:43.183036+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":450,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.206397+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:26:57.206397+0000","last_peered":"2024-08-29T15:26:57.206397+0000","last_clean":"2024-08-29T15:26:57.206397+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:26:57.206397+0000","last_undegraded":"2024-08-29T15:26:57.206397+0000","last_fullsized":"2024-08-29T15:26:57.206397+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":452,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.206815+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:26:57.206815+0000","last_peered":"2024-08-29T15:26:57.206815+0000","last_clean":"2024-08-29T15:26:57.206815+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:26:57.206815+0000","last_undegraded":"2024-08-29T15:26:57.206815+0000","last_fullsized":"2024-08-29T15:26:57.206815+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":278,"state":"active+clean","last_fresh":"2024-08-29T15:26:43.184124+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:26:43.184124+0000","last_peered":"2024-08-29T15:26:43.184124+0000","last_clean":"2024-08-29T15:26:43.184124+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:26:43.184124+0000","last_undegraded":"2024-08-29T15:26:43.184124+0000","last_fullsized":"2024-08-29T15:26:43.184124+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":459,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.211426+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:26:57.211426+0000","last_peered":"2024-08-29T15:26:57.211426+0000","last_clean":"2024-08-29T15:26:57.211426+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:26:57.211426+0000","last_undegraded":"2024-08-29T15:26:57.211426+0000","last_fullsized":"2024-08-29T15:26:57.211426+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2318925848,"num_objects":35385,"num_object_clones":0,"num_object_copies":106155,"num_objects_missing_on_primary":3165,"num_objects_missing":3165,"num_objects_degraded":25982,"num_objects_misplaced":7665,"num_objects_unfound":0,"num_objects_dirty":35385,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":215465,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20838,"num_bytes_recovered":1245642752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7355052032,"data_stored":7355039816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124342,"ondisk_log_size":124342,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234388,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1497608,"kb_used_data":1155584,"kb_used_omap":1,"kb_used_meta":342014,"kb_avail":92247544,"statfs":{"total":95995035648,"available":94461485056,"internally_reserved":0,"allocated":1183318016,"data_stored":1181687643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":350222795},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496020,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":210884,"kb_used_data":17340,"kb_used_omap":1,"kb_used_meta":193534,"kb_avail":93534268,"statfs":{"total":95995035648,"available":95779090432,"internally_reserved":0,"allocated":17756160,"data_stored":16129587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":198179271},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,2,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426606,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":787960,"kb_used_data":718896,"kb_used_omap":8,"kb_used_meta":69047,"kb_avail":92957192,"statfs":{"total":95995035648,"available":95188164608,"internally_reserved":0,"allocated":736149504,"data_stored":734530397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":70705034},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019298,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1036328,"kb_used_data":720544,"kb_used_omap":1,"kb_used_meta":315774,"kb_avail":92708824,"statfs":{"total":95995035648,"available":94933835776,"internally_reserved":0,"allocated":737837056,"data_stored":736233732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":323353030},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435483,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1743092,"kb_used_data":1422700,"kb_used_omap":4,"kb_used_meta":320379,"kb_avail":92002060,"statfs":{"total":95995035648,"available":94210109440,"internally_reserved":0,"allocated":1456844800,"data_stored":1455229633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":328068361},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903335,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646520,"kb_used_data":1492592,"kb_used_omap":10,"kb_used_meta":153909,"kb_avail":92098632,"statfs":{"total":95995035648,"available":94308999168,"internally_reserved":0,"allocated":1528414208,"data_stored":1526798977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":157603344},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762536,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1417396,"kb_used_data":1146924,"kb_used_omap":7,"kb_used_meta":270456,"kb_avail":92327756,"statfs":{"total":95995035648,"available":94543622144,"internally_reserved":0,"allocated":1174450176,"data_stored":1172833769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":276947298},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926063,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":895100,"kb_used_data":570484,"kb_used_omap":7,"kb_used_meta":324600,"kb_avail":92850052,"statfs":{"total":95995035648,"available":95078453248,"internally_reserved":0,"allocated":584175616,"data_stored":582551988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":332390543},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":575016960,"data_stored":575012888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1159991296,"data_stored":1159987224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1524695040,"data_stored":1524695040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1445658624,"data_stored":1445658624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":733937664,"data_stored":733937664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":729223168,"data_stored":729219096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":13434880,"data_stored":13434880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1173094400,"data_stored":1173094400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:27:06.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:06.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:06.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:06.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:06.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:06.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:07.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:07.053 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:07.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:07.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:07.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:07.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:07.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:07.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:07 smithi027 ceph-mon[93921]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 26011/106239 objects degraded (24.483%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:07 smithi027 ceph-mon[98970]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 26011/106239 objects degraded (24.483%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:07.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:07.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:07.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:07 smithi171 ceph-mon[90031]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 26011/106239 objects degraded (24.483%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:07.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:07.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:07.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:07.699 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:07.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:07.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:07.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:07.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:07.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:07.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:08.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:08.070 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:27:08.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:08.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:08.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:08.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:08.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:08 smithi027 ceph-mon[93921]: pgmap v961: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 25612/102516 objects degraded (24.983%); 7557/102516 objects misplaced (7.372%); 4.3 MiB/s, 69 objects/s recovering 2024-08-29T15:27:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3697017474' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:08 smithi027 ceph-mon[98970]: pgmap v961: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 25612/102516 objects degraded (24.983%); 7557/102516 objects misplaced (7.372%); 4.3 MiB/s, 69 objects/s recovering 2024-08-29T15:27:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3697017474' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:08.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:08.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:08.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:08 smithi171 ceph-mon[90031]: pgmap v961: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 25612/102516 objects degraded (24.983%); 7557/102516 objects misplaced (7.372%); 4.3 MiB/s, 69 objects/s recovering 2024-08-29T15:27:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3697017474' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:08.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:08.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:08.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:08.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:08.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:08.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:08.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:08.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:08.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:08.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:08.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:09.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:09.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:09.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:09.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:09.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:09.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:09.244 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:09 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:27:09.244 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3697017474' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:09.245 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:09 smithi027 ceph-mon[93921]: osdmap e280: 8 total, 8 up, 8 in 2024-08-29T15:27:09.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:09 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:27:09.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3697017474' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:09.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:09 smithi027 ceph-mon[98970]: osdmap e280: 8 total, 8 up, 8 in 2024-08-29T15:27:09.245 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:27:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:27:09] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.51.0" 2024-08-29T15:27:09.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:09.342 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:09.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:09.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:09 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:27:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3697017474' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:09 smithi171 ceph-mon[90031]: osdmap e280: 8 total, 8 up, 8 in 2024-08-29T15:27:09.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:09.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:09.699 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:09.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:09.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:27:09.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:09.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:09.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:09.953 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:09.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:09.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:09.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:27:09.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:27:09.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:27:09.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:27:09.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:27:09.994 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:27:09.995 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:27:09.995 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:27:09.995 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:27:09.995 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:27:10.012 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:27:10.012 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:27:10.032 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:27:10.032 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:27:10.050 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:27:10.051 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:27:10.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:10.103 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:10.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:10.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:10.220 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:27:10.220 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:27:10.229 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:10.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:10.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:10.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:10.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:10.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:10.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:10.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:10.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:10.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:10.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:10 smithi171 ceph-mon[90031]: pgmap v963: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 25612/102516 objects degraded (24.983%); 7557/102516 objects misplaced (7.372%); 4.3 MiB/s, 69 objects/s recovering 2024-08-29T15:27:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3182744103' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:10.534 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:10 smithi027 ceph-mon[98970]: pgmap v963: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 25612/102516 objects degraded (24.983%); 7557/102516 objects misplaced (7.372%); 4.3 MiB/s, 69 objects/s recovering 2024-08-29T15:27:10.535 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3182744103' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:10.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:10 smithi027 ceph-mon[93921]: pgmap v963: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 25612/102516 objects degraded (24.983%); 7557/102516 objects misplaced (7.372%); 4.3 MiB/s, 69 objects/s recovering 2024-08-29T15:27:10.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3182744103' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:10.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:10.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:10.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:10.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:10.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:10.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:10.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:10.866 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:10.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:10.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:10.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:27:10.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:11.007 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:11.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:11.136 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:11.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:11.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:11.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:11.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:11.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:11.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:11.518 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:11.620 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:11.642 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:11.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:11.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:11.767 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:11.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:11.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:11.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:11.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:11.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:11.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:12.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:12.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:12.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:12.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:12.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:12.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:12.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:12.272 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:12.276 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:12.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:12.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:12 smithi027 ceph-mon[98970]: pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 25443/101232 objects degraded (25.133%); 7472/101232 objects misplaced (7.381%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:27:12.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 25443/101232 objects degraded (25.133%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:12.395 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:12 smithi027 ceph-mon[93921]: pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 25443/101232 objects degraded (25.133%); 7472/101232 objects misplaced (7.381%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:27:12.395 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 25443/101232 objects degraded (25.133%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:12.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:12.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:12.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:12 smithi171 ceph-mon[90031]: pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 25443/101232 objects degraded (25.133%); 7472/101232 objects misplaced (7.381%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:27:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 25443/101232 objects degraded (25.133%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:12.523 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:12.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:12.660 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:12.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:12.793 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:12.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:12.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:12.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:12.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:12.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:13.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:13.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:13.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:13.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:13.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:13.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:13.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:13.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:13.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:13.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:13.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:13.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.673 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:13.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:13.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:27:13.774 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:13.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:13.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:13.926 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:13.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:14.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:14.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:14.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:14.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:14.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:14.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:14.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:27:14.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:14.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:14.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:14.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:14.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:14.467 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:14 smithi171 ceph-mon[90031]: pgmap v965: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 284 op/s; 25200/98499 objects degraded (25.584%); 7396/98499 objects misplaced (7.509%); 4.4 MiB/s, 71 objects/s recovering 2024-08-29T15:27:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:14.536 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:14.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:14 smithi027 ceph-mon[98970]: pgmap v965: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 284 op/s; 25200/98499 objects degraded (25.584%); 7396/98499 objects misplaced (7.509%); 4.4 MiB/s, 71 objects/s recovering 2024-08-29T15:27:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:14 smithi027 ceph-mon[93921]: pgmap v965: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 284 op/s; 25200/98499 objects degraded (25.584%); 7396/98499 objects misplaced (7.509%); 4.4 MiB/s, 71 objects/s recovering 2024-08-29T15:27:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:14.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:14.684 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:14.786 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:14.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:14.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:14.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:14.934 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:14.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:15.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:15.061 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:15.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:15.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:15.273 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:15.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:15.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:15.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:15.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:15.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:15.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:15.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:15.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:15.706 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:15.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:15.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:15.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:15.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:15.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:16.025 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:27:16.025 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:27:16.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:16.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:16.148 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:16 smithi027 ceph-mon[93921]: pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 25109/97818 objects degraded (25.669%); 7305/97818 objects misplaced (7.468%); 4.5 MiB/s, 71 objects/s recovering 2024-08-29T15:27:16.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:16 smithi027 ceph-mon[98970]: pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 25109/97818 objects degraded (25.669%); 7305/97818 objects misplaced (7.468%); 4.5 MiB/s, 71 objects/s recovering 2024-08-29T15:27:16.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:16.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:16.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:16 smithi171 ceph-mon[90031]: pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 25109/97818 objects degraded (25.669%); 7305/97818 objects misplaced (7.468%); 4.5 MiB/s, 71 objects/s recovering 2024-08-29T15:27:16.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:16.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:16.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:16.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:16.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.468 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:16.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:16.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:16.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:16.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:16.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:16.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:16.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:16.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:16.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:16.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:16.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:16.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:16.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:16.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:17.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:17.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:17.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:17.226 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:17.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 25109/97818 objects degraded (25.669%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:17.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 25109/97818 objects degraded (25.669%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:17.329 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:17.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:17.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:17.455 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:17.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:17.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:17.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 25109/97818 objects degraded (25.669%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:17.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:17.603 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:17.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:17.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:17.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:17.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:17.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:17.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:17.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:17.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:17.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:18.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:18.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:18.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:18.239 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:18.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:18.362 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:18.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:18.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:18.487 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:18 smithi171 ceph-mon[90031]: pgmap v967: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 24780/94800 objects degraded (26.139%); 7238/94800 objects misplaced (7.635%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:18 smithi027 ceph-mon[98970]: pgmap v967: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 24780/94800 objects degraded (26.139%); 7238/94800 objects misplaced (7.635%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:18 smithi027 ceph-mon[93921]: pgmap v967: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 24780/94800 objects degraded (26.139%); 7238/94800 objects misplaced (7.635%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:18.589 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:18.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:18.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:18.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:18.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.863 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:18.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:18.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:18.992 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:18.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:18.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:19.094 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:19.119 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:19.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:19.166 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:27:19.167 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:27:19.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:19.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:19.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:19.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:27:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:27:19] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.51.0" 2024-08-29T15:27:19.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:19.364 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:19.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:19.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:19.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:19.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:19.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:19.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:19.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:19.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:19.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:19.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:19.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:19.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:19.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:19.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:19.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:19.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:27:20.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:20.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:20.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:20.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:20.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:20.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:20.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:20 smithi027 ceph-mon[93921]: pgmap v968: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 24780/94800 objects degraded (26.139%); 7238/94800 objects misplaced (7.635%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:20.197 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:20 smithi027 ceph-mon[93921]: from='client.46016 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:20.197 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:20 smithi027 ceph-mon[98970]: pgmap v968: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 24780/94800 objects degraded (26.139%); 7238/94800 objects misplaced (7.635%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:20.197 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:20 smithi027 ceph-mon[98970]: from='client.46016 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:20.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:20.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:20.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:20.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:20.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:20.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:20 smithi171 ceph-mon[90031]: pgmap v968: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 242 op/s; 24780/94800 objects degraded (26.139%); 7238/94800 objects misplaced (7.635%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:27:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:20 smithi171 ceph-mon[90031]: from='client.46016 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:20.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:20.567 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:20.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:20.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:20.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:20.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:20.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:20.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:20.898 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":968,"stamp":"2024-08-29T15:27:18.664682+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2073827325,"num_objects":31600,"num_object_clones":0,"num_object_copies":94800,"num_objects_missing_on_primary":2997,"num_objects_missing":2997,"num_objects_degraded":24780,"num_objects_misplaced":7238,"num_objects_unfound":0,"num_objects_dirty":31600,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":220401,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21931,"num_bytes_recovered":1316097229,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125337,"ondisk_log_size":125337,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8650760,"kb_used_data":6640264,"kb_used_omap":43,"kb_used_meta":2010388,"kb_avail":741310456,"statfs":{"total":767960285184,"available":759101906944,"internally_reserved":0,"allocated":6799630336,"data_stored":6786619342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2058637804},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-168558592,"num_objects":-2572,"num_object_clones":0,"num_object_copies":-7716,"num_objects_missing_on_primary":-176,"num_objects_missing":-176,"num_objects_degraded":-832,"num_objects_misplaced":-319,"num_objects_unfound":0,"num_objects_dirty":-2572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2572,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":684,"num_bytes_recovered":44826624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.594289"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":446,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.074167+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:27:08.074167+0000","last_peered":"2024-08-29T15:27:08.074167+0000","last_clean":"2024-08-29T15:27:08.074167+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:27:08.074167+0000","last_undegraded":"2024-08-29T15:27:08.074167+0000","last_fullsized":"2024-08-29T15:27:08.074167+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":59,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:57.208426+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:26:57.208426+0000","last_peered":"2024-08-29T15:26:57.208426+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:26:57.208426+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:26:57.208426+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":149,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.071454+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:27:08.071454+0000","last_peered":"2024-08-29T15:27:08.071454+0000","last_clean":"2024-08-29T15:27:08.071454+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:27:08.071454+0000","last_undegraded":"2024-08-29T15:27:08.071454+0000","last_fullsized":"2024-08-29T15:27:08.071454+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":412,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091290+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:27:08.091290+0000","last_peered":"2024-08-29T15:27:08.091290+0000","last_clean":"2024-08-29T15:27:08.091290+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:27:08.091290+0000","last_undegraded":"2024-08-29T15:27:08.091290+0000","last_fullsized":"2024-08-29T15:27:08.091290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":412,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091330+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:27:08.091330+0000","last_peered":"2024-08-29T15:27:08.091330+0000","last_clean":"2024-08-29T15:27:08.091330+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:27:08.091330+0000","last_undegraded":"2024-08-29T15:27:08.091330+0000","last_fullsized":"2024-08-29T15:27:08.091330+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208095+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:26:57.208095+0000","last_peered":"2024-08-29T15:26:57.208095+0000","last_clean":"2024-08-29T15:26:57.208095+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:26:57.208095+0000","last_undegraded":"2024-08-29T15:26:57.208095+0000","last_fullsized":"2024-08-29T15:26:57.208095+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":223,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.076241+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:27:08.076241+0000","last_peered":"2024-08-29T15:27:08.076241+0000","last_clean":"2024-08-29T15:27:08.076241+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:27:08.076241+0000","last_undegraded":"2024-08-29T15:27:08.076241+0000","last_fullsized":"2024-08-29T15:27:08.076241+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":150,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207793+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:26:57.207793+0000","last_peered":"2024-08-29T15:26:57.207793+0000","last_clean":"2024-08-29T15:26:57.207793+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:26:57.207793+0000","last_undegraded":"2024-08-29T15:26:57.207793+0000","last_fullsized":"2024-08-29T15:26:57.207793+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"280'13861","reported_seq":21921,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:16.197875+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:27:16.197875+0000","last_peered":"2024-08-29T15:27:16.197875+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:27:16.197875+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"179'3800","ondisk_log_start":"179'3800","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10061,"log_dups_size":0,"ondisk_log_size":10061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134414336,"num_objects":2051,"num_object_clones":0,"num_object_copies":6153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13861,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,5","objects":-874}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":58,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:57.208784+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:26:57.208784+0000","last_peered":"2024-08-29T15:26:57.208784+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:26:57.208784+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:26:57.208784+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":167,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205741+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:26:57.205741+0000","last_peered":"2024-08-29T15:26:57.205741+0000","last_clean":"2024-08-29T15:26:57.205741+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:26:57.205741+0000","last_undegraded":"2024-08-29T15:26:57.205741+0000","last_fullsized":"2024-08-29T15:26:57.205741+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"280'13551","reported_seq":13791,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:16.276314+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:27:16.276314+0000","last_peered":"2024-08-29T15:27:16.276314+0000","last_clean":"2024-08-29T15:27:16.276314+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:27:16.276314+0000","last_undegraded":"2024-08-29T15:27:16.276314+0000","last_fullsized":"2024-08-29T15:27:16.276314+0000","mapping_epoch":168,"log_start":"187'6828","ondisk_log_start":"187'6828","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6723,"log_dups_size":0,"ondisk_log_size":6723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006464,"num_objects":1999,"num_object_clones":0,"num_object_copies":5997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13551,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":58,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:57.216742+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:26:57.216742+0000","last_peered":"2024-08-29T15:26:57.216742+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:26:57.216742+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:26:57.216742+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207886+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:26:57.207886+0000","last_peered":"2024-08-29T15:26:57.207886+0000","last_clean":"2024-08-29T15:26:57.207886+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:26:57.207886+0000","last_undegraded":"2024-08-29T15:26:57.207886+0000","last_fullsized":"2024-08-29T15:26:57.207886+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"280'13707","reported_seq":13917,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:16.350719+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:27:16.350719+0000","last_peered":"2024-08-29T15:27:16.350719+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:27:16.350719+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"187'6974","ondisk_log_start":"187'6974","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6733,"log_dups_size":0,"ondisk_log_size":6733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131399680,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13707,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-472}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":228,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091085+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:27:08.091085+0000","last_peered":"2024-08-29T15:27:08.091085+0000","last_clean":"2024-08-29T15:27:08.091085+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:27:08.091085+0000","last_undegraded":"2024-08-29T15:27:08.091085+0000","last_fullsized":"2024-08-29T15:27:08.091085+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":105,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208131+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:26:57.208131+0000","last_peered":"2024-08-29T15:26:57.208131+0000","last_clean":"2024-08-29T15:26:57.208131+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:26:57.208131+0000","last_undegraded":"2024-08-29T15:26:57.208131+0000","last_fullsized":"2024-08-29T15:26:57.208131+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"280'13780","reported_seq":19240,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:13.971501+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:27:13.971501+0000","last_peered":"2024-08-29T15:27:13.971501+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:27:13.971501+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"185'5662","ondisk_log_start":"185'5662","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8118,"log_dups_size":0,"ondisk_log_size":8118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130940928,"num_objects":1998,"num_object_clones":0,"num_object_copies":5994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13780,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1998}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.206316+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:26:57.206316+0000","last_peered":"2024-08-29T15:26:57.206316+0000","last_clean":"2024-08-29T15:26:57.206316+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:26:57.206316+0000","last_undegraded":"2024-08-29T15:26:57.206316+0000","last_fullsized":"2024-08-29T15:26:57.206316+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":426,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091413+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:27:08.091413+0000","last_peered":"2024-08-29T15:27:08.091413+0000","last_clean":"2024-08-29T15:27:08.091413+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:27:08.091413+0000","last_undegraded":"2024-08-29T15:27:08.091413+0000","last_fullsized":"2024-08-29T15:27:08.091413+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"280'13789","reported_seq":14861,"reported_epoch":280,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:14.716379+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:27:14.716379+0000","last_peered":"2024-08-29T15:27:14.716379+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:27:14.716379+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:27:14.716379+0000","mapping_epoch":234,"log_start":"186'5833","ondisk_log_start":"186'5833","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7956,"log_dups_size":0,"ondisk_log_size":7956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3208,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13789,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":452,"num_bytes_recovered":29622272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":2025}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":421,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.073084+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:27:08.073084+0000","last_peered":"2024-08-29T15:27:08.073084+0000","last_clean":"2024-08-29T15:27:08.073084+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:27:08.073084+0000","last_undegraded":"2024-08-29T15:27:08.073084+0000","last_fullsized":"2024-08-29T15:27:08.073084+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208850+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:26:57.208850+0000","last_peered":"2024-08-29T15:26:57.208850+0000","last_clean":"2024-08-29T15:26:57.208850+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:26:57.208850+0000","last_undegraded":"2024-08-29T15:26:57.208850+0000","last_fullsized":"2024-08-29T15:26:57.208850+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"280'13847","reported_seq":15043,"reported_epoch":280,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:16.308525+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:27:16.308525+0000","last_peered":"2024-08-29T15:27:16.308525+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:27:16.308525+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:27:16.308525+0000","mapping_epoch":234,"log_start":"187'7168","ondisk_log_start":"187'7168","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6679,"log_dups_size":0,"ondisk_log_size":6679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3170,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13847,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":470,"num_bytes_recovered":30801920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":1995}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208004+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:26:57.208004+0000","last_peered":"2024-08-29T15:26:57.208004+0000","last_clean":"2024-08-29T15:26:57.208004+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:26:57.208004+0000","last_undegraded":"2024-08-29T15:26:57.208004+0000","last_fullsized":"2024-08-29T15:26:57.208004+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208764+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:26:57.208764+0000","last_peered":"2024-08-29T15:26:57.208764+0000","last_clean":"2024-08-29T15:26:57.208764+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:26:57.208764+0000","last_undegraded":"2024-08-29T15:26:57.208764+0000","last_fullsized":"2024-08-29T15:26:57.208764+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"280'13707","reported_seq":14148,"reported_epoch":280,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:27:16.369764+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:27:16.369764+0000","last_peered":"2024-08-29T15:27:16.369764+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:27:16.369764+0000","last_undegraded":"2024-08-29T15:27:16.369764+0000","last_fullsized":"2024-08-29T15:27:16.369764+0000","mapping_epoch":202,"log_start":"187'6967","ondisk_log_start":"187'6967","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127860736,"num_objects":1951,"num_object_clones":0,"num_object_copies":5853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1951,"num_objects_unfound":0,"num_objects_dirty":1951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13707,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1951}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207956+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:26:57.207956+0000","last_peered":"2024-08-29T15:26:57.207956+0000","last_clean":"2024-08-29T15:26:57.207956+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:26:57.207956+0000","last_undegraded":"2024-08-29T15:26:57.207956+0000","last_fullsized":"2024-08-29T15:26:57.207956+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":266,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.076113+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:27:08.076113+0000","last_peered":"2024-08-29T15:27:08.076113+0000","last_clean":"2024-08-29T15:27:08.076113+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:27:08.076113+0000","last_undegraded":"2024-08-29T15:27:08.076113+0000","last_fullsized":"2024-08-29T15:27:08.076113+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"280'13802","reported_seq":13964,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:16.143957+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:27:16.143957+0000","last_peered":"2024-08-29T15:27:16.143957+0000","last_clean":"2024-08-29T15:27:16.143957+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:27:16.143957+0000","last_undegraded":"2024-08-29T15:27:16.143957+0000","last_fullsized":"2024-08-29T15:27:16.143957+0000","mapping_epoch":218,"log_start":"179'3778","ondisk_log_start":"179'3778","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10024,"log_dups_size":0,"ondisk_log_size":10024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128253976,"num_objects":1958,"num_object_clones":0,"num_object_copies":5874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1958,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13802,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":424,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.073678+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:27:08.073678+0000","last_peered":"2024-08-29T15:27:08.073678+0000","last_clean":"2024-08-29T15:27:08.073678+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:27:08.073678+0000","last_undegraded":"2024-08-29T15:27:08.073678+0000","last_fullsized":"2024-08-29T15:27:08.073678+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":416,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.076312+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:27:08.076312+0000","last_peered":"2024-08-29T15:27:08.076312+0000","last_clean":"2024-08-29T15:27:08.076312+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:27:08.076312+0000","last_undegraded":"2024-08-29T15:27:08.076312+0000","last_fullsized":"2024-08-29T15:27:08.076312+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"280'13249","reported_seq":13459,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:16.068353+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:27:16.068353+0000","last_peered":"2024-08-29T15:27:16.068353+0000","last_clean":"2024-08-29T15:27:16.068353+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:27:16.068353+0000","last_undegraded":"2024-08-29T15:27:16.068353+0000","last_fullsized":"2024-08-29T15:27:16.068353+0000","mapping_epoch":201,"log_start":"178'3200","ondisk_log_start":"178'3200","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10049,"log_dups_size":0,"ondisk_log_size":10049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128385024,"num_objects":1959,"num_object_clones":0,"num_object_copies":5877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13249,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":515,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.090727+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:27:08.090727+0000","last_peered":"2024-08-29T15:27:08.090727+0000","last_clean":"2024-08-29T15:27:08.090727+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:27:08.090727+0000","last_undegraded":"2024-08-29T15:27:08.090727+0000","last_fullsized":"2024-08-29T15:27:08.090727+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":751,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205898+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:26:57.205898+0000","last_peered":"2024-08-29T15:26:57.205898+0000","last_clean":"2024-08-29T15:26:57.205898+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:26:57.205898+0000","last_undegraded":"2024-08-29T15:26:57.205898+0000","last_fullsized":"2024-08-29T15:26:57.205898+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":259,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.206083+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:26:57.206083+0000","last_peered":"2024-08-29T15:26:57.206083+0000","last_clean":"2024-08-29T15:26:57.206083+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:26:57.206083+0000","last_undegraded":"2024-08-29T15:26:57.206083+0000","last_fullsized":"2024-08-29T15:26:57.206083+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":121,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208570+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:26:57.208570+0000","last_peered":"2024-08-29T15:26:57.208570+0000","last_clean":"2024-08-29T15:26:57.208570+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:26:57.208570+0000","last_undegraded":"2024-08-29T15:26:57.208570+0000","last_fullsized":"2024-08-29T15:26:57.208570+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"280'13754","reported_seq":14192,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:15.759597+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:27:15.759597+0000","last_peered":"2024-08-29T15:27:15.759597+0000","last_clean":"2024-08-29T15:27:15.759597+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:27:15.759597+0000","last_undegraded":"2024-08-29T15:27:15.759597+0000","last_fullsized":"2024-08-29T15:27:15.759597+0000","mapping_epoch":201,"log_start":"187'6753","ondisk_log_start":"187'6753","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126353408,"num_objects":1928,"num_object_clones":0,"num_object_copies":5784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13754,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":889,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.075426+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:27:08.075426+0000","last_peered":"2024-08-29T15:27:08.075426+0000","last_clean":"2024-08-29T15:27:08.075426+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:27:08.075426+0000","last_undegraded":"2024-08-29T15:27:08.075426+0000","last_fullsized":"2024-08-29T15:27:08.075426+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":427,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091224+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:27:08.091224+0000","last_peered":"2024-08-29T15:27:08.091224+0000","last_clean":"2024-08-29T15:27:08.091224+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:27:08.091224+0000","last_undegraded":"2024-08-29T15:27:08.091224+0000","last_fullsized":"2024-08-29T15:27:08.091224+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":435,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091245+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:27:08.091245+0000","last_peered":"2024-08-29T15:27:08.091245+0000","last_clean":"2024-08-29T15:27:08.091245+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:27:08.091245+0000","last_undegraded":"2024-08-29T15:27:08.091245+0000","last_fullsized":"2024-08-29T15:27:08.091245+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"280'13741","reported_seq":24681,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:16.380404+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:27:16.380404+0000","last_peered":"2024-08-29T15:27:16.380404+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:27:16.380404+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"187'7030","ondisk_log_start":"187'7030","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127729664,"num_objects":1949,"num_object_clones":0,"num_object_copies":5847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2426,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13741,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5365,"num_bytes_recovered":351600640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"3,6","objects":54},{"shards":"3,7","objects":1895}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":698,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.072242+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:27:08.072242+0000","last_peered":"2024-08-29T15:27:08.072242+0000","last_clean":"2024-08-29T15:27:08.072242+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:27:08.072242+0000","last_undegraded":"2024-08-29T15:27:08.072242+0000","last_fullsized":"2024-08-29T15:27:08.072242+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":112,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207150+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:26:57.207150+0000","last_peered":"2024-08-29T15:26:57.207150+0000","last_clean":"2024-08-29T15:26:57.207150+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:26:57.207150+0000","last_undegraded":"2024-08-29T15:26:57.207150+0000","last_fullsized":"2024-08-29T15:26:57.207150+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":461,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.076123+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:27:08.076123+0000","last_peered":"2024-08-29T15:27:08.076123+0000","last_clean":"2024-08-29T15:27:08.076123+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:27:08.076123+0000","last_undegraded":"2024-08-29T15:27:08.076123+0000","last_fullsized":"2024-08-29T15:27:08.076123+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"280'13764","reported_seq":14014,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:16.288385+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:27:16.288385+0000","last_peered":"2024-08-29T15:27:16.288385+0000","last_clean":"2024-08-29T15:27:16.288385+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:27:16.288385+0000","last_undegraded":"2024-08-29T15:27:16.288385+0000","last_fullsized":"2024-08-29T15:27:16.288385+0000","mapping_epoch":234,"log_start":"187'7097","ondisk_log_start":"187'7097","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6667,"log_dups_size":0,"ondisk_log_size":6667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127664128,"num_objects":1948,"num_object_clones":0,"num_object_copies":5844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13764,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":219,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205990+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:26:57.205990+0000","last_peered":"2024-08-29T15:26:57.205990+0000","last_clean":"2024-08-29T15:26:57.205990+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:26:57.205990+0000","last_undegraded":"2024-08-29T15:26:57.205990+0000","last_fullsized":"2024-08-29T15:26:57.205990+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208300+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:26:57.208300+0000","last_peered":"2024-08-29T15:26:57.208300+0000","last_clean":"2024-08-29T15:26:57.208300+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:26:57.208300+0000","last_undegraded":"2024-08-29T15:26:57.208300+0000","last_fullsized":"2024-08-29T15:26:57.208300+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":126,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205866+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:26:57.205866+0000","last_peered":"2024-08-29T15:26:57.205866+0000","last_clean":"2024-08-29T15:26:57.205866+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:26:57.205866+0000","last_undegraded":"2024-08-29T15:26:57.205866+0000","last_fullsized":"2024-08-29T15:26:57.205866+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"280'13810","reported_seq":14058,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:13.962542+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:27:13.962542+0000","last_peered":"2024-08-29T15:27:13.962542+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:27:13.962542+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"184'5642","ondisk_log_start":"184'5642","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8168,"log_dups_size":0,"ondisk_log_size":8168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132120576,"num_objects":2016,"num_object_clones":0,"num_object_copies":6048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13810,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-407}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205834+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:26:57.205834+0000","last_peered":"2024-08-29T15:26:57.205834+0000","last_clean":"2024-08-29T15:26:57.205834+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:26:57.205834+0000","last_undegraded":"2024-08-29T15:26:57.205834+0000","last_fullsized":"2024-08-29T15:26:57.205834+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":854,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.076188+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:27:08.076188+0000","last_peered":"2024-08-29T15:27:08.076188+0000","last_clean":"2024-08-29T15:27:08.076188+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:27:08.076188+0000","last_undegraded":"2024-08-29T15:27:08.076188+0000","last_fullsized":"2024-08-29T15:27:08.076188+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":431,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.075801+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:27:08.075801+0000","last_peered":"2024-08-29T15:27:08.075801+0000","last_clean":"2024-08-29T15:27:08.075801+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:27:08.075801+0000","last_undegraded":"2024-08-29T15:27:08.075801+0000","last_fullsized":"2024-08-29T15:27:08.075801+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"280'13752","reported_seq":14556,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:15.528348+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:27:15.528348+0000","last_peered":"2024-08-29T15:27:15.528348+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:27:15.528348+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"187'6680","ondisk_log_start":"187'6680","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125566976,"num_objects":1916,"num_object_clones":0,"num_object_copies":5748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1905,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":1916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13752,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":10},{"shards":"1,7","objects":-235}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":436,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.076095+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:27:08.076095+0000","last_peered":"2024-08-29T15:27:08.076095+0000","last_clean":"2024-08-29T15:27:08.076095+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:27:08.076095+0000","last_undegraded":"2024-08-29T15:27:08.076095+0000","last_fullsized":"2024-08-29T15:27:08.076095+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":708,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:57.214900+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:26:57.214900+0000","last_peered":"2024-08-29T15:26:57.214900+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:26:57.214900+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:26:57.214900+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":427,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.090880+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:27:08.090880+0000","last_peered":"2024-08-29T15:27:08.090880+0000","last_clean":"2024-08-29T15:27:08.090880+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:27:08.090880+0000","last_undegraded":"2024-08-29T15:27:08.090880+0000","last_fullsized":"2024-08-29T15:27:08.090880+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"280'13510","reported_seq":20676,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:14.627173+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:27:14.627173+0000","last_peered":"2024-08-29T15:27:14.627173+0000","last_clean":"2024-08-29T15:27:14.627173+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:27:14.627173+0000","last_undegraded":"2024-08-29T15:27:14.627173+0000","last_fullsized":"2024-08-29T15:27:14.627173+0000","mapping_epoch":234,"log_start":"185'5599","ondisk_log_start":"185'5599","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7911,"log_dups_size":0,"ondisk_log_size":7911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125698048,"num_objects":1918,"num_object_clones":0,"num_object_copies":5754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13510,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":421,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.075049+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:27:08.075049+0000","last_peered":"2024-08-29T15:27:08.075049+0000","last_clean":"2024-08-29T15:27:08.075049+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:27:08.075049+0000","last_undegraded":"2024-08-29T15:27:08.075049+0000","last_fullsized":"2024-08-29T15:27:08.075049+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1043,"reported_epoch":280,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:08.075241+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:27:08.075241+0000","last_peered":"2024-08-29T15:27:08.075241+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:27:08.075241+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":419,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.072067+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:27:08.072067+0000","last_peered":"2024-08-29T15:27:08.072067+0000","last_clean":"2024-08-29T15:27:08.072067+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:27:08.072067+0000","last_undegraded":"2024-08-29T15:27:08.072067+0000","last_fullsized":"2024-08-29T15:27:08.072067+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"280'13654","reported_seq":23721,"reported_epoch":280,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:14.526327+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:27:14.526327+0000","last_peered":"2024-08-29T15:27:14.526327+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:27:14.526327+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"186'6112","ondisk_log_start":"186'6112","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7542,"log_dups_size":0,"ondisk_log_size":7542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128188416,"num_objects":1956,"num_object_clones":0,"num_object_copies":5868,"num_objects_missing_on_primary":2989,"num_objects_missing":2989,"num_objects_degraded":1882,"num_objects_misplaced":1952,"num_objects_unfound":0,"num_objects_dirty":1956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13654,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4734,"num_bytes_recovered":310116352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":70},{"shards":"1,7","objects":1882},{"shards":"2,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":417,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.090759+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:27:08.090759+0000","last_peered":"2024-08-29T15:27:08.090759+0000","last_clean":"2024-08-29T15:27:08.090759+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:27:08.090759+0000","last_undegraded":"2024-08-29T15:27:08.090759+0000","last_fullsized":"2024-08-29T15:27:08.090759+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":132,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208334+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:26:57.208334+0000","last_peered":"2024-08-29T15:26:57.208334+0000","last_clean":"2024-08-29T15:26:57.208334+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:26:57.208334+0000","last_undegraded":"2024-08-29T15:26:57.208334+0000","last_fullsized":"2024-08-29T15:26:57.208334+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208023+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:26:57.208023+0000","last_peered":"2024-08-29T15:26:57.208023+0000","last_clean":"2024-08-29T15:26:57.208023+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:26:57.208023+0000","last_undegraded":"2024-08-29T15:26:57.208023+0000","last_fullsized":"2024-08-29T15:26:57.208023+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208466+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:26:57.208466+0000","last_peered":"2024-08-29T15:26:57.208466+0000","last_clean":"2024-08-29T15:26:57.208466+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:26:57.208466+0000","last_undegraded":"2024-08-29T15:26:57.208466+0000","last_fullsized":"2024-08-29T15:26:57.208466+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":58,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:57.208768+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:26:57.208768+0000","last_peered":"2024-08-29T15:26:57.208768+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:26:57.208768+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:26:57.208768+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":425,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.075705+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:27:08.075705+0000","last_peered":"2024-08-29T15:27:08.075705+0000","last_clean":"2024-08-29T15:27:08.075705+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:27:08.075705+0000","last_undegraded":"2024-08-29T15:27:08.075705+0000","last_fullsized":"2024-08-29T15:27:08.075705+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":175,"reported_epoch":280,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:08.073571+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:27:08.073571+0000","last_peered":"2024-08-29T15:27:08.073571+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:27:08.073571+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:27:08.073571+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":149,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.071569+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:27:08.071569+0000","last_peered":"2024-08-29T15:27:08.071569+0000","last_clean":"2024-08-29T15:27:08.071569+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:27:08.071569+0000","last_undegraded":"2024-08-29T15:27:08.071569+0000","last_fullsized":"2024-08-29T15:27:08.071569+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":384,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.072005+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:27:08.072005+0000","last_peered":"2024-08-29T15:27:08.072005+0000","last_clean":"2024-08-29T15:27:08.072005+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:27:08.072005+0000","last_undegraded":"2024-08-29T15:27:08.072005+0000","last_fullsized":"2024-08-29T15:27:08.072005+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":438,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091015+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:27:08.091015+0000","last_peered":"2024-08-29T15:27:08.091015+0000","last_clean":"2024-08-29T15:27:08.091015+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:27:08.091015+0000","last_undegraded":"2024-08-29T15:27:08.091015+0000","last_fullsized":"2024-08-29T15:27:08.091015+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":436,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.071966+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:27:08.071966+0000","last_peered":"2024-08-29T15:27:08.071966+0000","last_clean":"2024-08-29T15:27:08.071966+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:27:08.071966+0000","last_undegraded":"2024-08-29T15:27:08.071966+0000","last_fullsized":"2024-08-29T15:27:08.071966+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":121,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.080245+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:27:08.080245+0000","last_peered":"2024-08-29T15:27:08.080245+0000","last_clean":"2024-08-29T15:27:08.080245+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:27:08.080245+0000","last_undegraded":"2024-08-29T15:27:08.080245+0000","last_fullsized":"2024-08-29T15:27:08.080245+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":293,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.090606+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:27:08.090606+0000","last_peered":"2024-08-29T15:27:08.090606+0000","last_clean":"2024-08-29T15:27:08.090606+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:27:08.090606+0000","last_undegraded":"2024-08-29T15:27:08.090606+0000","last_fullsized":"2024-08-29T15:27:08.090606+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":413,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.091156+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:27:08.091156+0000","last_peered":"2024-08-29T15:27:08.091156+0000","last_clean":"2024-08-29T15:27:08.091156+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:27:08.091156+0000","last_undegraded":"2024-08-29T15:27:08.091156+0000","last_fullsized":"2024-08-29T15:27:08.091156+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":417,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.075659+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:27:08.075659+0000","last_peered":"2024-08-29T15:27:08.075659+0000","last_clean":"2024-08-29T15:27:08.075659+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:27:08.075659+0000","last_undegraded":"2024-08-29T15:27:08.075659+0000","last_fullsized":"2024-08-29T15:27:08.075659+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":421,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.073413+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:27:08.073413+0000","last_peered":"2024-08-29T15:27:08.073413+0000","last_clean":"2024-08-29T15:27:08.073413+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:27:08.073413+0000","last_undegraded":"2024-08-29T15:27:08.073413+0000","last_fullsized":"2024-08-29T15:27:08.073413+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":59,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207731+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:26:57.207731+0000","last_peered":"2024-08-29T15:26:57.207731+0000","last_clean":"2024-08-29T15:26:57.207731+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:26:57.207731+0000","last_undegraded":"2024-08-29T15:26:57.207731+0000","last_fullsized":"2024-08-29T15:26:57.207731+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":441,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.076307+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:27:08.076307+0000","last_peered":"2024-08-29T15:27:08.076307+0000","last_clean":"2024-08-29T15:27:08.076307+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:27:08.076307+0000","last_undegraded":"2024-08-29T15:27:08.076307+0000","last_fullsized":"2024-08-29T15:27:08.076307+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":404,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.090805+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:27:08.090805+0000","last_peered":"2024-08-29T15:27:08.090805+0000","last_clean":"2024-08-29T15:27:08.090805+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:27:08.090805+0000","last_undegraded":"2024-08-29T15:27:08.090805+0000","last_fullsized":"2024-08-29T15:27:08.090805+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":59,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.208228+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:26:57.208228+0000","last_peered":"2024-08-29T15:26:57.208228+0000","last_clean":"2024-08-29T15:26:57.208228+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:26:57.208228+0000","last_undegraded":"2024-08-29T15:26:57.208228+0000","last_fullsized":"2024-08-29T15:26:57.208228+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":172,"reported_epoch":279,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:26:57.206027+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:26:57.206027+0000","last_peered":"2024-08-29T15:26:57.206027+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:26:57.206027+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:26:57.206027+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":59,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.207796+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:26:57.207796+0000","last_peered":"2024-08-29T15:26:57.207796+0000","last_clean":"2024-08-29T15:26:57.207796+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:26:57.207796+0000","last_undegraded":"2024-08-29T15:26:57.207796+0000","last_fullsized":"2024-08-29T15:26:57.207796+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":178,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:26:57.205939+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:26:57.205939+0000","last_peered":"2024-08-29T15:26:57.205939+0000","last_clean":"2024-08-29T15:26:57.205939+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:26:57.205939+0000","last_undegraded":"2024-08-29T15:26:57.205939+0000","last_fullsized":"2024-08-29T15:26:57.205939+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":452,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.072328+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:27:08.072328+0000","last_peered":"2024-08-29T15:27:08.072328+0000","last_clean":"2024-08-29T15:27:08.072328+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:27:08.072328+0000","last_undegraded":"2024-08-29T15:27:08.072328+0000","last_fullsized":"2024-08-29T15:27:08.072328+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":454,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.073904+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:27:08.073904+0000","last_peered":"2024-08-29T15:27:08.073904+0000","last_clean":"2024-08-29T15:27:08.073904+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:27:08.073904+0000","last_undegraded":"2024-08-29T15:27:08.073904+0000","last_fullsized":"2024-08-29T15:27:08.073904+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":279,"state":"active+clean","last_fresh":"2024-08-29T15:27:04.847333+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:27:04.847333+0000","last_peered":"2024-08-29T15:27:04.847333+0000","last_clean":"2024-08-29T15:27:04.847333+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:27:04.847333+0000","last_undegraded":"2024-08-29T15:27:04.847333+0000","last_fullsized":"2024-08-29T15:27:04.847333+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":461,"reported_epoch":280,"state":"active+clean","last_fresh":"2024-08-29T15:27:08.090867+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:27:08.090867+0000","last_peered":"2024-08-29T15:27:08.090867+0000","last_clean":"2024-08-29T15:27:08.090867+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:27:08.090867+0000","last_undegraded":"2024-08-29T15:27:08.090867+0000","last_fullsized":"2024-08-29T15:27:08.090867+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2069037080,"num_objects":31572,"num_object_clones":0,"num_object_copies":94716,"num_objects_missing_on_primary":2989,"num_objects_missing":2989,"num_objects_degraded":24751,"num_objects_misplaced":7238,"num_objects_unfound":0,"num_objects_dirty":31572,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":219278,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21822,"num_bytes_recovered":1310130176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6732656640,"data_stored":6732644424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124155,"ondisk_log_size":124155,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234391,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357720,"kb_used_data":1023888,"kb_used_omap":1,"kb_used_meta":333822,"kb_avail":92387432,"statfs":{"total":95995035648,"available":94604730368,"internally_reserved":0,"allocated":1048461312,"data_stored":1046823291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":341834187},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496022,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":214932,"kb_used_data":17292,"kb_used_omap":1,"kb_used_meta":197630,"kb_avail":93530220,"statfs":{"total":95995035648,"available":95774945280,"internally_reserved":0,"allocated":17707008,"data_stored":16072787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":202373575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426609,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":710152,"kb_used_data":636992,"kb_used_omap":8,"kb_used_meta":73143,"kb_avail":93035000,"statfs":{"total":95995035648,"available":95267840000,"internally_reserved":0,"allocated":652279808,"data_stored":650653053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":74899338},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019301,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":966584,"kb_used_data":642608,"kb_used_omap":1,"kb_used_meta":323966,"kb_avail":92778568,"statfs":{"total":95995035648,"available":95005253632,"internally_reserved":0,"allocated":658030592,"data_stored":656419620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":331741638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435486,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1594372,"kb_used_data":1265660,"kb_used_omap":4,"kb_used_meta":328699,"kb_avail":92150780,"statfs":{"total":95995035648,"available":94362398720,"internally_reserved":0,"allocated":1296035840,"data_stored":1294413025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":336588041},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903338,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1612808,"kb_used_data":1463040,"kb_used_omap":10,"kb_used_meta":149749,"kb_avail":92132344,"statfs":{"total":95995035648,"available":94343520256,"internally_reserved":0,"allocated":1498152960,"data_stored":1496530081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":153343504},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762539,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1339332,"kb_used_data":1064700,"kb_used_omap":7,"kb_used_meta":274616,"kb_avail":92405820,"statfs":{"total":95995035648,"available":94623559680,"internally_reserved":0,"allocated":1090252800,"data_stored":1088628745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":281207138},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926065,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":854860,"kb_used_data":526084,"kb_used_omap":7,"kb_used_meta":328760,"kb_avail":92890292,"statfs":{"total":95995035648,"available":95119659008,"internally_reserved":0,"allocated":538710016,"data_stored":537078740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":336650383},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":529993728,"data_stored":529989656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1078726656,"data_stored":1078722584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1494417408,"data_stored":1494417408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1282801664,"data_stored":1282801664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":650117120,"data_stored":650117120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":644747264,"data_stored":644743192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":13434880,"data_stored":13434880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1038417920,"data_stored":1038417920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:27:20.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:27:20.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:20.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:20.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:21.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:21.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:21.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:21.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:21.197 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:21.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:21.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:21.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:21.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:21.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.454 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:21.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:21.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:21.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:21.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:21.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:21.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:21.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:21.826 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:27:21.826 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:27:21.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:21.833 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:21.841 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:27:21.841 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:27:21.860 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:27:21.860 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:27:21.886 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:27:21.886 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:27:21.904 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:27:21.904 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:27:21.922 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:27:21.922 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:27:21.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:21.943 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:27:21.943 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:27:21.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:21.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:21.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:21.963 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:27:21.963 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:27:22.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:22.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:22.101 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:27:22.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:22.206 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:22.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[98970]: pgmap v969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 249 op/s; 24613/93546 objects degraded (26.311%); 7144/93546 objects misplaced (7.637%); 4.4 MiB/s, 71 objects/s recovering 2024-08-29T15:27:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2721114780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3757115604' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:27:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:27:22.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 24613/93546 objects degraded (26.311%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:22.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[93921]: pgmap v969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 249 op/s; 24613/93546 objects degraded (26.311%); 7144/93546 objects misplaced (7.637%); 4.4 MiB/s, 71 objects/s recovering 2024-08-29T15:27:22.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2721114780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3757115604' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:27:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:27:22.232 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 24613/93546 objects degraded (26.311%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:22.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:22.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:22.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:22.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:22.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:22 smithi171 ceph-mon[90031]: pgmap v969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 249 op/s; 24613/93546 objects degraded (26.311%); 7144/93546 objects misplaced (7.637%); 4.4 MiB/s, 71 objects/s recovering 2024-08-29T15:27:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2721114780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3757115604' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:27:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:27:22.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 24613/93546 objects degraded (26.311%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:22.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:22.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:22.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:22.706 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:22.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:22.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:22.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:22.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:22.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:23.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:23.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:23.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:23.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:23.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:23.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:23 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:23 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:27:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:23 smithi027 ceph-mon[93921]: osdmap e281: 8 total, 8 up, 8 in 2024-08-29T15:27:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:23 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:27:23.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:23 smithi027 ceph-mon[98970]: osdmap e281: 8 total, 8 up, 8 in 2024-08-29T15:27:23.337 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:23.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:23.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:23.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:23 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:27:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:23 smithi171 ceph-mon[90031]: osdmap e281: 8 total, 8 up, 8 in 2024-08-29T15:27:23.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:23.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:23.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:23.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:23.712 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:23.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:23.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:23.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:23.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:23.921 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:23.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:23.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:23.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:23.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:24.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:24.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:24.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:24.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:24.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:24 smithi027 ceph-mon[98970]: pgmap v971: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 24356/90837 objects degraded (26.813%); 7057/90837 objects misplaced (7.769%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:24 smithi027 ceph-mon[93921]: pgmap v971: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 24356/90837 objects degraded (26.813%); 7057/90837 objects misplaced (7.769%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:24.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:24.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:24.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:24.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:24.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:24 smithi171 ceph-mon[90031]: pgmap v971: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 24356/90837 objects degraded (26.813%); 7057/90837 objects misplaced (7.769%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:24.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:24.604 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:24.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:24.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:24.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:24.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:24.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:24.861 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:24.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:24.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:24.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:24.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:25.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:25.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:25.216 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:25.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:25.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:25.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.368 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:25.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:25.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:25.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:25.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:25.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:25.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:27:25.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:25.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:25.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:25.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:25.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:25.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:25.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:26.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:26.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:26.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:26.224 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:26.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:26.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:26.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:26.376 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:26.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:26.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:26.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:27:26.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:26.498 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:26 smithi027 ceph-mon[93921]: pgmap v972: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 24356/90315 objects degraded (26.968%); 7057/90315 objects misplaced (7.814%); 3.7 MiB/s, 59 objects/s recovering 2024-08-29T15:27:26.498 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:26 smithi027 ceph-mon[98970]: pgmap v972: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 24356/90315 objects degraded (26.968%); 7057/90315 objects misplaced (7.814%); 3.7 MiB/s, 59 objects/s recovering 2024-08-29T15:27:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:26 smithi171 ceph-mon[90031]: pgmap v972: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 24356/90315 objects degraded (26.968%); 7057/90315 objects misplaced (7.814%); 3.7 MiB/s, 59 objects/s recovering 2024-08-29T15:27:26.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:26.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:26.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:26.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:26.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:26.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:26.787 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:26.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:26.889 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:26.911 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:26.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:26.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:27.013 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:27.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:27.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:27.137 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:27.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:27.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:27.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:27.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:27.310 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 24356/90315 objects degraded (26.968%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:27.310 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 24356/90315 objects degraded (26.968%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:27.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:222579 6.6 6:6d52f7d0:::benchmark_data_smithi027_169445_object97152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:27:27.350158+0000", 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.007363667, 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073899680000000002, 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:27:27.417 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 222579 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.350158+0000", 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.350158+0000", 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.350159+0000", 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4600000000000003e-07 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.350163+0000", 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0679999999999998e-06 2024-08-29T15:27:27.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.350173+0000", 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7170000000000003e-06 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.350180+0000", 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0500000000000003e-06 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:27:27.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.353444+0000", 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032638540000000001 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.353567+0000", 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012296 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.353690+0000", 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012293100000000001 2024-08-29T15:27:27.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.356683+0000", 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002993239 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.357395+0000", 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00071216300000000003 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:27:27.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:222584 6.6 6.94d4d586 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:27:27.357595+0000", 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.9999266, 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5856000000000002e-05, 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 222584 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:27:27.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.357595+0000", 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.357595+0000", 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.357597+0000", 2024-08-29T15:27:27.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3170000000000001e-06 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.357602+0000", 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1610000000000002e-06 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.357617+0000", 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5404e-05 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:27:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:27:27.357625+0000", 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.199e-06 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 2 2024-08-29T15:27:27.426 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:27:27.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 24356/90315 objects degraded (26.968%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:27.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:27.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:27.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:27.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:27.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:27.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:27.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:27.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:27.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:27.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:28.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:28.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:28.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:28.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:28.282 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:28.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:28 smithi027 ceph-mon[98970]: pgmap v973: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 275 op/s; 23896/86526 objects degraded (27.617%); 6867/86526 objects misplaced (7.936%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:27:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:28 smithi027 ceph-mon[93921]: pgmap v973: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 275 op/s; 23896/86526 objects degraded (27.617%); 6867/86526 objects misplaced (7.936%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:27:28.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:28.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:28 smithi171 ceph-mon[90031]: pgmap v973: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 275 op/s; 23896/86526 objects degraded (27.617%); 6867/86526 objects misplaced (7.936%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:27:28.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:28.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:28.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:28.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:28.685 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:28.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:27:28.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:28.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:28.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:28.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:28.942 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:28.964 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:28.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:29.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:29.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:29.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:29.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:29.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:29.223 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:27:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:27:29] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-08-29T15:27:29.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:29.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:29.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:29.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:29.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:29.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:29.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:29.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:29.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:29.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:29.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:29.845 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:29 smithi027 ceph-mon[98970]: pgmap v974: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 275 op/s; 23896/86526 objects degraded (27.617%); 6867/86526 objects misplaced (7.936%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:27:29.845 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:29.846 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:29.846 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:29 smithi027 ceph-mon[93921]: pgmap v974: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 275 op/s; 23896/86526 objects degraded (27.617%); 6867/86526 objects misplaced (7.936%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:27:29.846 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:29.846 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:29.859 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:29.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:29.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:29.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:29.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:29 smithi171 ceph-mon[90031]: pgmap v974: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 275 op/s; 23896/86526 objects degraded (27.617%); 6867/86526 objects misplaced (7.936%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:27:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:30.093 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:30.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:30.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:30.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:30.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:30.255 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:30.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:30.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:30.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:30.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:30.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:30.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:30.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:30.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:30.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:30.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:30.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:30.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:30.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:30.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:30.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:30.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:30.862 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:30.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:30.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:30.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:30.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:31.016 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:31.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:31.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:31.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:31.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:31.267 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:31.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:31.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:31.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:31.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:31.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:31.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:31.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:31.718 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:27:31.718 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23719/85179 objects degraded (27.846%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1433,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":281,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":28393,"data_bytes":1863653373,"bytes_used":8325111808,"bytes_avail":759635173376,"bytes_total":767960285184,"degraded_objects":23719,"degraded_total":85179,"degraded_ratio":0.27846065344744597,"misplaced_objects":6769,"misplaced_total":85179,"misplaced_ratio":0.079467943976801791,"recovering_objects_per_sec":70,"recovering_bytes_per_sec":4605842,"recovering_keys_per_sec":0,"num_objects_recovered":703,"num_bytes_recovered":46071808,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":278},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:27:31.766 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.794 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:31.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:31.895 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:31.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:31.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:31.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:31 smithi171 ceph-mon[90031]: pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 23719/85179 objects degraded (27.846%); 6769/85179 objects misplaced (7.947%); 4.4 MiB/s, 70 objects/s recovering 2024-08-29T15:27:32.023 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:32.051 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:31 smithi027 ceph-mon[98970]: pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 23719/85179 objects degraded (27.846%); 6769/85179 objects misplaced (7.947%); 4.4 MiB/s, 70 objects/s recovering 2024-08-29T15:27:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:31 smithi027 ceph-mon[93921]: pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 23719/85179 objects degraded (27.846%); 6769/85179 objects misplaced (7.947%); 4.4 MiB/s, 70 objects/s recovering 2024-08-29T15:27:32.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:32.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:32.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:32.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:32.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:32.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:32.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:32.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:32.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:32.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:32.643 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:27:32.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:27:32.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:32.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:32.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:32.795 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:27:32.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:32.849 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:32.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:32.951 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:32.974 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:32.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:32.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/716604030' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:27:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 23719/85179 objects degraded (27.846%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3026990261' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:33.023 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/716604030' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:27:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 23719/85179 objects degraded (27.846%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3026990261' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/716604030' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:27:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 23719/85179 objects degraded (27.846%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:33.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3026990261' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:33.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:33.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:33.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:33.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:33.258 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:33.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:33.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:33.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:33.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:33.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:33.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:33.510 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:33.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:33.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:33.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:33.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:33.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:33.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:33.775 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:33.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:33.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:33.877 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:33.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:33.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:34.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:33 smithi171 ceph-mon[90031]: pgmap v976: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 23451/82146 objects degraded (28.548%); 6685/82146 objects misplaced (8.138%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:33 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3026990261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:33 smithi171 ceph-mon[90031]: osdmap e282: 8 total, 8 up, 8 in 2024-08-29T15:27:34.028 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:34.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:34.065 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[93921]: pgmap v976: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 23451/82146 objects degraded (28.548%); 6685/82146 objects misplaced (8.138%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:34.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:34.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3026990261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:34.066 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[93921]: osdmap e282: 8 total, 8 up, 8 in 2024-08-29T15:27:34.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[98970]: pgmap v976: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 23451/82146 objects degraded (28.548%); 6685/82146 objects misplaced (8.138%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:27:34.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:34.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3026990261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:34.066 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:33 smithi027 ceph-mon[98970]: osdmap e282: 8 total, 8 up, 8 in 2024-08-29T15:27:34.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:34.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:34.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:34.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:34.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:34.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:34.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:34.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:34.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:34.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:34.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:34.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:34.773 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:34.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:34.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:34.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:34.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:35.024 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:35.046 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:35.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:35.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:35.171 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:35.273 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:35.299 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:35.309 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:35.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:35.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:35.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:35.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:35.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:35.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:35.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:35.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:35.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:35.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:35.872 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:35 smithi027 ceph-mon[98970]: pgmap v978: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 295 op/s; 23451/81456 objects degraded (28.790%); 6685/81456 objects misplaced (8.207%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:27:35.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:35 smithi027 ceph-mon[93921]: pgmap v978: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 295 op/s; 23451/81456 objects degraded (28.790%); 6685/81456 objects misplaced (8.207%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:27:35.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:35.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:35.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:35.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:35 smithi171 ceph-mon[90031]: pgmap v978: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 295 op/s; 23451/81456 objects degraded (28.790%); 6685/81456 objects misplaced (8.207%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:27:36.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:36.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:36.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:36.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:36.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:36.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:36.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:36.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:27:36.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:36.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:36.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:27:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:27:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:27:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:27:36.497 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:27:36.497 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:27:36.498 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:27:36.498 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:27:36.498 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:27:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:27:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:27:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:27:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:27:36.499 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:27:36.499 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:27:36.499 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:27:36.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:36.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:36.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:36.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:36.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1579925985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:36.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:36.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:36.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:36.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:36.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1579925985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:36.977 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:36.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:36.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1579925985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:37.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:37.101 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:37.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:37.203 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:37.225 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:37.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:37.327 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:37.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:37.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:37.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:37.487 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:37.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:37.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:37.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:37.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:37.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:37.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:37.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:37.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:37.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:37.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:37.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:37.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:38.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:37 smithi171 ceph-mon[90031]: pgmap v979: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 22947/77337 objects degraded (29.671%); 6491/77337 objects misplaced (8.393%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:27:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 22947/77337 objects degraded (29.671%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:37 smithi027 ceph-mon[93921]: pgmap v979: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 22947/77337 objects degraded (29.671%); 6491/77337 objects misplaced (8.393%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:27:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 22947/77337 objects degraded (29.671%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:38.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:37 smithi027 ceph-mon[98970]: pgmap v979: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 22947/77337 objects degraded (29.671%); 6491/77337 objects misplaced (8.393%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:27:38.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 22947/77337 objects degraded (29.671%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:38.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:38.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:38.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:38.229 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:38.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:38.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:38.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:38.483 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:38.492 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:27:38.492 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:27:38.507 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:38.608 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:38.636 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:38.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:38.724 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:38.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:38.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:38.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:38.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:38.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:38.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:38.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:39.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:39.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:39.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:39.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:39.171 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:27:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:27:39] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-08-29T15:27:39.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:39.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:39.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:39.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:39.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:39.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:39.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:39.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:39.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:39.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:39.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:39.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:39.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:39.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:40.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:39 smithi171 ceph-mon[90031]: pgmap v980: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 22947/77337 objects degraded (29.671%); 6491/77337 objects misplaced (8.393%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:27:40.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:40.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:39 smithi027 ceph-mon[98970]: pgmap v980: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 22947/77337 objects degraded (29.671%); 6491/77337 objects misplaced (8.393%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:27:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:39 smithi027 ceph-mon[93921]: pgmap v980: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.6 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 22947/77337 objects degraded (29.671%); 6491/77337 objects misplaced (8.393%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:27:40.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:40.171 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:40.273 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:40.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:40.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:40.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:40.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:40.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:40.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:40.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:40.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:40.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:40.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:40.815 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:40.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:40.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:40.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:40.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:41.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:41.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:41.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:41.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:41.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:27:41.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:41.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:41.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:27:41.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:41.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:41.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:41.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:41.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:41.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:41.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:41.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:41.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:41.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:41.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:41.964 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:41.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:41.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:41 smithi171 ceph-mon[90031]: pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 22735/75759 objects degraded (30.010%); 6387/75759 objects misplaced (8.431%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:27:42.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:41 smithi027 ceph-mon[93921]: pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 22735/75759 objects degraded (30.010%); 6387/75759 objects misplaced (8.431%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:27:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:41 smithi027 ceph-mon[98970]: pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 313 op/s; 22735/75759 objects degraded (30.010%); 6387/75759 objects misplaced (8.431%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:27:42.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:42.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:42.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:42.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:42.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:42.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:42.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:42.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:42.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:42.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:42.441 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:27:42.442 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:27:42.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:42.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:42.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:42.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:42.624 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:42.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:42.721 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:42.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:42.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 22735/75759 objects degraded (30.010%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:42.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:42 smithi027 ceph-mon[93921]: from='client.55819 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:42.743 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:27:42.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:42.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:42.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:42.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:42.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:42.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:43.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 22735/75759 objects degraded (30.010%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:42 smithi171 ceph-mon[90031]: from='client.55819 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2759510077' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:27:43.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 22735/75759 objects degraded (30.010%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:42 smithi027 ceph-mon[98970]: from='client.55819 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:43.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2759510077' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:27:43.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2759510077' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:27:43.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:43.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:43.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:43.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:43.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:43.330 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":981,"stamp":"2024-08-29T15:27:40.671135+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1657870333,"num_objects":25253,"num_object_clones":0,"num_object_copies":75759,"num_objects_missing_on_primary":2507,"num_objects_missing":2507,"num_objects_degraded":22735,"num_objects_misplaced":6387,"num_objects_unfound":0,"num_objects_dirty":25253,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":226748,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23598,"num_bytes_recovered":1425345741,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125584,"ondisk_log_size":125584,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7691148,"kb_used_data":5612872,"kb_used_omap":43,"kb_used_meta":2078164,"kb_avail":742270068,"statfs":{"total":767960285184,"available":760084549632,"internally_reserved":0,"allocated":5747580928,"data_stored":5734447566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2128040428},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,1,3,1,0,1,9,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-205783040,"num_objects":-3140,"num_object_clones":0,"num_object_copies":-9420,"num_objects_missing_on_primary":-205,"num_objects_missing":-205,"num_objects_degraded":-984,"num_objects_misplaced":-382,"num_objects_unfound":0,"num_objects_dirty":-3140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3140,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":795,"num_bytes_recovered":52101120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003024"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":450,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.711605+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:27:32.711605+0000","last_peered":"2024-08-29T15:27:32.711605+0000","last_clean":"2024-08-29T15:27:32.711605+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:27:32.711605+0000","last_undegraded":"2024-08-29T15:27:32.711605+0000","last_fullsized":"2024-08-29T15:27:32.711605+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":61,"reported_epoch":281,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:22.101840+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:27:22.101840+0000","last_peered":"2024-08-29T15:27:22.101840+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:27:22.101840+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:27:22.101840+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":151,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.098187+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:27:22.098187+0000","last_peered":"2024-08-29T15:27:22.098187+0000","last_clean":"2024-08-29T15:27:22.098187+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:27:22.098187+0000","last_undegraded":"2024-08-29T15:27:22.098187+0000","last_fullsized":"2024-08-29T15:27:22.098187+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":416,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.726954+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:27:32.726954+0000","last_peered":"2024-08-29T15:27:32.726954+0000","last_clean":"2024-08-29T15:27:32.726954+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:27:32.726954+0000","last_undegraded":"2024-08-29T15:27:32.726954+0000","last_fullsized":"2024-08-29T15:27:32.726954+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":416,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727489+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:27:32.727489+0000","last_peered":"2024-08-29T15:27:32.727489+0000","last_clean":"2024-08-29T15:27:32.727489+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:27:32.727489+0000","last_undegraded":"2024-08-29T15:27:32.727489+0000","last_fullsized":"2024-08-29T15:27:32.727489+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101442+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:27:22.101442+0000","last_peered":"2024-08-29T15:27:22.101442+0000","last_clean":"2024-08-29T15:27:22.101442+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:27:22.101442+0000","last_undegraded":"2024-08-29T15:27:22.101442+0000","last_fullsized":"2024-08-29T15:27:22.101442+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":227,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.715163+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:27:32.715163+0000","last_peered":"2024-08-29T15:27:32.715163+0000","last_clean":"2024-08-29T15:27:32.715163+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:27:32.715163+0000","last_undegraded":"2024-08-29T15:27:32.715163+0000","last_fullsized":"2024-08-29T15:27:32.715163+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":152,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099767+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:27:22.099767+0000","last_peered":"2024-08-29T15:27:22.099767+0000","last_clean":"2024-08-29T15:27:22.099767+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:27:22.099767+0000","last_undegraded":"2024-08-29T15:27:22.099767+0000","last_fullsized":"2024-08-29T15:27:22.099767+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"282'14234","reported_seq":22296,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:36.188737+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:27:36.188737+0000","last_peered":"2024-08-29T15:27:36.188737+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:27:36.188737+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"179'4200","ondisk_log_start":"179'4200","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10034,"log_dups_size":0,"ondisk_log_size":10034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14234,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,5","objects":-1247}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":60,"reported_epoch":281,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:22.102098+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:27:22.102098+0000","last_peered":"2024-08-29T15:27:22.102098+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:27:22.102098+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:27:22.102098+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":169,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.097815+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:27:22.097815+0000","last_peered":"2024-08-29T15:27:22.097815+0000","last_clean":"2024-08-29T15:27:22.097815+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:27:22.097815+0000","last_undegraded":"2024-08-29T15:27:22.097815+0000","last_fullsized":"2024-08-29T15:27:22.097815+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"282'13909","reported_seq":14153,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:36.387324+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:27:36.387324+0000","last_peered":"2024-08-29T15:27:36.387324+0000","last_clean":"2024-08-29T15:27:36.387324+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:27:36.387324+0000","last_undegraded":"2024-08-29T15:27:36.387324+0000","last_fullsized":"2024-08-29T15:27:36.387324+0000","mapping_epoch":168,"log_start":"188'7228","ondisk_log_start":"188'7228","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107544576,"num_objects":1641,"num_object_clones":0,"num_object_copies":4923,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13909,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":60,"reported_epoch":281,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:22.103638+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:27:22.103638+0000","last_peered":"2024-08-29T15:27:22.103638+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:27:22.103638+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:27:22.103638+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101333+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:27:22.101333+0000","last_peered":"2024-08-29T15:27:22.101333+0000","last_clean":"2024-08-29T15:27:22.101333+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:27:22.101333+0000","last_undegraded":"2024-08-29T15:27:22.101333+0000","last_fullsized":"2024-08-29T15:27:22.101333+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"282'14040","reported_seq":14254,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:36.393601+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:27:36.393601+0000","last_peered":"2024-08-29T15:27:36.393601+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:27:36.393601+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"188'7274","ondisk_log_start":"188'7274","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6766,"log_dups_size":0,"ondisk_log_size":6766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109576192,"num_objects":1672,"num_object_clones":0,"num_object_copies":5016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14040,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-805}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":232,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.731873+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:27:32.731873+0000","last_peered":"2024-08-29T15:27:32.731873+0000","last_clean":"2024-08-29T15:27:32.731873+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:27:32.731873+0000","last_undegraded":"2024-08-29T15:27:32.731873+0000","last_fullsized":"2024-08-29T15:27:32.731873+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":107,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099162+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:27:22.099162+0000","last_peered":"2024-08-29T15:27:22.099162+0000","last_clean":"2024-08-29T15:27:22.099162+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:27:22.099162+0000","last_undegraded":"2024-08-29T15:27:22.099162+0000","last_fullsized":"2024-08-29T15:27:22.099162+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"282'14243","reported_seq":19707,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:38.827394+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:27:38.827394+0000","last_peered":"2024-08-29T15:27:38.827394+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:27:38.827394+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"186'6062","ondisk_log_start":"186'6062","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100597760,"num_objects":1535,"num_object_clones":0,"num_object_copies":4605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14243,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1535}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":169,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099084+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:27:22.099084+0000","last_peered":"2024-08-29T15:27:22.099084+0000","last_clean":"2024-08-29T15:27:22.099084+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:27:22.099084+0000","last_undegraded":"2024-08-29T15:27:22.099084+0000","last_fullsized":"2024-08-29T15:27:22.099084+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":431,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727260+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:27:32.727260+0000","last_peered":"2024-08-29T15:27:32.727260+0000","last_clean":"2024-08-29T15:27:32.727260+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:27:32.727260+0000","last_undegraded":"2024-08-29T15:27:32.727260+0000","last_fullsized":"2024-08-29T15:27:32.727260+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"282'14262","reported_seq":16282,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:39.787419+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:27:39.787419+0000","last_peered":"2024-08-29T15:27:39.787419+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:27:39.787419+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:27:39.787419+0000","mapping_epoch":234,"log_start":"187'6333","ondisk_log_start":"187'6333","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7929,"log_dups_size":0,"ondisk_log_size":7929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101711872,"num_objects":1552,"num_object_clones":0,"num_object_copies":4656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2735,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14262,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":925,"num_bytes_recovered":60620800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":1552}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":425,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710578+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:27:32.710578+0000","last_peered":"2024-08-29T15:27:32.710578+0000","last_clean":"2024-08-29T15:27:32.710578+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:27:32.710578+0000","last_undegraded":"2024-08-29T15:27:32.710578+0000","last_fullsized":"2024-08-29T15:27:32.710578+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101773+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:27:22.101773+0000","last_peered":"2024-08-29T15:27:22.101773+0000","last_clean":"2024-08-29T15:27:22.101773+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:27:22.101773+0000","last_undegraded":"2024-08-29T15:27:22.101773+0000","last_fullsized":"2024-08-29T15:27:22.101773+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"282'14198","reported_seq":16100,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:36.395974+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:27:36.395974+0000","last_peered":"2024-08-29T15:27:36.395974+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:27:36.395974+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:27:36.395974+0000","mapping_epoch":234,"log_start":"188'7468","ondisk_log_start":"188'7468","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107741184,"num_objects":1644,"num_object_clones":0,"num_object_copies":4932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14198,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":821,"num_bytes_recovered":53805056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":1644}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":125,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.098656+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:27:22.098656+0000","last_peered":"2024-08-29T15:27:22.098656+0000","last_clean":"2024-08-29T15:27:22.098656+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:27:22.098656+0000","last_undegraded":"2024-08-29T15:27:22.098656+0000","last_fullsized":"2024-08-29T15:27:22.098656+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.102103+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:27:22.102103+0000","last_peered":"2024-08-29T15:27:22.102103+0000","last_clean":"2024-08-29T15:27:22.102103+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:27:22.102103+0000","last_undegraded":"2024-08-29T15:27:22.102103+0000","last_fullsized":"2024-08-29T15:27:22.102103+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"282'14067","reported_seq":14512,"reported_epoch":282,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:27:36.410478+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:27:36.410478+0000","last_peered":"2024-08-29T15:27:36.410478+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:27:36.410478+0000","last_undegraded":"2024-08-29T15:27:36.410478+0000","last_fullsized":"2024-08-29T15:27:36.410478+0000","mapping_epoch":202,"log_start":"188'7367","ondisk_log_start":"188'7367","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104267776,"num_objects":1591,"num_object_clones":0,"num_object_copies":4773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1591,"num_objects_unfound":0,"num_objects_dirty":1591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14067,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1591}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101369+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:27:22.101369+0000","last_peered":"2024-08-29T15:27:22.101369+0000","last_clean":"2024-08-29T15:27:22.101369+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:27:22.101369+0000","last_undegraded":"2024-08-29T15:27:22.101369+0000","last_fullsized":"2024-08-29T15:27:22.101369+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":268,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.103727+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:27:22.103727+0000","last_peered":"2024-08-29T15:27:22.103727+0000","last_clean":"2024-08-29T15:27:22.103727+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:27:22.103727+0000","last_undegraded":"2024-08-29T15:27:22.103727+0000","last_fullsized":"2024-08-29T15:27:22.103727+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"282'14148","reported_seq":14312,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:36.134017+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:27:36.134017+0000","last_peered":"2024-08-29T15:27:36.134017+0000","last_clean":"2024-08-29T15:27:36.134017+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:27:36.134017+0000","last_undegraded":"2024-08-29T15:27:36.134017+0000","last_fullsized":"2024-08-29T15:27:36.134017+0000","mapping_epoch":218,"log_start":"179'4078","ondisk_log_start":"179'4078","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10070,"log_dups_size":0,"ondisk_log_size":10070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105578520,"num_objects":1612,"num_object_clones":0,"num_object_copies":4836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1612,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14148,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":428,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710808+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:27:32.710808+0000","last_peered":"2024-08-29T15:27:32.710808+0000","last_clean":"2024-08-29T15:27:32.710808+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:27:32.710808+0000","last_undegraded":"2024-08-29T15:27:32.710808+0000","last_fullsized":"2024-08-29T15:27:32.710808+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":420,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.715246+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:27:32.715246+0000","last_peered":"2024-08-29T15:27:32.715246+0000","last_clean":"2024-08-29T15:27:32.715246+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:27:32.715246+0000","last_undegraded":"2024-08-29T15:27:32.715246+0000","last_fullsized":"2024-08-29T15:27:32.715246+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"282'13615","reported_seq":13827,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:36.143878+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:27:36.143878+0000","last_peered":"2024-08-29T15:27:36.143878+0000","last_clean":"2024-08-29T15:27:36.143878+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:27:36.143878+0000","last_undegraded":"2024-08-29T15:27:36.143878+0000","last_fullsized":"2024-08-29T15:27:36.143878+0000","mapping_epoch":201,"log_start":"179'3600","ondisk_log_start":"179'3600","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10015,"log_dups_size":0,"ondisk_log_size":10015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104398848,"num_objects":1593,"num_object_clones":0,"num_object_copies":4779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13615,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":519,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.730297+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:27:32.730297+0000","last_peered":"2024-08-29T15:27:32.730297+0000","last_clean":"2024-08-29T15:27:32.730297+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:27:32.730297+0000","last_undegraded":"2024-08-29T15:27:32.730297+0000","last_fullsized":"2024-08-29T15:27:32.730297+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":753,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.098221+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:27:22.098221+0000","last_peered":"2024-08-29T15:27:22.098221+0000","last_clean":"2024-08-29T15:27:22.098221+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:27:22.098221+0000","last_undegraded":"2024-08-29T15:27:22.098221+0000","last_fullsized":"2024-08-29T15:27:22.098221+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":261,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.098451+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:27:22.098451+0000","last_peered":"2024-08-29T15:27:22.098451+0000","last_clean":"2024-08-29T15:27:22.098451+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:27:22.098451+0000","last_undegraded":"2024-08-29T15:27:22.098451+0000","last_fullsized":"2024-08-29T15:27:22.098451+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":124,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709224+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:27:32.709224+0000","last_peered":"2024-08-29T15:27:32.709224+0000","last_clean":"2024-08-29T15:27:32.709224+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:27:32.709224+0000","last_undegraded":"2024-08-29T15:27:32.709224+0000","last_fullsized":"2024-08-29T15:27:32.709224+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"282'14104","reported_seq":14546,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:35.710238+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:27:35.710238+0000","last_peered":"2024-08-29T15:27:35.710238+0000","last_clean":"2024-08-29T15:27:35.710238+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:27:35.710238+0000","last_undegraded":"2024-08-29T15:27:35.710238+0000","last_fullsized":"2024-08-29T15:27:35.710238+0000","mapping_epoch":201,"log_start":"187'7053","ondisk_log_start":"187'7053","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103415808,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14104,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":893,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.714530+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:27:32.714530+0000","last_peered":"2024-08-29T15:27:32.714530+0000","last_clean":"2024-08-29T15:27:32.714530+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:27:32.714530+0000","last_undegraded":"2024-08-29T15:27:32.714530+0000","last_fullsized":"2024-08-29T15:27:32.714530+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":431,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.726788+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:27:32.726788+0000","last_peered":"2024-08-29T15:27:32.726788+0000","last_clean":"2024-08-29T15:27:32.726788+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:27:32.726788+0000","last_undegraded":"2024-08-29T15:27:32.726788+0000","last_fullsized":"2024-08-29T15:27:32.726788+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":439,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727710+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:27:32.727710+0000","last_peered":"2024-08-29T15:27:32.727710+0000","last_clean":"2024-08-29T15:27:32.727710+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:27:32.727710+0000","last_undegraded":"2024-08-29T15:27:32.727710+0000","last_fullsized":"2024-08-29T15:27:32.727710+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"282'14093","reported_seq":25741,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:36.398401+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:27:36.398401+0000","last_peered":"2024-08-29T15:27:36.398401+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:27:36.398401+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"188'7330","ondisk_log_start":"188'7330","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6763,"log_dups_size":0,"ondisk_log_size":6763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104660992,"num_objects":1597,"num_object_clones":0,"num_object_copies":4791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14093,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5717,"num_bytes_recovered":374669312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"3,6","objects":54},{"shards":"3,7","objects":1543}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":702,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710539+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:27:32.710539+0000","last_peered":"2024-08-29T15:27:32.710539+0000","last_clean":"2024-08-29T15:27:32.710539+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:27:32.710539+0000","last_undegraded":"2024-08-29T15:27:32.710539+0000","last_fullsized":"2024-08-29T15:27:32.710539+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":114,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099652+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:27:22.099652+0000","last_peered":"2024-08-29T15:27:22.099652+0000","last_clean":"2024-08-29T15:27:22.099652+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:27:22.099652+0000","last_undegraded":"2024-08-29T15:27:22.099652+0000","last_fullsized":"2024-08-29T15:27:22.099652+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":466,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.715037+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:27:32.715037+0000","last_peered":"2024-08-29T15:27:32.715037+0000","last_clean":"2024-08-29T15:27:32.715037+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:27:32.715037+0000","last_undegraded":"2024-08-29T15:27:32.715037+0000","last_fullsized":"2024-08-29T15:27:32.715037+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"282'14141","reported_seq":14395,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:36.428686+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:27:36.428686+0000","last_peered":"2024-08-29T15:27:36.428686+0000","last_clean":"2024-08-29T15:27:36.428686+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:27:36.428686+0000","last_undegraded":"2024-08-29T15:27:36.428686+0000","last_fullsized":"2024-08-29T15:27:36.428686+0000","mapping_epoch":234,"log_start":"188'7397","ondisk_log_start":"188'7397","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102957056,"num_objects":1571,"num_object_clones":0,"num_object_copies":4713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14141,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":221,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.098366+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:27:22.098366+0000","last_peered":"2024-08-29T15:27:22.098366+0000","last_clean":"2024-08-29T15:27:22.098366+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:27:22.098366+0000","last_undegraded":"2024-08-29T15:27:22.098366+0000","last_fullsized":"2024-08-29T15:27:22.098366+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101672+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:27:22.101672+0000","last_peered":"2024-08-29T15:27:22.101672+0000","last_clean":"2024-08-29T15:27:22.101672+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:27:22.101672+0000","last_undegraded":"2024-08-29T15:27:22.101672+0000","last_fullsized":"2024-08-29T15:27:22.101672+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":128,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.098591+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:27:22.098591+0000","last_peered":"2024-08-29T15:27:22.098591+0000","last_clean":"2024-08-29T15:27:22.098591+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:27:22.098591+0000","last_undegraded":"2024-08-29T15:27:22.098591+0000","last_fullsized":"2024-08-29T15:27:22.098591+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"282'14305","reported_seq":14558,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:39.006275+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:27:39.006275+0000","last_peered":"2024-08-29T15:27:39.006275+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:27:39.006275+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"186'6142","ondisk_log_start":"186'6142","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8163,"log_dups_size":0,"ondisk_log_size":8163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99680256,"num_objects":1521,"num_object_clones":0,"num_object_copies":4563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14305,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-902}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":153,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.098117+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:27:22.098117+0000","last_peered":"2024-08-29T15:27:22.098117+0000","last_clean":"2024-08-29T15:27:22.098117+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:27:22.098117+0000","last_undegraded":"2024-08-29T15:27:22.098117+0000","last_fullsized":"2024-08-29T15:27:22.098117+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":858,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.715962+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:27:32.715962+0000","last_peered":"2024-08-29T15:27:32.715962+0000","last_clean":"2024-08-29T15:27:32.715962+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:27:32.715962+0000","last_undegraded":"2024-08-29T15:27:32.715962+0000","last_fullsized":"2024-08-29T15:27:32.715962+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":435,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.714792+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:27:32.714792+0000","last_peered":"2024-08-29T15:27:32.714792+0000","last_clean":"2024-08-29T15:27:32.714792+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:27:32.714792+0000","last_undegraded":"2024-08-29T15:27:32.714792+0000","last_fullsized":"2024-08-29T15:27:32.714792+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"282'14132","reported_seq":14940,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:35.701596+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:27:35.701596+0000","last_peered":"2024-08-29T15:27:35.701596+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:27:35.701596+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"187'7080","ondisk_log_start":"187'7080","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7052,"log_dups_size":0,"ondisk_log_size":7052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100663296,"num_objects":1536,"num_object_clones":0,"num_object_copies":4608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1526,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":1536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14132,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":9},{"shards":"1,7","objects":-615}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":440,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.715872+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:27:32.715872+0000","last_peered":"2024-08-29T15:27:32.715872+0000","last_clean":"2024-08-29T15:27:32.715872+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:27:32.715872+0000","last_undegraded":"2024-08-29T15:27:32.715872+0000","last_fullsized":"2024-08-29T15:27:32.715872+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":710,"reported_epoch":281,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:22.117581+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:27:22.117581+0000","last_peered":"2024-08-29T15:27:22.117581+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:27:22.117581+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:27:22.117581+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":431,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727152+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:27:32.727152+0000","last_peered":"2024-08-29T15:27:32.727152+0000","last_clean":"2024-08-29T15:27:32.727152+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:27:32.727152+0000","last_undegraded":"2024-08-29T15:27:32.727152+0000","last_fullsized":"2024-08-29T15:27:32.727152+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"282'13986","reported_seq":21154,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:39.713804+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:27:39.713804+0000","last_peered":"2024-08-29T15:27:39.713804+0000","last_clean":"2024-08-29T15:27:39.713804+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:27:39.713804+0000","last_undegraded":"2024-08-29T15:27:39.713804+0000","last_fullsized":"2024-08-29T15:27:39.713804+0000","mapping_epoch":234,"log_start":"186'5999","ondisk_log_start":"186'5999","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7987,"log_dups_size":0,"ondisk_log_size":7987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94502912,"num_objects":1442,"num_object_clones":0,"num_object_copies":4326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13986,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":425,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.724922+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:27:32.724922+0000","last_peered":"2024-08-29T15:27:32.724922+0000","last_clean":"2024-08-29T15:27:32.724922+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:27:32.724922+0000","last_undegraded":"2024-08-29T15:27:32.724922+0000","last_fullsized":"2024-08-29T15:27:32.724922+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1048,"reported_epoch":282,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:32.717281+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:27:32.717281+0000","last_peered":"2024-08-29T15:27:32.717281+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:27:32.717281+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":423,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710469+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:27:32.710469+0000","last_peered":"2024-08-29T15:27:32.710469+0000","last_clean":"2024-08-29T15:27:32.710469+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:27:32.710469+0000","last_undegraded":"2024-08-29T15:27:32.710469+0000","last_fullsized":"2024-08-29T15:27:32.710469+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"282'14148","reported_seq":25688,"reported_epoch":282,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:39.762110+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:27:39.762110+0000","last_peered":"2024-08-29T15:27:39.762110+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:27:39.762110+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"187'6612","ondisk_log_start":"187'6612","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7536,"log_dups_size":0,"ondisk_log_size":7536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95813632,"num_objects":1462,"num_object_clones":0,"num_object_copies":4386,"num_objects_missing_on_primary":2499,"num_objects_missing":2499,"num_objects_degraded":1392,"num_objects_misplaced":1461,"num_objects_unfound":0,"num_objects_dirty":1462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14148,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5225,"num_bytes_recovered":342294528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":69},{"shards":"1,7","objects":1392},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":421,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727400+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:27:32.727400+0000","last_peered":"2024-08-29T15:27:32.727400+0000","last_clean":"2024-08-29T15:27:32.727400+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:27:32.727400+0000","last_undegraded":"2024-08-29T15:27:32.727400+0000","last_fullsized":"2024-08-29T15:27:32.727400+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":134,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099063+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:27:22.099063+0000","last_peered":"2024-08-29T15:27:22.099063+0000","last_clean":"2024-08-29T15:27:22.099063+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:27:22.099063+0000","last_undegraded":"2024-08-29T15:27:22.099063+0000","last_fullsized":"2024-08-29T15:27:22.099063+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101395+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:27:22.101395+0000","last_peered":"2024-08-29T15:27:22.101395+0000","last_clean":"2024-08-29T15:27:22.101395+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:27:22.101395+0000","last_undegraded":"2024-08-29T15:27:22.101395+0000","last_fullsized":"2024-08-29T15:27:22.101395+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101792+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:27:22.101792+0000","last_peered":"2024-08-29T15:27:22.101792+0000","last_clean":"2024-08-29T15:27:22.101792+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:27:22.101792+0000","last_undegraded":"2024-08-29T15:27:22.101792+0000","last_fullsized":"2024-08-29T15:27:22.101792+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":60,"reported_epoch":281,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:22.101854+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:27:22.101854+0000","last_peered":"2024-08-29T15:27:22.101854+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:27:22.101854+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:27:22.101854+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":429,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.714638+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:27:32.714638+0000","last_peered":"2024-08-29T15:27:32.714638+0000","last_clean":"2024-08-29T15:27:32.714638+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:27:32.714638+0000","last_undegraded":"2024-08-29T15:27:32.714638+0000","last_fullsized":"2024-08-29T15:27:32.714638+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":179,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:32.715921+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:27:32.715921+0000","last_peered":"2024-08-29T15:27:32.715921+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:27:32.715921+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:27:32.715921+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099381+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:27:22.099381+0000","last_peered":"2024-08-29T15:27:22.099381+0000","last_clean":"2024-08-29T15:27:22.099381+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:27:22.099381+0000","last_undegraded":"2024-08-29T15:27:22.099381+0000","last_fullsized":"2024-08-29T15:27:22.099381+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":389,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710330+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:27:32.710330+0000","last_peered":"2024-08-29T15:27:32.710330+0000","last_clean":"2024-08-29T15:27:32.710330+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:27:32.710330+0000","last_undegraded":"2024-08-29T15:27:32.710330+0000","last_fullsized":"2024-08-29T15:27:32.710330+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":442,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727906+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:27:32.727906+0000","last_peered":"2024-08-29T15:27:32.727906+0000","last_clean":"2024-08-29T15:27:32.727906+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:27:32.727906+0000","last_undegraded":"2024-08-29T15:27:32.727906+0000","last_fullsized":"2024-08-29T15:27:32.727906+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":441,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710418+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:27:32.710418+0000","last_peered":"2024-08-29T15:27:32.710418+0000","last_clean":"2024-08-29T15:27:32.710418+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:27:32.710418+0000","last_undegraded":"2024-08-29T15:27:32.710418+0000","last_fullsized":"2024-08-29T15:27:32.710418+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099040+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:27:22.099040+0000","last_peered":"2024-08-29T15:27:22.099040+0000","last_clean":"2024-08-29T15:27:22.099040+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:27:22.099040+0000","last_undegraded":"2024-08-29T15:27:22.099040+0000","last_fullsized":"2024-08-29T15:27:22.099040+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":297,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.730215+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:27:32.730215+0000","last_peered":"2024-08-29T15:27:32.730215+0000","last_clean":"2024-08-29T15:27:32.730215+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:27:32.730215+0000","last_undegraded":"2024-08-29T15:27:32.730215+0000","last_fullsized":"2024-08-29T15:27:32.730215+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":418,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727619+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:27:32.727619+0000","last_peered":"2024-08-29T15:27:32.727619+0000","last_clean":"2024-08-29T15:27:32.727619+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:27:32.727619+0000","last_undegraded":"2024-08-29T15:27:32.727619+0000","last_fullsized":"2024-08-29T15:27:32.727619+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":421,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.719012+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:27:32.719012+0000","last_peered":"2024-08-29T15:27:32.719012+0000","last_clean":"2024-08-29T15:27:32.719012+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:27:32.719012+0000","last_undegraded":"2024-08-29T15:27:32.719012+0000","last_fullsized":"2024-08-29T15:27:32.719012+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":425,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.715737+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:27:32.715737+0000","last_peered":"2024-08-29T15:27:32.715737+0000","last_clean":"2024-08-29T15:27:32.715737+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:27:32.715737+0000","last_undegraded":"2024-08-29T15:27:32.715737+0000","last_fullsized":"2024-08-29T15:27:32.715737+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101555+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:27:22.101555+0000","last_peered":"2024-08-29T15:27:22.101555+0000","last_clean":"2024-08-29T15:27:22.101555+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:27:22.101555+0000","last_undegraded":"2024-08-29T15:27:22.101555+0000","last_fullsized":"2024-08-29T15:27:22.101555+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":445,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.716050+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:27:32.716050+0000","last_peered":"2024-08-29T15:27:32.716050+0000","last_clean":"2024-08-29T15:27:32.716050+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:27:32.716050+0000","last_undegraded":"2024-08-29T15:27:32.716050+0000","last_fullsized":"2024-08-29T15:27:32.716050+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":409,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727057+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:27:32.727057+0000","last_peered":"2024-08-29T15:27:32.727057+0000","last_clean":"2024-08-29T15:27:32.727057+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:27:32.727057+0000","last_undegraded":"2024-08-29T15:27:32.727057+0000","last_fullsized":"2024-08-29T15:27:32.727057+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101673+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:27:22.101673+0000","last_peered":"2024-08-29T15:27:22.101673+0000","last_clean":"2024-08-29T15:27:22.101673+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:27:22.101673+0000","last_undegraded":"2024-08-29T15:27:22.101673+0000","last_fullsized":"2024-08-29T15:27:22.101673+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":174,"reported_epoch":281,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:22.098766+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:27:22.098766+0000","last_peered":"2024-08-29T15:27:22.098766+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:27:22.098766+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:27:22.098766+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101562+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:27:22.101562+0000","last_peered":"2024-08-29T15:27:22.101562+0000","last_clean":"2024-08-29T15:27:22.101562+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:27:22.101562+0000","last_undegraded":"2024-08-29T15:27:22.101562+0000","last_fullsized":"2024-08-29T15:27:22.101562+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":180,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.099336+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:27:22.099336+0000","last_peered":"2024-08-29T15:27:22.099336+0000","last_clean":"2024-08-29T15:27:22.099336+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:27:22.099336+0000","last_undegraded":"2024-08-29T15:27:22.099336+0000","last_fullsized":"2024-08-29T15:27:22.099336+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":456,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710644+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:27:32.710644+0000","last_peered":"2024-08-29T15:27:32.710644+0000","last_clean":"2024-08-29T15:27:32.710644+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:27:32.710644+0000","last_undegraded":"2024-08-29T15:27:32.710644+0000","last_fullsized":"2024-08-29T15:27:32.710644+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":458,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.713086+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:27:32.713086+0000","last_peered":"2024-08-29T15:27:32.713086+0000","last_clean":"2024-08-29T15:27:32.713086+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:27:32.713086+0000","last_undegraded":"2024-08-29T15:27:32.713086+0000","last_fullsized":"2024-08-29T15:27:32.713086+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":62,"reported_epoch":281,"state":"active+clean","last_fresh":"2024-08-29T15:27:22.101218+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:27:22.101218+0000","last_peered":"2024-08-29T15:27:22.101218+0000","last_clean":"2024-08-29T15:27:22.101218+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:27:22.101218+0000","last_undegraded":"2024-08-29T15:27:22.101218+0000","last_fullsized":"2024-08-29T15:27:22.101218+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":465,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.727807+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:27:32.727807+0000","last_peered":"2024-08-29T15:27:32.727807+0000","last_clean":"2024-08-29T15:27:32.727807+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:27:32.727807+0000","last_undegraded":"2024-08-29T15:27:32.727807+0000","last_fullsized":"2024-08-29T15:27:32.727807+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1653080088,"num_objects":25225,"num_object_clones":0,"num_object_copies":75675,"num_objects_missing_on_primary":2499,"num_objects_missing":2499,"num_objects_degraded":22706,"num_objects_misplaced":6387,"num_objects_unfound":0,"num_objects_dirty":25225,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":225625,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23489,"num_bytes_recovered":1419378688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5687422976,"data_stored":5687410760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124402,"ondisk_log_size":124402,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234396,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1137080,"kb_used_data":786864,"kb_used_omap":1,"kb_used_meta":350206,"kb_avail":92608072,"statfs":{"total":95995035648,"available":94830665728,"internally_reserved":0,"allocated":805748736,"data_stored":804095419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":358611403},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496027,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":232564,"kb_used_data":16876,"kb_used_omap":1,"kb_used_meta":215678,"kb_avail":93512588,"statfs":{"total":95995035648,"available":95756890112,"internally_reserved":0,"allocated":17281024,"data_stored":15631507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":220854727},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3,1,0,1,9,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426613,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":602856,"kb_used_data":521504,"kb_used_omap":8,"kb_used_meta":81335,"kb_avail":93142296,"statfs":{"total":95995035648,"available":95377711104,"internally_reserved":0,"allocated":534020096,"data_stored":532378045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":83287946},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019306,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":822424,"kb_used_data":488592,"kb_used_omap":1,"kb_used_meta":333822,"kb_avail":92922728,"statfs":{"total":95995035648,"available":95152873472,"internally_reserved":0,"allocated":500318208,"data_stored":498691940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":341834182},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435490,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1354084,"kb_used_data":1025372,"kb_used_omap":4,"kb_used_meta":328699,"kb_avail":92391068,"statfs":{"total":95995035648,"available":94608453632,"internally_reserved":0,"allocated":1049980928,"data_stored":1048342817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":336588041},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903342,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1582760,"kb_used_data":1414944,"kb_used_omap":10,"kb_used_meta":167797,"kb_avail":92162392,"statfs":{"total":95995035648,"available":94374289408,"internally_reserved":0,"allocated":1448902656,"data_stored":1447264481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":171824656},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762543,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1236712,"kb_used_data":952348,"kb_used_omap":7,"kb_used_meta":284344,"kb_avail":92508440,"statfs":{"total":95995035648,"available":94728642560,"internally_reserved":0,"allocated":975204352,"data_stored":973565001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":291168610},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926070,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":722668,"kb_used_data":406372,"kb_used_omap":7,"kb_used_meta":316280,"kb_avail":93022484,"statfs":{"total":95995035648,"available":95255023616,"internally_reserved":0,"allocated":416124928,"data_stored":414478356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":323870863},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":406654976,"data_stored":406650904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":959844352,"data_stored":959840280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1444675584,"data_stored":1444675584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1046151168,"data_stored":1046151168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":496173056,"data_stored":496173056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":527437824,"data_stored":527433752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":13107200,"data_stored":13107200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":793378816,"data_stored":793378816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:27:43.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:43.408 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:43.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:43.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:43.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:43.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:43.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:43.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:43.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.796 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:43.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:43.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:43.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:43.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:43.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:43 smithi171 ceph-mon[90031]: pgmap v982: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 22386/71952 objects degraded (31.112%); 6274/71952 objects misplaced (8.720%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:27:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:43 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2759510077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:27:44.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:43 smithi171 ceph-mon[90031]: osdmap e283: 8 total, 8 up, 8 in 2024-08-29T15:27:44.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:44.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:44.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:44.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[98970]: pgmap v982: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 22386/71952 objects degraded (31.112%); 6274/71952 objects misplaced (8.720%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:27:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2759510077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:27:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[98970]: osdmap e283: 8 total, 8 up, 8 in 2024-08-29T15:27:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[93921]: pgmap v982: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 339 op/s; 22386/71952 objects degraded (31.112%); 6274/71952 objects misplaced (8.720%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:27:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:27:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2759510077' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:27:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[93921]: osdmap e283: 8 total, 8 up, 8 in 2024-08-29T15:27:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:44.085 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:44.150 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:44.174 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:44.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:44.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:44.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:44.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.431 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:44.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:44.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:44.562 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:44.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:44.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:44.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:44.833 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:44.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:44.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:44.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:45.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:45.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:45.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:45.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:45.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:45.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:45.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:45.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:45.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:45.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:45.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:45.601 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:45.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:45.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:45.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:45.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:45.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:45.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:45.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:45.989 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:45.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:45.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:46.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:45 smithi171 ceph-mon[90031]: pgmap v984: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 22386/71232 objects degraded (31.427%); 6274/71232 objects misplaced (8.808%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:27:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:45 smithi027 ceph-mon[98970]: pgmap v984: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 22386/71232 objects degraded (31.427%); 6274/71232 objects misplaced (8.808%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:27:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:45 smithi027 ceph-mon[93921]: pgmap v984: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 22386/71232 objects degraded (31.427%); 6274/71232 objects misplaced (8.808%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:27:46.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:46.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:27:46.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:46.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:46.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:46.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:46.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:46.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:27:46.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:46.365 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:46.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:46.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:46.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:46.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:46.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:46.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:46.626 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:46.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:46.728 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:46.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:46.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:46.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:46.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:46.890 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:46.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:46.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:46.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:47.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:47.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:47.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:47.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:47.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:47.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:47.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:47.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:47.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:47.520 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:47.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:47.622 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:47.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:47.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:47.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:47.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:47.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:47.901 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:27:47.901 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:27:47.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:47.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:47.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:48.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:47 smithi171 ceph-mon[90031]: pgmap v985: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21821/66321 objects degraded (32.902%); 6031/66321 objects misplaced (9.094%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:27:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 21821/66321 objects degraded (32.902%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:48.054 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:48.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:47 smithi027 ceph-mon[93921]: pgmap v985: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21821/66321 objects degraded (32.902%); 6031/66321 objects misplaced (9.094%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:27:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 21821/66321 objects degraded (32.902%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:48.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:47 smithi027 ceph-mon[98970]: pgmap v985: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21821/66321 objects degraded (32.902%); 6031/66321 objects misplaced (9.094%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:27:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 21821/66321 objects degraded (32.902%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:48.156 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:48.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:48.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:48.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:48.372 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:48.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:48.428 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:27:48.428 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:27:48.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:48.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:48.456 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:27:48.456 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:27:48.480 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:27:48.480 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:27:48.502 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:27:48.502 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:27:48.524 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:27:48.524 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:27:48.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:48.549 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:27:48.549 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:27:48.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:48.575 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:27:48.575 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:27:48.599 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:27:48.599 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:27:48.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:48.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:48.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:48.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3766547287' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:48.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3766547287' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:48.822 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:48.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:48.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:48.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:48.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:48.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:27:49.015 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3766547287' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:49.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:49.083 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:49.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:49.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:49.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:49.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:49.277 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:27:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:27:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:27:49] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-08-29T15:27:49.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:49.347 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:49.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:49.472 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:49.502 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:49.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:49.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:49.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:49.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:49.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:49.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:49.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:49.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:49.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:49.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:49.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:49.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:50.000 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:49 smithi171 ceph-mon[90031]: pgmap v986: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21821/66321 objects degraded (32.902%); 6031/66321 objects misplaced (9.094%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:27:50.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:49 smithi027 ceph-mon[93921]: pgmap v986: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21821/66321 objects degraded (32.902%); 6031/66321 objects misplaced (9.094%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:27:50.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:49 smithi027 ceph-mon[98970]: pgmap v986: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 367 op/s; 21821/66321 objects degraded (32.902%); 6031/66321 objects misplaced (9.094%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:27:50.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:50.180 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:50.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:50.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:50.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:50.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:50.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:50.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:50.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:50.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:50.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:50.572 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:50.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:50.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:50.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:50.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:50.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:50.823 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:50.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:50.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:50.925 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:50.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:50.951 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:50.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:51.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:51.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:51.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:51.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:51.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:51.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:51.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:51.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:51.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:51.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:51.438 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:27:51.439 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:27:51.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.439 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.44912.0:230968 6.7 e283/201 6:ecfd6c42:::benchmark_data_smithi027_169445_object105541:head v 283'13960, mlcod=283'13958)", 2024-08-29T15:27:51.439 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:51.436108+0000", 2024-08-29T15:27:51.440 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00062591899999999995, 2024-08-29T15:27:51.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00065487100000000001, 2024-08-29T15:27:51.440 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:27:51.440 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:27:51.440 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:27:51.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:27:51.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436108+0000", 2024-08-29T15:27:51.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:27:51.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:27:51.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:27:51.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436108+0000", 2024-08-29T15:27:51.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:27:51.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:27:51.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:27:51.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436107+0000", 2024-08-29T15:27:51.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:27:51.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:27:51.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:27:51.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436116+0000", 2024-08-29T15:27:51.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6239999999999997e-06 2024-08-29T15:27:51.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:27:51.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:27:51.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436132+0000", 2024-08-29T15:27:51.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6069000000000001e-05 2024-08-29T15:27:51.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:27:51.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436141+0000", 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2169999999999994e-06 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436215+0000", 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4487999999999996e-05 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:51.436234+0000", 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8623000000000001e-05 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:27:51.445 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:27:51.446 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:27:51.446 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:27:51.446 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:27:51.446 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 1 2024-08-29T15:27:51.446 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:27:51.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:51.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:51.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:51.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:51.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:51.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:51.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:51.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:51.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:51.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:51.841 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:51.871 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:51.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:51.884 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:51 smithi027 ceph-mon[98970]: pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 21576/64476 objects degraded (33.464%); 5920/64476 objects misplaced (9.182%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:27:51.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:51 smithi027 ceph-mon[93921]: pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 21576/64476 objects degraded (33.464%); 5920/64476 objects misplaced (9.182%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:27:51.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:52.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:52.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:52.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:51 smithi171 ceph-mon[90031]: pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 21576/64476 objects degraded (33.464%); 5920/64476 objects misplaced (9.182%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:27:52.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:27:52.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:52.138 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:52.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:27:52.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:52.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:52.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:52.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:52.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:52.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:52.528 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:27:52.530 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:52.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:52.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:52.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:52.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:52.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:52.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:52.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:52.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:52.937 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 21576/64476 objects degraded (33.464%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:52.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:52.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:52.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:53.031 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 21576/64476 objects degraded (33.464%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:53.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 21576/64476 objects degraded (33.464%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:53.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:53.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:53.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:53.204 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:53.224 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:53.306 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:53.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:53.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:53.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:53.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:53.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:53.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:53.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:53.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:53.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:53.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:53.784 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:27:53.786 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:27:53.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:53.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.866 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:53.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:53.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:53.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:53.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:53.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:53 smithi171 ceph-mon[90031]: pgmap v988: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 21207/60525 objects degraded (35.038%); 5816/60525 objects misplaced (9.609%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:27:54.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:53 smithi027 ceph-mon[93921]: pgmap v988: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 21207/60525 objects degraded (35.038%); 5816/60525 objects misplaced (9.609%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:27:54.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:53 smithi027 ceph-mon[98970]: pgmap v988: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 21207/60525 objects degraded (35.038%); 5816/60525 objects misplaced (9.609%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:27:54.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:54.135 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:54.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:54.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:54.236 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:54.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:54.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:54.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:54.399 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:54.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:54.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:54.479 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:27:54.479 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:27:54.479 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:27:54.479 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:27:54.479 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:27:54.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:54.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:54.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:54.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:54.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:54.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:54.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:54.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:54.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:54.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:54 smithi027 ceph-mon[98970]: from='client.55837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:54 smithi027 ceph-mon[93921]: from='client.55837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:54.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:54.914 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:54.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:55.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:55.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:54 smithi171 ceph-mon[90031]: from='client.55837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:27:55.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:55.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:55.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:55.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:55.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:55.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:55.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:55.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:55.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:55.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:55.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:55.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:55.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:55.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:55.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:27:55.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:27:55.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:55.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:27:55.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:55.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:55.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:55.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:55.964 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:27:56.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:55 smithi171 ceph-mon[90031]: pgmap v989: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 21207/59778 objects degraded (35.476%); 5816/59778 objects misplaced (9.729%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:27:56.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:56.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:55 smithi027 ceph-mon[93921]: pgmap v989: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 21207/59778 objects degraded (35.476%); 5816/59778 objects misplaced (9.729%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:27:56.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:55 smithi027 ceph-mon[98970]: pgmap v989: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 21207/59778 objects degraded (35.476%); 5816/59778 objects misplaced (9.729%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:27:56.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:56.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:56.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:56.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:56.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:56.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:56.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:56.355 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:56.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:56.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:56.392 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:27:56.392 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:27:56.423 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:27:56.423 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:27:56.452 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:27:56.452 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:27:56.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:56.488 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:27:56.489 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:27:56.495 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:56.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:56.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:56.518 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:27:56.518 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:27:56.543 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:27:56.543 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:27:56.573 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:27:56.573 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:27:56.597 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:56.601 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:27:56.602 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:27:56.625 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:56.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:56.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:56.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:56.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:56.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:56.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:56.785 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:27:56.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:56.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:56.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:56.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:56.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:27:57.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:57.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:27:57.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3523257756' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:57.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3460531948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:57.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3523257756' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:57.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3460531948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:57.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:57.134 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3523257756' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:27:57.135 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3460531948' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:27:57.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:57.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:57.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:57.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:57.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:57.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:57.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:57.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:57.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:57.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:57.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:57.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:57.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:57.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:57.933 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:57.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:57.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:58.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:58.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:58.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[93921]: pgmap v990: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 20557/53922 objects degraded (38.124%); 5558/53922 objects misplaced (10.307%); 7.7 MiB/s, 122 objects/s recovering 2024-08-29T15:27:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:27:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3523257756' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:58.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[93921]: osdmap e284: 8 total, 8 up, 8 in 2024-08-29T15:27:58.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 20557/53922 objects degraded (38.124%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[98970]: pgmap v990: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 20557/53922 objects degraded (38.124%); 5558/53922 objects misplaced (10.307%); 7.7 MiB/s, 122 objects/s recovering 2024-08-29T15:27:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:27:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3523257756' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[98970]: osdmap e284: 8 total, 8 up, 8 in 2024-08-29T15:27:58.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 20557/53922 objects degraded (38.124%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:58.140 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:57 smithi171 ceph-mon[90031]: pgmap v990: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 20557/53922 objects degraded (38.124%); 5558/53922 objects misplaced (10.307%); 7.7 MiB/s, 122 objects/s recovering 2024-08-29T15:27:58.140 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:57 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:27:58.140 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3523257756' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:27:58.141 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:57 smithi171 ceph-mon[90031]: osdmap e284: 8 total, 8 up, 8 in 2024-08-29T15:27:58.141 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 20557/53922 objects degraded (38.124%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:27:58.156 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":988,"stamp":"2024-08-29T15:27:52.674775+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1325078525,"num_objects":20175,"num_object_clones":0,"num_object_copies":60525,"num_objects_missing_on_primary":2287,"num_objects_missing":2287,"num_objects_degraded":21207,"num_objects_misplaced":5816,"num_objects_unfound":0,"num_objects_dirty":20175,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":231826,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24784,"num_bytes_recovered":1503071437,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125462,"ondisk_log_size":125462,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6953160,"kb_used_data":4878920,"kb_used_omap":43,"kb_used_meta":2074132,"kb_avail":743008056,"statfs":{"total":767960285184,"available":760840249344,"internally_reserved":0,"allocated":4996014080,"data_stored":4982819534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2123911660},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,3,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-249626624,"num_objects":-3809,"num_object_clones":0,"num_object_copies":-11427,"num_objects_missing_on_primary":-220,"num_objects_missing":-220,"num_objects_degraded":-1179,"num_objects_misplaced":-458,"num_objects_unfound":0,"num_objects_dirty":-3809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3809,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":951,"num_bytes_recovered":62324736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002962"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":453,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.748867+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:27:42.748867+0000","last_peered":"2024-08-29T15:27:42.748867+0000","last_clean":"2024-08-29T15:27:42.748867+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:27:42.748867+0000","last_undegraded":"2024-08-29T15:27:42.748867+0000","last_fullsized":"2024-08-29T15:27:42.748867+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":62,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:32.710563+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:27:32.710563+0000","last_peered":"2024-08-29T15:27:32.710563+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:27:32.710563+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:27:32.710563+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":152,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707719+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:27:32.707719+0000","last_peered":"2024-08-29T15:27:32.707719+0000","last_clean":"2024-08-29T15:27:32.707719+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:27:32.707719+0000","last_undegraded":"2024-08-29T15:27:32.707719+0000","last_fullsized":"2024-08-29T15:27:32.707719+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":418,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.757813+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:27:42.757813+0000","last_peered":"2024-08-29T15:27:42.757813+0000","last_clean":"2024-08-29T15:27:42.757813+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:27:42.757813+0000","last_undegraded":"2024-08-29T15:27:42.757813+0000","last_fullsized":"2024-08-29T15:27:42.757813+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":418,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.760015+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:27:42.760015+0000","last_peered":"2024-08-29T15:27:42.760015+0000","last_clean":"2024-08-29T15:27:42.760015+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:27:42.760015+0000","last_undegraded":"2024-08-29T15:27:42.760015+0000","last_fullsized":"2024-08-29T15:27:42.760015+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709804+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:27:32.709804+0000","last_peered":"2024-08-29T15:27:32.709804+0000","last_clean":"2024-08-29T15:27:32.709804+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:27:32.709804+0000","last_undegraded":"2024-08-29T15:27:32.709804+0000","last_fullsized":"2024-08-29T15:27:32.709804+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":230,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753903+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:27:42.753903+0000","last_peered":"2024-08-29T15:27:42.753903+0000","last_clean":"2024-08-29T15:27:42.753903+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:27:42.753903+0000","last_undegraded":"2024-08-29T15:27:42.753903+0000","last_fullsized":"2024-08-29T15:27:42.753903+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":153,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.708896+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:27:32.708896+0000","last_peered":"2024-08-29T15:27:32.708896+0000","last_clean":"2024-08-29T15:27:32.708896+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:27:32.708896+0000","last_undegraded":"2024-08-29T15:27:32.708896+0000","last_fullsized":"2024-08-29T15:27:32.708896+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"283'14604","reported_seq":22667,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:51.185243+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:27:51.185243+0000","last_peered":"2024-08-29T15:27:51.185243+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:27:51.185243+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"180'4600","ondisk_log_start":"180'4600","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10004,"log_dups_size":0,"ondisk_log_size":10004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85721088,"num_objects":1308,"num_object_clones":0,"num_object_copies":3924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14604,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,5","objects":-1617}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":61,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:32.710613+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:27:32.710613+0000","last_peered":"2024-08-29T15:27:32.710613+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:27:32.710613+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:27:32.710613+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":170,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707150+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:27:32.707150+0000","last_peered":"2024-08-29T15:27:32.707150+0000","last_clean":"2024-08-29T15:27:32.707150+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:27:32.707150+0000","last_undegraded":"2024-08-29T15:27:32.707150+0000","last_fullsized":"2024-08-29T15:27:32.707150+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"283'14289","reported_seq":14535,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:51.381720+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:27:51.381720+0000","last_peered":"2024-08-29T15:27:51.381720+0000","last_clean":"2024-08-29T15:27:51.381720+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:27:51.381720+0000","last_undegraded":"2024-08-29T15:27:51.381720+0000","last_fullsized":"2024-08-29T15:27:51.381720+0000","mapping_epoch":168,"log_start":"188'7528","ondisk_log_start":"188'7528","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82640896,"num_objects":1261,"num_object_clones":0,"num_object_copies":3783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14289,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":61,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:32.712459+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:27:32.712459+0000","last_peered":"2024-08-29T15:27:32.712459+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:27:32.712459+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:27:32.712459+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709647+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:27:32.709647+0000","last_peered":"2024-08-29T15:27:32.709647+0000","last_clean":"2024-08-29T15:27:32.709647+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:27:32.709647+0000","last_undegraded":"2024-08-29T15:27:32.709647+0000","last_fullsized":"2024-08-29T15:27:32.709647+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"283'14394","reported_seq":14610,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:51.425826+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:27:51.425826+0000","last_peered":"2024-08-29T15:27:51.425826+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:27:51.425826+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"188'7674","ondisk_log_start":"188'7674","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86376448,"num_objects":1318,"num_object_clones":0,"num_object_copies":3954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14394,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1159}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":235,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.759860+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:27:42.759860+0000","last_peered":"2024-08-29T15:27:42.759860+0000","last_clean":"2024-08-29T15:27:42.759860+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:27:42.759860+0000","last_undegraded":"2024-08-29T15:27:42.759860+0000","last_fullsized":"2024-08-29T15:27:42.759860+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709550+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:27:32.709550+0000","last_peered":"2024-08-29T15:27:32.709550+0000","last_clean":"2024-08-29T15:27:32.709550+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:27:32.709550+0000","last_undegraded":"2024-08-29T15:27:32.709550+0000","last_fullsized":"2024-08-29T15:27:32.709550+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"283'14492","reported_seq":19958,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:48.892620+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:27:48.892620+0000","last_peered":"2024-08-29T15:27:48.892620+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:27:48.892620+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"187'6362","ondisk_log_start":"187'6362","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84279296,"num_objects":1286,"num_object_clones":0,"num_object_copies":3858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14492,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1286}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707627+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:27:32.707627+0000","last_peered":"2024-08-29T15:27:32.707627+0000","last_clean":"2024-08-29T15:27:32.707627+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:27:32.707627+0000","last_undegraded":"2024-08-29T15:27:32.707627+0000","last_fullsized":"2024-08-29T15:27:32.707627+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":433,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.759687+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:27:42.759687+0000","last_peered":"2024-08-29T15:27:42.759687+0000","last_clean":"2024-08-29T15:27:42.759687+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:27:42.759687+0000","last_undegraded":"2024-08-29T15:27:42.759687+0000","last_fullsized":"2024-08-29T15:27:42.759687+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"283'14501","reported_seq":17000,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:49.758472+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:27:49.758472+0000","last_peered":"2024-08-29T15:27:49.758472+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:27:49.758472+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:27:49.758472+0000","mapping_epoch":234,"log_start":"187'6533","ondisk_log_start":"187'6533","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7968,"log_dups_size":0,"ondisk_log_size":7968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14501,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1164,"num_bytes_recovered":76283904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":1313}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":427,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.748663+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:27:42.748663+0000","last_peered":"2024-08-29T15:27:42.748663+0000","last_clean":"2024-08-29T15:27:42.748663+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:27:42.748663+0000","last_undegraded":"2024-08-29T15:27:42.748663+0000","last_fullsized":"2024-08-29T15:27:42.748663+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710186+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:27:32.710186+0000","last_peered":"2024-08-29T15:27:32.710186+0000","last_clean":"2024-08-29T15:27:32.710186+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:27:32.710186+0000","last_undegraded":"2024-08-29T15:27:32.710186+0000","last_fullsized":"2024-08-29T15:27:32.710186+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"283'14580","reported_seq":17249,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:51.493477+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:27:51.493477+0000","last_peered":"2024-08-29T15:27:51.493477+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:27:51.493477+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:27:51.493477+0000","mapping_epoch":234,"log_start":"191'7868","ondisk_log_start":"191'7868","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82706432,"num_objects":1262,"num_object_clones":0,"num_object_copies":3786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14580,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1203,"num_bytes_recovered":78839808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":1262}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709474+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:27:32.709474+0000","last_peered":"2024-08-29T15:27:32.709474+0000","last_clean":"2024-08-29T15:27:32.709474+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:27:32.709474+0000","last_undegraded":"2024-08-29T15:27:32.709474+0000","last_fullsized":"2024-08-29T15:27:32.709474+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710390+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:27:32.710390+0000","last_peered":"2024-08-29T15:27:32.710390+0000","last_clean":"2024-08-29T15:27:32.710390+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:27:32.710390+0000","last_undegraded":"2024-08-29T15:27:32.710390+0000","last_fullsized":"2024-08-29T15:27:32.710390+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"283'14418","reported_seq":14865,"reported_epoch":283,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:27:51.464507+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:27:51.464507+0000","last_peered":"2024-08-29T15:27:51.464507+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:27:51.464507+0000","last_undegraded":"2024-08-29T15:27:51.464507+0000","last_fullsized":"2024-08-29T15:27:51.464507+0000","mapping_epoch":202,"log_start":"188'7667","ondisk_log_start":"188'7667","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81264640,"num_objects":1240,"num_object_clones":0,"num_object_copies":3720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1240,"num_objects_unfound":0,"num_objects_dirty":1240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14418,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1240}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709732+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:27:32.709732+0000","last_peered":"2024-08-29T15:27:32.709732+0000","last_clean":"2024-08-29T15:27:32.709732+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:27:32.709732+0000","last_undegraded":"2024-08-29T15:27:32.709732+0000","last_fullsized":"2024-08-29T15:27:32.709732+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":269,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.712464+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:27:32.712464+0000","last_peered":"2024-08-29T15:27:32.712464+0000","last_clean":"2024-08-29T15:27:32.712464+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:27:32.712464+0000","last_undegraded":"2024-08-29T15:27:32.712464+0000","last_fullsized":"2024-08-29T15:27:32.712464+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"283'14506","reported_seq":14671,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:51.185675+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:27:51.185675+0000","last_peered":"2024-08-29T15:27:51.185675+0000","last_clean":"2024-08-29T15:27:51.185675+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:27:51.185675+0000","last_undegraded":"2024-08-29T15:27:51.185675+0000","last_fullsized":"2024-08-29T15:27:51.185675+0000","mapping_epoch":218,"log_start":"180'4478","ondisk_log_start":"180'4478","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10028,"log_dups_size":0,"ondisk_log_size":10028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82116632,"num_objects":1254,"num_object_clones":0,"num_object_copies":3762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1254,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14506,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":430,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.752031+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:27:42.752031+0000","last_peered":"2024-08-29T15:27:42.752031+0000","last_clean":"2024-08-29T15:27:42.752031+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:27:42.752031+0000","last_undegraded":"2024-08-29T15:27:42.752031+0000","last_fullsized":"2024-08-29T15:27:42.752031+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":422,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753951+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:27:42.753951+0000","last_peered":"2024-08-29T15:27:42.753951+0000","last_clean":"2024-08-29T15:27:42.753951+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:27:42.753951+0000","last_undegraded":"2024-08-29T15:27:42.753951+0000","last_fullsized":"2024-08-29T15:27:42.753951+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"283'13953","reported_seq":14166,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:51.080199+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:27:51.080199+0000","last_peered":"2024-08-29T15:27:51.080199+0000","last_clean":"2024-08-29T15:27:51.080199+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:27:51.080199+0000","last_undegraded":"2024-08-29T15:27:51.080199+0000","last_fullsized":"2024-08-29T15:27:51.080199+0000","mapping_epoch":201,"log_start":"179'3900","ondisk_log_start":"179'3900","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10053,"log_dups_size":0,"ondisk_log_size":10053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82247680,"num_objects":1255,"num_object_clones":0,"num_object_copies":3765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13953,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":521,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.758168+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:27:42.758168+0000","last_peered":"2024-08-29T15:27:42.758168+0000","last_clean":"2024-08-29T15:27:42.758168+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:27:42.758168+0000","last_undegraded":"2024-08-29T15:27:42.758168+0000","last_fullsized":"2024-08-29T15:27:42.758168+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":754,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707335+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:27:32.707335+0000","last_peered":"2024-08-29T15:27:32.707335+0000","last_clean":"2024-08-29T15:27:32.707335+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:27:32.707335+0000","last_undegraded":"2024-08-29T15:27:32.707335+0000","last_fullsized":"2024-08-29T15:27:32.707335+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":262,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707315+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:27:32.707315+0000","last_peered":"2024-08-29T15:27:32.707315+0000","last_clean":"2024-08-29T15:27:32.707315+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:27:32.707315+0000","last_undegraded":"2024-08-29T15:27:32.707315+0000","last_fullsized":"2024-08-29T15:27:32.707315+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":125,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:34.763127+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:27:34.763127+0000","last_peered":"2024-08-29T15:27:34.763127+0000","last_clean":"2024-08-29T15:27:34.763127+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:27:34.763127+0000","last_undegraded":"2024-08-29T15:27:34.763127+0000","last_fullsized":"2024-08-29T15:27:34.763127+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"283'14426","reported_seq":14871,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:50.746285+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:27:50.746285+0000","last_peered":"2024-08-29T15:27:50.746285+0000","last_clean":"2024-08-29T15:27:50.746285+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:27:50.746285+0000","last_undegraded":"2024-08-29T15:27:50.746285+0000","last_fullsized":"2024-08-29T15:27:50.746285+0000","mapping_epoch":201,"log_start":"188'7353","ondisk_log_start":"188'7353","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82313216,"num_objects":1256,"num_object_clones":0,"num_object_copies":3768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14426,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":895,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751338+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:27:42.751338+0000","last_peered":"2024-08-29T15:27:42.751338+0000","last_clean":"2024-08-29T15:27:42.751338+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:27:42.751338+0000","last_undegraded":"2024-08-29T15:27:42.751338+0000","last_fullsized":"2024-08-29T15:27:42.751338+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":433,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.758637+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:27:42.758637+0000","last_peered":"2024-08-29T15:27:42.758637+0000","last_clean":"2024-08-29T15:27:42.758637+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:27:42.758637+0000","last_undegraded":"2024-08-29T15:27:42.758637+0000","last_fullsized":"2024-08-29T15:27:42.758637+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":441,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.760242+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:27:42.760242+0000","last_peered":"2024-08-29T15:27:42.760242+0000","last_clean":"2024-08-29T15:27:42.760242+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:27:42.760242+0000","last_undegraded":"2024-08-29T15:27:42.760242+0000","last_fullsized":"2024-08-29T15:27:42.760242+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"283'14438","reported_seq":26778,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:51.493849+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:27:51.493849+0000","last_peered":"2024-08-29T15:27:51.493849+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:27:51.493849+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"188'7730","ondisk_log_start":"188'7730","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6708,"log_dups_size":0,"ondisk_log_size":6708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82051072,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14438,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6062,"num_bytes_recovered":397279232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"3,6","objects":54},{"shards":"3,7","objects":1198}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":704,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.748189+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:27:42.748189+0000","last_peered":"2024-08-29T15:27:42.748189+0000","last_clean":"2024-08-29T15:27:42.748189+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:27:42.748189+0000","last_undegraded":"2024-08-29T15:27:42.748189+0000","last_fullsized":"2024-08-29T15:27:42.748189+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":115,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.708759+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:27:32.708759+0000","last_peered":"2024-08-29T15:27:32.708759+0000","last_clean":"2024-08-29T15:27:32.708759+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:27:32.708759+0000","last_undegraded":"2024-08-29T15:27:32.708759+0000","last_fullsized":"2024-08-29T15:27:32.708759+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":468,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753705+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:27:42.753705+0000","last_peered":"2024-08-29T15:27:42.753705+0000","last_clean":"2024-08-29T15:27:42.753705+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:27:42.753705+0000","last_undegraded":"2024-08-29T15:27:42.753705+0000","last_fullsized":"2024-08-29T15:27:42.753705+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"283'14492","reported_seq":14748,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:51.350361+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:27:51.350361+0000","last_peered":"2024-08-29T15:27:51.350361+0000","last_clean":"2024-08-29T15:27:51.350361+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:27:51.350361+0000","last_undegraded":"2024-08-29T15:27:51.350361+0000","last_fullsized":"2024-08-29T15:27:51.350361+0000","mapping_epoch":234,"log_start":"191'7797","ondisk_log_start":"191'7797","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6695,"log_dups_size":0,"ondisk_log_size":6695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14492,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":222,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707195+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:27:32.707195+0000","last_peered":"2024-08-29T15:27:32.707195+0000","last_clean":"2024-08-29T15:27:32.707195+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:27:32.707195+0000","last_undegraded":"2024-08-29T15:27:32.707195+0000","last_fullsized":"2024-08-29T15:27:32.707195+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710080+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:27:32.710080+0000","last_peered":"2024-08-29T15:27:32.710080+0000","last_clean":"2024-08-29T15:27:32.710080+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:27:32.710080+0000","last_undegraded":"2024-08-29T15:27:32.710080+0000","last_fullsized":"2024-08-29T15:27:32.710080+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":129,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707404+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:27:32.707404+0000","last_peered":"2024-08-29T15:27:32.707404+0000","last_clean":"2024-08-29T15:27:32.707404+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:27:32.707404+0000","last_undegraded":"2024-08-29T15:27:32.707404+0000","last_fullsized":"2024-08-29T15:27:32.707404+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"283'14536","reported_seq":14791,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:48.994876+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:27:48.994876+0000","last_peered":"2024-08-29T15:27:48.994876+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:27:48.994876+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"187'6342","ondisk_log_start":"187'6342","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84541440,"num_objects":1290,"num_object_clones":0,"num_object_copies":3870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14536,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-1133}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707434+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:27:32.707434+0000","last_peered":"2024-08-29T15:27:32.707434+0000","last_clean":"2024-08-29T15:27:32.707434+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:27:32.707434+0000","last_undegraded":"2024-08-29T15:27:32.707434+0000","last_fullsized":"2024-08-29T15:27:32.707434+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":860,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751796+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:27:42.751796+0000","last_peered":"2024-08-29T15:27:42.751796+0000","last_clean":"2024-08-29T15:27:42.751796+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:27:42.751796+0000","last_undegraded":"2024-08-29T15:27:42.751796+0000","last_fullsized":"2024-08-29T15:27:42.751796+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":437,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751647+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:27:42.751647+0000","last_peered":"2024-08-29T15:27:42.751647+0000","last_clean":"2024-08-29T15:27:42.751647+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:27:42.751647+0000","last_undegraded":"2024-08-29T15:27:42.751647+0000","last_fullsized":"2024-08-29T15:27:42.751647+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"283'14477","reported_seq":15288,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:50.664106+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:27:50.664106+0000","last_peered":"2024-08-29T15:27:50.664106+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:27:50.664106+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"188'7480","ondisk_log_start":"188'7480","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78053376,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14477,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":6},{"shards":"1,7","objects":-960}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":442,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751709+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:27:42.751709+0000","last_peered":"2024-08-29T15:27:42.751709+0000","last_clean":"2024-08-29T15:27:42.751709+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:27:42.751709+0000","last_undegraded":"2024-08-29T15:27:42.751709+0000","last_fullsized":"2024-08-29T15:27:42.751709+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":712,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:42.760016+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:27:42.760016+0000","last_peered":"2024-08-29T15:27:42.760016+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:27:42.760016+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:27:42.760016+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":433,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.759605+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:27:42.759605+0000","last_peered":"2024-08-29T15:27:42.759605+0000","last_clean":"2024-08-29T15:27:42.759605+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:27:42.759605+0000","last_undegraded":"2024-08-29T15:27:42.759605+0000","last_fullsized":"2024-08-29T15:27:42.759605+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"283'14228","reported_seq":21397,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:49.809162+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:27:49.809162+0000","last_peered":"2024-08-29T15:27:49.809162+0000","last_clean":"2024-08-29T15:27:49.809162+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:27:49.809162+0000","last_undegraded":"2024-08-29T15:27:49.809162+0000","last_fullsized":"2024-08-29T15:27:49.809162+0000","mapping_epoch":234,"log_start":"187'6299","ondisk_log_start":"187'6299","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7929,"log_dups_size":0,"ondisk_log_size":7929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78643200,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14228,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":427,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.749639+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:27:42.749639+0000","last_peered":"2024-08-29T15:27:42.749639+0000","last_clean":"2024-08-29T15:27:42.749639+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:27:42.749639+0000","last_undegraded":"2024-08-29T15:27:42.749639+0000","last_fullsized":"2024-08-29T15:27:42.749639+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1050,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:42.752030+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:27:42.752030+0000","last_peered":"2024-08-29T15:27:42.752030+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:27:42.752030+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":425,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751285+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:27:42.751285+0000","last_peered":"2024-08-29T15:27:42.751285+0000","last_clean":"2024-08-29T15:27:42.751285+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:27:42.751285+0000","last_undegraded":"2024-08-29T15:27:42.751285+0000","last_fullsized":"2024-08-29T15:27:42.751285+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"283'14369","reported_seq":26571,"reported_epoch":283,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:27:49.795849+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:27:49.795849+0000","last_peered":"2024-08-29T15:27:49.795849+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:27:49.795849+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"187'6812","ondisk_log_start":"187'6812","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81330176,"num_objects":1241,"num_object_clones":0,"num_object_copies":3723,"num_objects_missing_on_primary":2279,"num_objects_missing":2279,"num_objects_degraded":1172,"num_objects_misplaced":1241,"num_objects_unfound":0,"num_objects_dirty":1241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14369,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5445,"num_bytes_recovered":356712448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":69},{"shards":"1,7","objects":1172}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":423,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.759923+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:27:42.759923+0000","last_peered":"2024-08-29T15:27:42.759923+0000","last_clean":"2024-08-29T15:27:42.759923+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:27:42.759923+0000","last_undegraded":"2024-08-29T15:27:42.759923+0000","last_fullsized":"2024-08-29T15:27:42.759923+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":136,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753807+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:27:42.753807+0000","last_peered":"2024-08-29T15:27:42.753807+0000","last_clean":"2024-08-29T15:27:42.753807+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:27:42.753807+0000","last_undegraded":"2024-08-29T15:27:42.753807+0000","last_fullsized":"2024-08-29T15:27:42.753807+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709732+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:27:32.709732+0000","last_peered":"2024-08-29T15:27:32.709732+0000","last_clean":"2024-08-29T15:27:32.709732+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:27:32.709732+0000","last_undegraded":"2024-08-29T15:27:32.709732+0000","last_fullsized":"2024-08-29T15:27:32.709732+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710159+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:27:32.710159+0000","last_peered":"2024-08-29T15:27:32.710159+0000","last_clean":"2024-08-29T15:27:32.710159+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:27:32.710159+0000","last_undegraded":"2024-08-29T15:27:32.710159+0000","last_fullsized":"2024-08-29T15:27:32.710159+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":61,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:32.710288+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:27:32.710288+0000","last_peered":"2024-08-29T15:27:32.710288+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:27:32.710288+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:27:32.710288+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":431,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751499+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:27:42.751499+0000","last_peered":"2024-08-29T15:27:42.751499+0000","last_clean":"2024-08-29T15:27:42.751499+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:27:42.751499+0000","last_undegraded":"2024-08-29T15:27:42.751499+0000","last_fullsized":"2024-08-29T15:27:42.751499+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":182,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:42.750633+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:27:42.750633+0000","last_peered":"2024-08-29T15:27:42.750633+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:27:42.750633+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:27:42.750633+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":152,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.707588+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:27:32.707588+0000","last_peered":"2024-08-29T15:27:32.707588+0000","last_clean":"2024-08-29T15:27:32.707588+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:27:32.707588+0000","last_undegraded":"2024-08-29T15:27:32.707588+0000","last_fullsized":"2024-08-29T15:27:32.707588+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":391,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.747805+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:27:42.747805+0000","last_peered":"2024-08-29T15:27:42.747805+0000","last_clean":"2024-08-29T15:27:42.747805+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:27:42.747805+0000","last_undegraded":"2024-08-29T15:27:42.747805+0000","last_fullsized":"2024-08-29T15:27:42.747805+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":444,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.760323+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:27:42.760323+0000","last_peered":"2024-08-29T15:27:42.760323+0000","last_clean":"2024-08-29T15:27:42.760323+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:27:42.760323+0000","last_undegraded":"2024-08-29T15:27:42.760323+0000","last_fullsized":"2024-08-29T15:27:42.760323+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":443,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751203+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:27:42.751203+0000","last_peered":"2024-08-29T15:27:42.751203+0000","last_clean":"2024-08-29T15:27:42.751203+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:27:42.751203+0000","last_undegraded":"2024-08-29T15:27:42.751203+0000","last_fullsized":"2024-08-29T15:27:42.751203+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":124,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709518+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:27:32.709518+0000","last_peered":"2024-08-29T15:27:32.709518+0000","last_clean":"2024-08-29T15:27:32.709518+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:27:32.709518+0000","last_undegraded":"2024-08-29T15:27:32.709518+0000","last_fullsized":"2024-08-29T15:27:32.709518+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.758043+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:27:42.758043+0000","last_peered":"2024-08-29T15:27:42.758043+0000","last_clean":"2024-08-29T15:27:42.758043+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:27:42.758043+0000","last_undegraded":"2024-08-29T15:27:42.758043+0000","last_fullsized":"2024-08-29T15:27:42.758043+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":420,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.760156+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:27:42.760156+0000","last_peered":"2024-08-29T15:27:42.760156+0000","last_clean":"2024-08-29T15:27:42.760156+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:27:42.760156+0000","last_undegraded":"2024-08-29T15:27:42.760156+0000","last_fullsized":"2024-08-29T15:27:42.760156+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":423,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.752811+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:27:42.752811+0000","last_peered":"2024-08-29T15:27:42.752811+0000","last_clean":"2024-08-29T15:27:42.752811+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:27:42.752811+0000","last_undegraded":"2024-08-29T15:27:42.752811+0000","last_fullsized":"2024-08-29T15:27:42.752811+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":427,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.750380+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:27:42.750380+0000","last_peered":"2024-08-29T15:27:42.750380+0000","last_clean":"2024-08-29T15:27:42.750380+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:27:42.750380+0000","last_undegraded":"2024-08-29T15:27:42.750380+0000","last_fullsized":"2024-08-29T15:27:42.750380+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709896+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:27:32.709896+0000","last_peered":"2024-08-29T15:27:32.709896+0000","last_clean":"2024-08-29T15:27:32.709896+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:27:32.709896+0000","last_undegraded":"2024-08-29T15:27:32.709896+0000","last_fullsized":"2024-08-29T15:27:32.709896+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":447,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.751854+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:27:42.751854+0000","last_peered":"2024-08-29T15:27:42.751854+0000","last_clean":"2024-08-29T15:27:42.751854+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:27:42.751854+0000","last_undegraded":"2024-08-29T15:27:42.751854+0000","last_fullsized":"2024-08-29T15:27:42.751854+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":411,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.759494+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:27:42.759494+0000","last_peered":"2024-08-29T15:27:42.759494+0000","last_clean":"2024-08-29T15:27:42.759494+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:27:42.759494+0000","last_undegraded":"2024-08-29T15:27:42.759494+0000","last_fullsized":"2024-08-29T15:27:42.759494+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.710063+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:27:32.710063+0000","last_peered":"2024-08-29T15:27:32.710063+0000","last_clean":"2024-08-29T15:27:32.710063+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:27:32.710063+0000","last_undegraded":"2024-08-29T15:27:32.710063+0000","last_fullsized":"2024-08-29T15:27:32.710063+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":176,"reported_epoch":282,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:41.217466+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:27:41.217466+0000","last_peered":"2024-08-29T15:27:41.217466+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:27:41.217466+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:27:41.217466+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709887+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:27:32.709887+0000","last_peered":"2024-08-29T15:27:32.709887+0000","last_clean":"2024-08-29T15:27:32.709887+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:27:32.709887+0000","last_undegraded":"2024-08-29T15:27:32.709887+0000","last_fullsized":"2024-08-29T15:27:32.709887+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":182,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:41.217533+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:27:41.217533+0000","last_peered":"2024-08-29T15:27:41.217533+0000","last_clean":"2024-08-29T15:27:41.217533+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:27:41.217533+0000","last_undegraded":"2024-08-29T15:27:41.217533+0000","last_fullsized":"2024-08-29T15:27:41.217533+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":458,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.748275+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:27:42.748275+0000","last_peered":"2024-08-29T15:27:42.748275+0000","last_clean":"2024-08-29T15:27:42.748275+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:27:42.748275+0000","last_undegraded":"2024-08-29T15:27:42.748275+0000","last_fullsized":"2024-08-29T15:27:42.748275+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":460,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.749295+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:27:42.749295+0000","last_peered":"2024-08-29T15:27:42.749295+0000","last_clean":"2024-08-29T15:27:42.749295+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:27:42.749295+0000","last_undegraded":"2024-08-29T15:27:42.749295+0000","last_fullsized":"2024-08-29T15:27:42.749295+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":282,"state":"active+clean","last_fresh":"2024-08-29T15:27:32.709004+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:27:32.709004+0000","last_peered":"2024-08-29T15:27:32.709004+0000","last_clean":"2024-08-29T15:27:32.709004+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:27:32.709004+0000","last_undegraded":"2024-08-29T15:27:32.709004+0000","last_fullsized":"2024-08-29T15:27:32.709004+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":467,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.760276+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:27:42.760276+0000","last_peered":"2024-08-29T15:27:42.760276+0000","last_clean":"2024-08-29T15:27:42.760276+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:27:42.760276+0000","last_undegraded":"2024-08-29T15:27:42.760276+0000","last_fullsized":"2024-08-29T15:27:42.760276+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1320288280,"num_objects":20147,"num_object_clones":0,"num_object_copies":60441,"num_objects_missing_on_primary":2279,"num_objects_missing":2279,"num_objects_degraded":21178,"num_objects_misplaced":5816,"num_objects_unfound":0,"num_objects_dirty":20147,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":230703,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24675,"num_bytes_recovered":1497104384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4913704960,"data_stored":4913692744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124280,"ondisk_log_size":124280,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234398,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005896,"kb_used_data":663872,"kb_used_omap":1,"kb_used_meta":342014,"kb_avail":92739256,"statfs":{"total":95995035648,"available":94964998144,"internally_reserved":0,"allocated":679804928,"data_stored":678143963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":350222795},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496029,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236868,"kb_used_data":17084,"kb_used_omap":1,"kb_used_meta":219774,"kb_avail":93508284,"statfs":{"total":95995035648,"available":95752482816,"internally_reserved":0,"allocated":17494016,"data_stored":15836851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":225049031},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426616,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":484408,"kb_used_data":411248,"kb_used_omap":8,"kb_used_meta":73143,"kb_avail":93260744,"statfs":{"total":95995035648,"available":95499001856,"internally_reserved":0,"allocated":421117952,"data_stored":419468253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":74899338},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019308,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":756264,"kb_used_data":418336,"kb_used_omap":1,"kb_used_meta":337918,"kb_avail":92988888,"statfs":{"total":95995035648,"available":95220621312,"internally_reserved":0,"allocated":428376064,"data_stored":426742148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":346028486},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435493,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1140148,"kb_used_data":815596,"kb_used_omap":4,"kb_used_meta":324539,"kb_avail":92605004,"statfs":{"total":95995035648,"available":94827524096,"internally_reserved":0,"allocated":835170304,"data_stored":833524545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":332328201},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903345,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1536824,"kb_used_data":1369008,"kb_used_omap":10,"kb_used_meta":167797,"kb_avail":92208328,"statfs":{"total":95995035648,"available":94421327872,"internally_reserved":0,"allocated":1401864192,"data_stored":1400218369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":171824656},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762546,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1128372,"kb_used_data":839852,"kb_used_omap":7,"kb_used_meta":288504,"kb_avail":92616780,"statfs":{"total":95995035648,"available":94839582720,"internally_reserved":0,"allocated":860008448,"data_stored":858361449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":295428450},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926072,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":664380,"kb_used_data":343924,"kb_used_omap":7,"kb_used_meta":320440,"kb_avail":93080772,"statfs":{"total":95995035648,"available":95314710528,"internally_reserved":0,"allocated":352178176,"data_stored":350523956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":328130703},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":343412736,"data_stored":343408664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":847122432,"data_stored":847118360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1397358592,"data_stored":1397358592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":815398912,"data_stored":815398912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":416546816,"data_stored":416546816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":413995008,"data_stored":413990936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12976128,"data_stored":12976128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":666894336,"data_stored":666894336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:27:58.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:27:58.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:58.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:58.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:58.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:27:58.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:27:58.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:58.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:58.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:27:58.412 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:27:58.412 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:27:58.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:58.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:58.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:58.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:58.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:58.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:58.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:58.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:58.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:27:58.748 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:27:58.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:58.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:27:58.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:58.854 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:27:58.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:58.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:58.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:58.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:59.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:59.122 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:59.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:59.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:27:59.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:59.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:27:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:27:59] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.51.0" 2024-08-29T15:27:59.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:59.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:27:59.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:27:59.468 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:27:59.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:27:59.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:27:59.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:59.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:59.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:27:59.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:27:59.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:27:59.879 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:59 smithi171 ceph-mon[90031]: pgmap v992: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 20557/53922 objects degraded (38.124%); 5558/53922 objects misplaced (10.307%); 6.4 MiB/s, 102 objects/s recovering 2024-08-29T15:27:59.879 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:27:59.879 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:27:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:27:59.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:27:59.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:27:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:27:59.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:00.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:00.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:59 smithi027 ceph-mon[98970]: pgmap v992: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 20557/53922 objects degraded (38.124%); 5558/53922 objects misplaced (10.307%); 6.4 MiB/s, 102 objects/s recovering 2024-08-29T15:28:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:27:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:59 smithi027 ceph-mon[93921]: pgmap v992: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 20557/53922 objects degraded (38.124%); 5558/53922 objects misplaced (10.307%); 6.4 MiB/s, 102 objects/s recovering 2024-08-29T15:28:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:00.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:27:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:00.149 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:00.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:00.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:00.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:00.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:00.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:00.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:00.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:00.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:00.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:00.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:00.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:00.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:00.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:00.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:00.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:28:00.822 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:00.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:00.952 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:00.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:00.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:00.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:01.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:01.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:01.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:01.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:01.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:01.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:01.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:01.315 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:28:01.316 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 20295/51948 objects degraded (39.068%), 17 pgs degraded, 8 pgs undersized","count":25},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1463,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":284,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":17316,"data_bytes":1137711101,"bytes_used":6648987648,"bytes_avail":761311297536,"bytes_total":767960285184,"degraded_objects":20295,"degraded_total":51948,"degraded_ratio":0.39067914067914067,"misplaced_objects":5431,"misplaced_total":51948,"misplaced_ratio":0.10454685454685456,"recovering_objects_per_sec":104,"recovering_bytes_per_sec":6859598,"recovering_keys_per_sec":0,"num_objects_recovered":1047,"num_bytes_recovered":68616192,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":417},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:28:01.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:01.373 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:01.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:01.425 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:28:01.425 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:28:01.475 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:01.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:01.500 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:01.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:01.601 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:01.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:01.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:01.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:01.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:01.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:01.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:01.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:01.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:01.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:01.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:01.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:01.999 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:01 smithi027 ceph-mon[98970]: pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 20295/51948 objects degraded (39.068%); 5431/51948 objects misplaced (10.455%); 6.5 MiB/s, 104 objects/s recovering 2024-08-29T15:28:01.999 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4002635003' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:28:02.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:01 smithi027 ceph-mon[93921]: pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 20295/51948 objects degraded (39.068%); 5431/51948 objects misplaced (10.455%); 6.5 MiB/s, 104 objects/s recovering 2024-08-29T15:28:02.000 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4002635003' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:28:02.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:01 smithi171 ceph-mon[90031]: pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1.1 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 20295/51948 objects degraded (39.068%); 5431/51948 objects misplaced (10.455%); 6.5 MiB/s, 104 objects/s recovering 2024-08-29T15:28:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4002635003' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:28:02.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:02.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:02.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:02.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:02.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:02.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:28:02.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:02.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:02.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:02.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:02.460 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:28:02.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:28:02.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:02.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:02.588 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:02.590 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:02.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:02.620 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:02.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:02.721 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:02.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:02.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:02.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:02.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:02.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:02.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:02.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:02.972 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 20295/51948 objects degraded (39.068%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:02.972 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:02 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:28:02.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 20295/51948 objects degraded (39.068%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:02.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:02 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:28:03.016 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:03.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:03.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 20295/51948 objects degraded (39.068%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:03.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:02 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:28:03.056 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:03.062 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:03.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:03.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:03.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:03.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:03.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:03.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:03.432 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:03.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:03.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:03.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:03.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:03.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:03.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:03.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:03.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:03.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:03 smithi171 ceph-mon[90031]: pgmap v994: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1005 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19958/48084 objects degraded (41.507%); 5332/48084 objects misplaced (11.089%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:28:04.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:03 smithi027 ceph-mon[93921]: pgmap v994: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1005 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19958/48084 objects degraded (41.507%); 5332/48084 objects misplaced (11.089%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:28:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:03 smithi027 ceph-mon[98970]: pgmap v994: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 1005 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19958/48084 objects degraded (41.507%); 5332/48084 objects misplaced (11.089%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:28:04.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:04.107 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:04.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:04.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:04.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:04.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:04.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:04.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:04.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:04.368 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:04.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:04.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:04.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:04.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:04.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:04.640 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:04.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:04.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:04.742 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:04.769 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:04.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:04.873 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:04.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:04.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:05.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:05.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:05.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:05.181 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:05.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:05.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:05.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:05.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:05.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:05.443 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:05.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:05.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:05.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:05.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:05.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:05.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:05.732 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:28:05.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:05.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:05.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:05.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:05.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:05.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:05 smithi171 ceph-mon[90031]: pgmap v995: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 989 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19958/47325 objects degraded (42.172%); 5332/47325 objects misplaced (11.267%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:28:06.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4066664263' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:06.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:05 smithi027 ceph-mon[93921]: pgmap v995: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 989 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19958/47325 objects degraded (42.172%); 5332/47325 objects misplaced (11.267%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:28:06.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4066664263' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:06.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:05 smithi027 ceph-mon[98970]: pgmap v995: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 989 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 19958/47325 objects degraded (42.172%); 5332/47325 objects misplaced (11.267%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:28:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4066664263' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:06.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:06.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:06.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:06.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:06.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:06.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:06.284 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:28:06.285 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:28:06.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:06.350 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:06.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:06.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:06.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:06.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:06.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:06.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:06.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:06.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:06.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:06.783 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:06.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:06.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:06.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:06 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:06.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:06 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:28:06.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:06 smithi027 ceph-mon[98970]: osdmap e285: 8 total, 8 up, 8 in 2024-08-29T15:28:06.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:06 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:06.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:06 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:28:06.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:06 smithi027 ceph-mon[93921]: osdmap e285: 8 total, 8 up, 8 in 2024-08-29T15:28:06.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:06.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:06.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:06.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:07.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:06 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:06 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:28:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:06 smithi171 ceph-mon[90031]: osdmap e285: 8 total, 8 up, 8 in 2024-08-29T15:28:07.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:07.145 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:07.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:07.215 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:28:07.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:07.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:07.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:07.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:07.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:07.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:07.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:07.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:07.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:07.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:07.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:07.858 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:07.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:07.982 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:07.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:07 smithi171 ceph-mon[90031]: pgmap v997: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 868 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 19250/41541 objects degraded (46.340%); 5065/41541 objects misplaced (12.193%); 6.7 MiB/s, 106 objects/s recovering 2024-08-29T15:28:08.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 19250/41541 objects degraded (46.340%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:08.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:07 smithi027 ceph-mon[98970]: pgmap v997: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 868 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 19250/41541 objects degraded (46.340%); 5065/41541 objects misplaced (12.193%); 6.7 MiB/s, 106 objects/s recovering 2024-08-29T15:28:08.046 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 19250/41541 objects degraded (46.340%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:08.047 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:07 smithi027 ceph-mon[93921]: pgmap v997: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 868 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 19250/41541 objects degraded (46.340%); 5065/41541 objects misplaced (12.193%); 6.7 MiB/s, 106 objects/s recovering 2024-08-29T15:28:08.047 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 19250/41541 objects degraded (46.340%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:08.059 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:08.085 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:08.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:08.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:08.224 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:08.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:08.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:08.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:08.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:08.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:08.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:08.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:08.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:08.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:08.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:08.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:28:08.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:08.632 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:08.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:08.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:08.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:08.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:08.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:08.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:08.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:08.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:08.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:08.991 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:28:08.991 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:28:08.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:09.020 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:09.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:09.024 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:28:09.025 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:28:09.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:28:09.056 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:28:09.056 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:28:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:28:09] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.51.0" 2024-08-29T15:28:09.091 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:28:09.092 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:28:09.120 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:28:09.120 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:28:09.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:09.147 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:28:09.148 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:28:09.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:09.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:09.171 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:28:09.171 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:28:09.192 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:28:09.192 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:28:09.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:09.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:09.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:09.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:09.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:09.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:09.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:09.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:09.548 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:09.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:09.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:09.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:09.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:09.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:09.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:09 smithi171 ceph-mon[90031]: pgmap v998: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 868 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 19250/41541 objects degraded (46.340%); 5065/41541 objects misplaced (12.193%); 6.6 MiB/s, 105 objects/s recovering 2024-08-29T15:28:09.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1007544358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:09.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:09.864 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:09.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:09.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:09.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:09 smithi027 ceph-mon[93921]: pgmap v998: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 868 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 19250/41541 objects degraded (46.340%); 5065/41541 objects misplaced (12.193%); 6.6 MiB/s, 105 objects/s recovering 2024-08-29T15:28:09.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1007544358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:09 smithi027 ceph-mon[98970]: pgmap v998: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 868 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 19250/41541 objects degraded (46.340%); 5065/41541 objects misplaced (12.193%); 6.6 MiB/s, 105 objects/s recovering 2024-08-29T15:28:09.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1007544358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:10.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:10.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:10.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:10.136 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:10.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:10.141 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:28:10.142 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:28:10.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:10.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:10.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:10.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:10.351 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:28:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:28:10.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:143590 6.3 6:c37afdd3:::benchmark_data_smithi027_169445_object18163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:21.634601+0000", 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 588.71369370900004, 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7060110659999999, 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.352 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 143590 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.634601+0000", 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.634601+0000", 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.634600+0000", 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.634611+0000", 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1737e-05 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.634631+0000", 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9854e-05 2024-08-29T15:28:10.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.634642+0000", 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0883999999999999e-05 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.755174+0000", 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12053190900000001 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.775395+0000", 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020220582000000001 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.930455+0000", 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.155060374 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.949133+0000", 2024-08-29T15:28:10.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01867829 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.204288+0000", 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2551548129999999 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.216036+0000", 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011747868 2024-08-29T15:28:10.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.233371+0000", 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017335494999999999 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.310204+0000", 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.076833061999999994 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.311296+0000", 2024-08-29T15:28:10.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0010919689999999999 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.314171+0000", 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028743359999999999 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.318217+0000", 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040461200000000003 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.340613+0000", 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022395703999999999 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:143614 6.3 6:c2994735:::benchmark_data_smithi027_169445_object18187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:21.768108+0000", 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 588.58018760499999, 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6250681179999997, 2024-08-29T15:28:10.360 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.361 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.361 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.361 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.361 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.361 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 143614 2024-08-29T15:28:10.361 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.361 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.768108+0000", 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.768108+0000", 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.768105+0000", 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.768120+0000", 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4423999999999999e-05 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.768140+0000", 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0835999999999998e-05 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:21.768153+0000", 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2999999999999999e-05 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.008767+0000", 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.24061329200000001 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.033316+0000", 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024549557 2024-08-29T15:28:10.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.215414+0000", 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.18209740099999999 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.237471+0000", 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022057388000000001 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.269269+0000", 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0317978019999998 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.276976+0000", 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0077074769999999999 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.294471+0000", 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017494849 2024-08-29T15:28:10.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.345383+0000", 2024-08-29T15:28:10.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.050912012999999999 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.347073+0000", 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016901920000000001 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.352826+0000", 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0057526069999999999 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.360266+0000", 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0074398019999999997 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.393176+0000", 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032909968999999997 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:143668 6.3 6:c00d0f3d:::benchmark_data_smithi027_169445_object18241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:22.465900+0000", 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 587.88239501199996, 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6179618959999997, 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.370 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 143668 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.465900+0000", 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.465900+0000", 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.465897+0000", 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.465910+0000", 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2789000000000001e-05 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.465930+0000", 2024-08-29T15:28:10.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9741e-05 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.465942+0000", 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2359e-05 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.551692+0000", 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085749561000000002 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.570164+0000", 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018472195 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.728667+0000", 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.158502797 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:22.752095+0000", 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023428326999999999 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.911108+0000", 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1590135469999998 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.919294+0000", 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081854979999999994 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:25.952422+0000", 2024-08-29T15:28:10.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033128195999999999 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:26.033299+0000", 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080876992999999994 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:26.035982+0000", 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002682631 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:26.041689+0000", 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0057075019999999997 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:26.047087+0000", 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005397705 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:26.083862+0000", 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036775126999999998 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144040 6.3 6:caf0f445:::benchmark_data_smithi027_169445_object18613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:27.130012+0000", 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 583.21828367900002, 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6265766460000002, 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.378 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144040 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.130012+0000", 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.130012+0000", 2024-08-29T15:28:10.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.130009+0000", 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.130019+0000", 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8539999999999992e-06 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.130035+0000", 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5855e-05 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.130044+0000", 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3500000000000003e-06 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.298149+0000", 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.168105058 2024-08-29T15:28:10.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.309066+0000", 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010916155 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.472375+0000", 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16330984700000001 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:27.482975+0000", 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010599374999999999 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.519652+0000", 2024-08-29T15:28:10.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0366775439999998 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.537042+0000", 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017389729 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.572106+0000", 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035063679 2024-08-29T15:28:10.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.688917+0000", 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.116811682 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.690840+0000", 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001922397 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.701172+0000", 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01033209 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.712226+0000", 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011054527 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.756588+0000", 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.044361711999999998 2024-08-29T15:28:10.386 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144143 6.3 6:cd31e93a:::benchmark_data_smithi027_169445_object18716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:29.407540+0000", 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 580.94075548800004, 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 25.068442658999999, 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.387 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144143 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.407540+0000", 2024-08-29T15:28:10.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.407540+0000", 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.407538+0000", 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.407547+0000", 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8119999999999997e-06 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.407558+0000", 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1578e-05 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.407568+0000", 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2399999999999996e-06 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.538859+0000", 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13129179699999999 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.566824+0000", 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027964982999999999 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.677033+0000", 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.110208162 2024-08-29T15:28:10.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.691975+0000", 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014942535 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:32.882768+0000", 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1907925499999998 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:32.893144+0000", 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010375865999999999 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:32.914094+0000", 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020950637000000001 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.039142+0000", 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12504760200000001 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196059+0000", 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15691672300000001 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196072+0000", 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3042e-05 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.977812+0000", 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.781740455 2024-08-29T15:28:10.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.977824+0000", 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2196e-05 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.349761+0000", 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 20.371936582 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for readable", 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.365440+0000", 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015679088000000001 2024-08-29T15:28:10.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.389217+0000", 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.02377686 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.399564+0000", 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010347090999999999 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.470267+0000", 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.070703496000000005 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.475982+0000", 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0057150209999999998 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144182 6.3 6:c7e1ab88:::benchmark_data_smithi027_169445_object18755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:29.654997+0000", 2024-08-29T15:28:10.397 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 580.693297992, 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 24.884676120000002, 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144182 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.654997+0000", 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.654997+0000", 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.654995+0000", 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.655004+0000", 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3340000000000002e-06 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.655018+0000", 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4575e-05 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.655025+0000", 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5339999999999996e-06 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.693853+0000", 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038828062000000003 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.708059+0000", 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014206007 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.836760+0000", 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12870073100000001 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.848574+0000", 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011814655 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:32.955192+0000", 2024-08-29T15:28:10.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1066176830000001 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:32.968060+0000", 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012867939 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.014783+0000", 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046723566000000001 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196101+0000", 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.18131773100000001 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196106+0000", 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0420000000000002e-06 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978599+0000", 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78249245499999998 2024-08-29T15:28:10.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978605+0000", 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1e-06 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.416473+0000", 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 20.43786836 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.428136+0000", 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011662830000000001 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.533924+0000", 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.105788192 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.539673+0000", 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0057491779999999998 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144196 6.3 6:ce3849f8:::benchmark_data_smithi027_169445_object18769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:29.717050+0000", 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 580.63124499900005, 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 25.169076531000002, 2024-08-29T15:28:10.407 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144196 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.717050+0000", 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.717050+0000", 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.717048+0000", 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.717055+0000", 2024-08-29T15:28:10.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6120000000000004e-06 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.717066+0000", 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0720000000000001e-05 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.717074+0000", 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.32e-06 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.889969+0000", 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17289512300000001 2024-08-29T15:28:10.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.901803+0000", 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01183413 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.015763+0000", 2024-08-29T15:28:10.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11395928399999999 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.025541+0000", 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0097788519999999993 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196234+0000", 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1706922240000002 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196238+0000", 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5639999999999998e-06 2024-08-29T15:28:10.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978721+0000", 2024-08-29T15:28:10.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78248258500000001 2024-08-29T15:28:10.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:10.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978725+0000", 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6e-06 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.777578+0000", 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 20.798852195999999 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.787580+0000", 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010002063 2024-08-29T15:28:10.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.802549+0000", 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014969552000000001 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.845609+0000", 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.043059442000000003 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.848326+0000", 2024-08-29T15:28:10.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027169830000000001 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.851024+0000", 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002698083 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.856046+0000", 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050226009999999998 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.886127+0000", 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030080407999999999 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144197 6.3 6:c2292e03:::benchmark_data_smithi027_169445_object18770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:29.862321+0000", 2024-08-29T15:28:10.423 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 580.48597375700001, 2024-08-29T15:28:10.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 25.859279956000002, 2024-08-29T15:28:10.424 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.424 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.424 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.424 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.424 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.424 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144197 2024-08-29T15:28:10.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.425 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.862321+0000", 2024-08-29T15:28:10.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.862321+0000", 2024-08-29T15:28:10.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.862319+0000", 2024-08-29T15:28:10.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:28:10.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.862332+0000", 2024-08-29T15:28:10.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2489e-05 2024-08-29T15:28:10.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.862350+0000", 2024-08-29T15:28:10.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8502e-05 2024-08-29T15:28:10.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.862363+0000", 2024-08-29T15:28:10.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.273e-05 2024-08-29T15:28:10.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.903063+0000", 2024-08-29T15:28:10.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040700446000000001 2024-08-29T15:28:10.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:29.912779+0000", 2024-08-29T15:28:10.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.009716255 2024-08-29T15:28:10.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.069764+0000", 2024-08-29T15:28:10.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15698461599999999 2024-08-29T15:28:10.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.080663+0000", 2024-08-29T15:28:10.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010898613 2024-08-29T15:28:10.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196503+0000", 2024-08-29T15:28:10.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1158407979999998 2024-08-29T15:28:10.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196510+0000", 2024-08-29T15:28:10.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3690000000000003e-06 2024-08-29T15:28:10.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978981+0000", 2024-08-29T15:28:10.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78247117799999999 2024-08-29T15:28:10.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.437 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978985+0000", 2024-08-29T15:28:10.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2710000000000003e-06 2024-08-29T15:28:10.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.554347+0000", 2024-08-29T15:28:10.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 21.575361866000001 2024-08-29T15:28:10.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.564190+0000", 2024-08-29T15:28:10.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0098430570000000005 2024-08-29T15:28:10.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.579066+0000", 2024-08-29T15:28:10.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014875423 2024-08-29T15:28:10.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.595464+0000", 2024-08-29T15:28:10.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016398630000000001 2024-08-29T15:28:10.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.597994+0000", 2024-08-29T15:28:10.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025296609999999999 2024-08-29T15:28:10.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.695324+0000", 2024-08-29T15:28:10.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.097329845999999998 2024-08-29T15:28:10.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.700475+0000", 2024-08-29T15:28:10.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051508780000000002 2024-08-29T15:28:10.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.721601+0000", 2024-08-29T15:28:10.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021126700000000002 2024-08-29T15:28:10.445 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.445 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.445 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.446 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144229 6.3 6:c468c891:::benchmark_data_smithi027_169445_object18802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.446 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:30.390272+0000", 2024-08-29T15:28:10.446 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 579.95802328699995, 2024-08-29T15:28:10.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 25.304175014999998, 2024-08-29T15:28:10.447 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.447 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.447 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.447 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.447 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.448 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144229 2024-08-29T15:28:10.448 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.448 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.448 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.390272+0000", 2024-08-29T15:28:10.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.390272+0000", 2024-08-29T15:28:10.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.390270+0000", 2024-08-29T15:28:10.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.390278+0000", 2024-08-29T15:28:10.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3990000000000004e-06 2024-08-29T15:28:10.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.390291+0000", 2024-08-29T15:28:10.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3175e-05 2024-08-29T15:28:10.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.390298+0000", 2024-08-29T15:28:10.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5139999999999998e-06 2024-08-29T15:28:10.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.448863+0000", 2024-08-29T15:28:10.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.058565327 2024-08-29T15:28:10.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.455 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.474798+0000", 2024-08-29T15:28:10.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025935078 2024-08-29T15:28:10.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.663151+0000", 2024-08-29T15:28:10.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.188352876 2024-08-29T15:28:10.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.682612+0000", 2024-08-29T15:28:10.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019460425999999999 2024-08-29T15:28:10.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196475+0000", 2024-08-29T15:28:10.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5138633070000003 2024-08-29T15:28:10.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196479+0000", 2024-08-29T15:28:10.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3069999999999996e-06 2024-08-29T15:28:10.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978962+0000", 2024-08-29T15:28:10.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78248280400000003 2024-08-29T15:28:10.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978966+0000", 2024-08-29T15:28:10.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.33e-06 2024-08-29T15:28:10.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.498967+0000", 2024-08-29T15:28:10.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 21.52000069 2024-08-29T15:28:10.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.519092+0000", 2024-08-29T15:28:10.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020125290000000001 2024-08-29T15:28:10.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.463 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.463 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.543823+0000", 2024-08-29T15:28:10.464 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024730413999999999 2024-08-29T15:28:10.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.590537+0000", 2024-08-29T15:28:10.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046714161999999997 2024-08-29T15:28:10.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.593041+0000", 2024-08-29T15:28:10.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025036310000000001 2024-08-29T15:28:10.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.466 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.466 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.466 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.670755+0000", 2024-08-29T15:28:10.466 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.077714138000000002 2024-08-29T15:28:10.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.674937+0000", 2024-08-29T15:28:10.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041821940000000002 2024-08-29T15:28:10.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.468 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.468 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.694447+0000", 2024-08-29T15:28:10.468 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019510071 2024-08-29T15:28:10.468 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.469 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.469 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.469 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.469 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144248 6.3 6:c730b05e:::benchmark_data_smithi027_169445_object18821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.469 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:30.628751+0000", 2024-08-29T15:28:10.469 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 579.71954438900002, 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 24.835219951999999, 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144248 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.628751+0000", 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.628751+0000", 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.628749+0000", 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.628756+0000", 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3939999999999998e-06 2024-08-29T15:28:10.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.628767+0000", 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0971e-05 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.628774+0000", 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6470000000000001e-06 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.759127+0000", 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13035318100000001 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.788492+0000", 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029365505 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.965642+0000", 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17714940300000001 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.980054+0000", 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014412545000000001 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196416+0000", 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.21636145 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196420+0000", 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4819999999999996e-06 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978909+0000", 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78248898600000005 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978914+0000", 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.177e-06 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.226062+0000", 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 21.247148255999999 2024-08-29T15:28:10.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.243685+0000", 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017622832000000001 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.265966+0000", 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022281817999999998 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.431796+0000", 2024-08-29T15:28:10.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16582921 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.434568+0000", 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027719289999999998 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.435417+0000", 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00084914699999999997 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.439525+0000", 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041086940000000004 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.463971+0000", 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024445346999999999 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144281 6.3 6:cebc35cc:::benchmark_data_smithi027_169445_object18854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.478 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:30.793118+0000", 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 579.55517765599996, 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 24.051488112000001, 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144281 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.793118+0000", 2024-08-29T15:28:10.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.793118+0000", 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.793115+0000", 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.793128+0000", 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2797e-05 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.793151+0000", 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3830000000000001e-05 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.793164+0000", 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2398999999999999e-05 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.036789+0000", 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.243625025 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.055726+0000", 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018937012999999999 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.215785+0000", 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16005873500000001 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.482 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.232528+0000", 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016743675 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196206+0000", 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.963677355 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196210+0000", 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8609999999999997e-06 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.483 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978685+0000", 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78247419100000004 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978689+0000", 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5199999999999999e-06 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.733505+0000", 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 20.754816266999999 2024-08-29T15:28:10.484 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.748831+0000", 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015325790000000001 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.767564+0000", 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018732764999999998 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.814114+0000", 2024-08-29T15:28:10.485 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046550151999999997 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.816585+0000", 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0024711540000000001 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.817469+0000", 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00088413999999999995 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.486 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.821604+0000", 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041341080000000001 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.844606+0000", 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023002091999999998 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144288 6.3 6:cc803573:::benchmark_data_smithi027_169445_object18861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:30.873282+0000", 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 579.47501293799996, 2024-08-29T15:28:10.487 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 24.198538055, 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144288 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.873282+0000", 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.488 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.873282+0000", 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.873280+0000", 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.873291+0000", 2024-08-29T15:28:10.489 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1531e-05 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.873313+0000", 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1792000000000001e-05 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:30.873325+0000", 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2011000000000001e-05 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.490 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.057858+0000", 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.18453298700000001 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.077410+0000", 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019551886000000001 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.291221+0000", 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.21381133499999999 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.491 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.302344+0000", 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011122359 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196288+0000", 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8939438 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196292+0000", 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3239999999999996e-06 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.492 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978776+0000", 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78248373199999999 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978780+0000", 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.515e-06 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.929810+0000", 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 20.951030160999998 2024-08-29T15:28:10.493 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.943820+0000", 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014009344 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.958551+0000", 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014730976999999999 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.975449+0000", 2024-08-29T15:28:10.494 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016898225999999999 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.977989+0000", 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025405750000000002 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.043846+0000", 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.065857010999999993 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.495 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.048074+0000", 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042276060000000001 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.071820+0000", 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023746242000000001 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144298 6.3 6:cd9b013f:::benchmark_data_smithi027_169445_object18871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:31.148113+0000", 2024-08-29T15:28:10.496 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 579.20018242399999, 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 23.894669874999998, 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144298 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.148113+0000", 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.148113+0000", 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.148111+0000", 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.148119+0000", 2024-08-29T15:28:10.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5709999999999998e-06 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.148133+0000", 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4005000000000001e-05 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.148141+0000", 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.452e-06 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.344377+0000", 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.196235777 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.360298+0000", 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015921185000000001 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.422792+0000", 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062493901999999997 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.437327+0000", 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014535019999999999 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196261+0000", 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7589338410000002 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196265+0000", 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3970000000000004e-06 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978749+0000", 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78248368800000001 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978753+0000", 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4050000000000002e-06 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.888108+0000", 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 20.909354474000001 2024-08-29T15:28:10.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.901581+0000", 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013473829999999999 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.916504+0000", 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014922749000000001 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.970391+0000", 2024-08-29T15:28:10.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.053886389 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.972898+0000", 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025075750000000002 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.018399+0000", 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.045500760000000001 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.023050+0000", 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046510609999999997 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:55.042783+0000", 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019732675000000002 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:144301 6.3 6:cf54418a:::benchmark_data_smithi027_169445_object18874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:18:31.318107+0000", 2024-08-29T15:28:10.505 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 579.03018775700002, 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 23.414141904000001, 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 144301 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.318107+0000", 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.318107+0000", 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.318104+0000", 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.318119+0000", 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4609e-05 2024-08-29T15:28:10.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.318141+0000", 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2362999999999999e-05 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.318154+0000", 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2206e-05 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.365753+0000", 2024-08-29T15:28:10.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.047599619000000003 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.377644+0000", 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011890421 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.486319+0000", 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.108675722 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for rw locks", 2024-08-29T15:28:10.509 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:31.501307+0000", 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014987423999999999 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196178+0000", 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6948711159999998 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.196183+0000", 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0799999999999996e-06 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978657+0000", 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.78247422700000002 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:33.978662+0000", 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4299999999999999e-06 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.608333+0000", 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 20.629671478999999 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.628596+0000", 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020263301000000001 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.649813+0000", 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021216727000000001 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.696005+0000", 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046192096000000002 2024-08-29T15:28:10.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.698715+0000", 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027094490000000001 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.701316+0000", 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026010920000000002 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.706604+0000", 2024-08-29T15:28:10.513 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0052881819999999998 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:18:54.732249+0000", 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025645333999999999 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:221963 6.3 6:c8074485:::benchmark_data_smithi027_169445_object96536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 45.107211278000001, 2024-08-29T15:28:10.514 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.875189047, 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221963 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.515 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241082+0000", 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241090+0000", 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6760000000000007e-06 2024-08-29T15:28:10.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241103+0000", 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3097e-05 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241110+0000", 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1189999999999999e-06 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.286106+0000", 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.044995852000000003 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.289616+0000", 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035105090000000002 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.307511+0000", 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017894131000000001 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:28:10.518 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.309413+0000", 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019022659999999999 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.986973+0000", 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.67756063300000002 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.989188+0000", 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022149930000000002 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.519 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.993558+0000", 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043696730000000001 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.022139+0000", 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028581273000000001 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.043843+0000", 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021703999000000002 2024-08-29T15:28:10.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.110257+0000", 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066413278000000006 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.111466+0000", 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001209578 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.116273+0000", 2024-08-29T15:28:10.521 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0048067459999999998 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:221967 6.3 6:c4ba2f51:::benchmark_data_smithi027_169445_object96540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 45.088582604000003, 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86481450699999995, 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.522 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221967 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259711+0000", 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259717+0000", 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4289999999999998e-06 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259726+0000", 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8389999999999996e-06 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.524 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259731+0000", 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3930000000000001e-06 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.289939+0000", 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.03020888 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.292731+0000", 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027917939999999998 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.344052+0000", 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051320328999999998 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.345748+0000", 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016965680000000001 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.009894+0000", 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.66414622700000003 2024-08-29T15:28:10.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.012206+0000", 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0023118190000000001 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.016501+0000", 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042943189999999996 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.076191+0000", 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.059690875999999997 2024-08-29T15:28:10.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.117656+0000", 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041464320999999998 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.118388+0000", 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00073274099999999997 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.528 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.119598+0000", 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012098269999999999 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.124527+0000", 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049288099999999996 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.529 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:222226 6.3 6:c82e9241:::benchmark_data_smithi027_169445_object96799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 44.055940716999999, 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86422102700000003, 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 222226 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.530 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292353+0000", 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292360+0000", 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4229999999999999e-06 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292373+0000", 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.291e-05 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292381+0000", 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1340000000000004e-06 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.308321+0000", 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015940122000000001 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.310974+0000", 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026525699999999999 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.338098+0000", 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027124519999999999 2024-08-29T15:28:10.533 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.339549+0000", 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001450724 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.061922+0000", 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.72237274799999995 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.064096+0000", 2024-08-29T15:28:10.534 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021742850000000002 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.068373+0000", 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042769779999999999 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.099354+0000", 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030980508 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.100191+0000", 2024-08-29T15:28:10.535 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00083792800000000005 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.149351+0000", 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049159714 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.150692+0000", 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013410850000000001 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.536 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.156575+0000", 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0058832499999999996 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:222756 6.3 6:c4cc6026:::benchmark_data_smithi027_169445_object97329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 42.552401508999999, 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86913315599999996, 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.537 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 222756 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795892+0000", 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795898+0000", 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4450000000000001e-06 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795911+0000", 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3353000000000001e-05 2024-08-29T15:28:10.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795918+0000", 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0160000000000003e-06 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.800436+0000", 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045175850000000002 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.801672+0000", 2024-08-29T15:28:10.540 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012355319999999999 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.890056+0000", 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088384024000000005 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.891521+0000", 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014655689999999999 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.541 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.630976+0000", 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.73945498399999998 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.633483+0000", 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025067330000000001 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.637874+0000", 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043910590000000001 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.542 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.657485+0000", 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019610737 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.658230+0000", 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00074571299999999995 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.658976+0000", 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00074594099999999997 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.660269+0000", 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012926579999999999 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.665027+0000", 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004757809 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:230075 6.3 6:c4c54f8a:::benchmark_data_smithi027_169445_object104648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 21.354523025999999, 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.89382495100000003, 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 230075 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.545 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993771+0000", 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993778+0000", 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8019999999999992e-06 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993792+0000", 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3959e-05 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993799+0000", 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0040000000000002e-06 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.547 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.231044+0000", 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.237244607 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.232235+0000", 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001191454 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.388426+0000", 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.156190521 2024-08-29T15:28:10.548 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.389841+0000", 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001415531 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.868908+0000", 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.479066673 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.870800+0000", 2024-08-29T15:28:10.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018920810000000001 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.874105+0000", 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033051000000000001 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.881899+0000", 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0077939180000000004 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.882530+0000", 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00063088800000000002 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.882723+0000", 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000193396 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.883672+0000", 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00094804899999999996 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.887597+0000", 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039255740000000003 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:231480 6.3 6:c1b858f2:::benchmark_data_smithi027_169445_object106053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.863209406999999, 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.92456820500000003, 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:10.552 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 231480 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:10.553 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485084+0000", 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485091+0000", 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485105+0000", 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3947999999999999e-05 2024-08-29T15:28:10.554 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485113+0000", 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5220000000000001e-06 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.563897+0000", 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078784891999999995 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.565127+0000", 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012290720000000001 2024-08-29T15:28:10.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.556 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.667308+0000", 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10218187500000001 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.668461+0000", 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001152882 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.377239+0000", 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.70877815099999997 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.379283+0000", 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002043703 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.383071+0000", 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037881690000000001 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.558 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.402937+0000", 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019865765000000001 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.403638+0000", 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000700682 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.403872+0000", 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00023457 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.405059+0000", 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00118629 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.409654+0000", 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045953820000000003 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:10.560 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:10.561 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:28:10.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:10.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:10.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:10.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:10.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:10.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:10.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:10.731 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:10.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:10 smithi027 ceph-mon[93921]: from='client.36171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:10.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:10.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:10.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:10.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:10.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:10.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:10.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:11.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:10 smithi171 ceph-mon[90031]: from='client.36171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:11.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:10 smithi027 ceph-mon[98970]: from='client.36171 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:11.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:11.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:11.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:11.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:11.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:11.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:11.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:11.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:11.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:11.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:11.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:11.450 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":998,"stamp":"2024-08-29T15:28:08.680850+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":910366717,"num_objects":13847,"num_object_clones":0,"num_object_copies":41541,"num_objects_missing_on_primary":1923,"num_objects_missing":1923,"num_objects_degraded":19250,"num_objects_misplaced":5065,"num_objects_unfound":0,"num_objects_dirty":13847,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":238154,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26362,"num_bytes_recovered":1606487245,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125390,"ondisk_log_size":125390,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5990720,"kb_used_data":3872184,"kb_used_omap":43,"kb_used_meta":2118420,"kb_avail":743970496,"statfs":{"total":767960285184,"available":761825787904,"internally_reserved":0,"allocated":3965116416,"data_stored":3951837742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2169262572},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-270467072,"num_objects":-4127,"num_object_clones":0,"num_object_copies":-12381,"num_objects_missing_on_primary":-251,"num_objects_missing":-251,"num_objects_degraded":-1307,"num_objects_misplaced":-493,"num_objects_unfound":0,"num_objects_dirty":-4127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4127,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1056,"num_bytes_recovered":69206016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004184"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":457,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.731006+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:28:05.731006+0000","last_peered":"2024-08-29T15:28:05.731006+0000","last_clean":"2024-08-29T15:28:05.731006+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:28:05.731006+0000","last_undegraded":"2024-08-29T15:28:05.731006+0000","last_fullsized":"2024-08-29T15:28:05.731006+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":63,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:42.743318+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:27:42.743318+0000","last_peered":"2024-08-29T15:27:42.743318+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:27:42.743318+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:27:42.743318+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":154,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.787244+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:27:56.787244+0000","last_peered":"2024-08-29T15:27:56.787244+0000","last_clean":"2024-08-29T15:27:56.787244+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:27:56.787244+0000","last_undegraded":"2024-08-29T15:27:56.787244+0000","last_fullsized":"2024-08-29T15:27:56.787244+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":420,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.802103+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:27:56.802103+0000","last_peered":"2024-08-29T15:27:56.802103+0000","last_clean":"2024-08-29T15:27:56.802103+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:27:56.802103+0000","last_undegraded":"2024-08-29T15:27:56.802103+0000","last_fullsized":"2024-08-29T15:27:56.802103+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":420,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.803736+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:27:56.803736+0000","last_peered":"2024-08-29T15:27:56.803736+0000","last_clean":"2024-08-29T15:27:56.803736+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:27:56.803736+0000","last_undegraded":"2024-08-29T15:27:56.803736+0000","last_fullsized":"2024-08-29T15:27:56.803736+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.742952+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:27:42.742952+0000","last_peered":"2024-08-29T15:27:42.742952+0000","last_clean":"2024-08-29T15:27:42.742952+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:27:42.742952+0000","last_undegraded":"2024-08-29T15:27:42.742952+0000","last_fullsized":"2024-08-29T15:27:42.742952+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":234,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728838+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:28:05.728838+0000","last_peered":"2024-08-29T15:28:05.728838+0000","last_clean":"2024-08-29T15:28:05.728838+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:28:05.728838+0000","last_undegraded":"2024-08-29T15:28:05.728838+0000","last_fullsized":"2024-08-29T15:28:05.728838+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":154,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753461+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:27:42.753461+0000","last_peered":"2024-08-29T15:27:42.753461+0000","last_clean":"2024-08-29T15:27:42.753461+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:27:42.753461+0000","last_undegraded":"2024-08-29T15:27:42.753461+0000","last_fullsized":"2024-08-29T15:27:42.753461+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"285'15024","reported_seq":23089,"reported_epoch":285,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:05.856688+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:28:05.856688+0000","last_peered":"2024-08-29T15:28:05.856688+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:28:05.856688+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"180'5000","ondisk_log_start":"180'5000","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10024,"log_dups_size":0,"ondisk_log_size":10024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58195968,"num_objects":888,"num_object_clones":0,"num_object_copies":2664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15024,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,5","objects":-2037}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":62,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:42.743260+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:27:42.743260+0000","last_peered":"2024-08-29T15:27:42.743260+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:27:42.743260+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:27:42.743260+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":172,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.786454+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:27:56.786454+0000","last_peered":"2024-08-29T15:27:56.786454+0000","last_clean":"2024-08-29T15:27:56.786454+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:27:56.786454+0000","last_undegraded":"2024-08-29T15:27:56.786454+0000","last_fullsized":"2024-08-29T15:27:56.786454+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"285'14688","reported_seq":14938,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:06.505888+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:28:06.505888+0000","last_peered":"2024-08-29T15:28:06.505888+0000","last_clean":"2024-08-29T15:28:06.505888+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:28:06.505888+0000","last_undegraded":"2024-08-29T15:28:06.505888+0000","last_fullsized":"2024-08-29T15:28:06.505888+0000","mapping_epoch":168,"log_start":"194'7928","ondisk_log_start":"194'7928","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6760,"log_dups_size":0,"ondisk_log_size":6760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14688,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":62,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:42.744144+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:27:42.744144+0000","last_peered":"2024-08-29T15:27:42.744144+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:27:42.744144+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:27:42.744144+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.742848+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:27:42.742848+0000","last_peered":"2024-08-29T15:27:42.742848+0000","last_clean":"2024-08-29T15:27:42.742848+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:27:42.742848+0000","last_undegraded":"2024-08-29T15:27:42.742848+0000","last_fullsized":"2024-08-29T15:27:42.742848+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"285'14817","reported_seq":15037,"reported_epoch":285,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:06.475217+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:28:06.475217+0000","last_peered":"2024-08-29T15:28:06.475217+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:28:06.475217+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"194'8074","ondisk_log_start":"194'8074","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58654720,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14817,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1582}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":237,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.806515+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:27:56.806515+0000","last_peered":"2024-08-29T15:27:56.806515+0000","last_clean":"2024-08-29T15:27:56.806515+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:27:56.806515+0000","last_undegraded":"2024-08-29T15:27:56.806515+0000","last_fullsized":"2024-08-29T15:27:56.806515+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.797173+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:27:56.797173+0000","last_peered":"2024-08-29T15:27:56.797173+0000","last_clean":"2024-08-29T15:27:56.797173+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:27:56.797173+0000","last_undegraded":"2024-08-29T15:27:56.797173+0000","last_fullsized":"2024-08-29T15:27:56.797173+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"284'14880","reported_seq":20348,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:04.004072+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:28:04.004072+0000","last_peered":"2024-08-29T15:28:04.004072+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:28:04.004072+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"187'6762","ondisk_log_start":"187'6762","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8118,"log_dups_size":0,"ondisk_log_size":8118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58851328,"num_objects":898,"num_object_clones":0,"num_object_copies":2694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14880,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":898}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":172,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.787412+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:27:56.787412+0000","last_peered":"2024-08-29T15:27:56.787412+0000","last_clean":"2024-08-29T15:27:56.787412+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:27:56.787412+0000","last_undegraded":"2024-08-29T15:27:56.787412+0000","last_fullsized":"2024-08-29T15:27:56.787412+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":435,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.802771+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:27:56.802771+0000","last_peered":"2024-08-29T15:27:56.802771+0000","last_clean":"2024-08-29T15:27:56.802771+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:27:56.802771+0000","last_undegraded":"2024-08-29T15:27:56.802771+0000","last_fullsized":"2024-08-29T15:27:56.802771+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"284'14910","reported_seq":18228,"reported_epoch":284,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:04.843436+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:28:04.843436+0000","last_peered":"2024-08-29T15:28:04.843436+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:28:04.843436+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:28:04.843436+0000","mapping_epoch":234,"log_start":"187'6933","ondisk_log_start":"187'6933","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59244544,"num_objects":904,"num_object_clones":0,"num_object_copies":2712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2087,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14910,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1573,"num_bytes_recovered":103088128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":904}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":431,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727450+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:28:05.727450+0000","last_peered":"2024-08-29T15:28:05.727450+0000","last_clean":"2024-08-29T15:28:05.727450+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:28:05.727450+0000","last_undegraded":"2024-08-29T15:28:05.727450+0000","last_fullsized":"2024-08-29T15:28:05.727450+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.743789+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:27:42.743789+0000","last_peered":"2024-08-29T15:27:42.743789+0000","last_clean":"2024-08-29T15:27:42.743789+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:27:42.743789+0000","last_undegraded":"2024-08-29T15:27:42.743789+0000","last_fullsized":"2024-08-29T15:27:42.743789+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"285'14981","reported_seq":18456,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:06.512523+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:28:06.512523+0000","last_peered":"2024-08-29T15:28:06.512523+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:28:06.512523+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:28:06.512523+0000","mapping_epoch":234,"log_start":"194'8268","ondisk_log_start":"194'8268","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6713,"log_dups_size":0,"ondisk_log_size":6713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56426496,"num_objects":861,"num_object_clones":0,"num_object_copies":2583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14981,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1604,"num_bytes_recovered":105119744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":861}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.797039+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:27:56.797039+0000","last_peered":"2024-08-29T15:27:56.797039+0000","last_clean":"2024-08-29T15:27:56.797039+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:27:56.797039+0000","last_undegraded":"2024-08-29T15:27:56.797039+0000","last_fullsized":"2024-08-29T15:27:56.797039+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.744239+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:27:42.744239+0000","last_peered":"2024-08-29T15:27:42.744239+0000","last_clean":"2024-08-29T15:27:42.744239+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:27:42.744239+0000","last_undegraded":"2024-08-29T15:27:42.744239+0000","last_fullsized":"2024-08-29T15:27:42.744239+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"285'14799","reported_seq":15250,"reported_epoch":285,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:28:06.470257+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:28:06.470257+0000","last_peered":"2024-08-29T15:28:06.470257+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:28:06.470257+0000","last_undegraded":"2024-08-29T15:28:06.470257+0000","last_fullsized":"2024-08-29T15:28:06.470257+0000","mapping_epoch":202,"log_start":"194'8067","ondisk_log_start":"194'8067","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6732,"log_dups_size":0,"ondisk_log_size":6732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295424,"num_objects":859,"num_object_clones":0,"num_object_copies":2577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":859,"num_objects_unfound":0,"num_objects_dirty":859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14799,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":859}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.742874+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:27:42.742874+0000","last_peered":"2024-08-29T15:27:42.742874+0000","last_clean":"2024-08-29T15:27:42.742874+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:27:42.742874+0000","last_undegraded":"2024-08-29T15:27:42.742874+0000","last_fullsized":"2024-08-29T15:27:42.742874+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":270,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.744354+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:27:42.744354+0000","last_peered":"2024-08-29T15:27:42.744354+0000","last_clean":"2024-08-29T15:27:42.744354+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:27:42.744354+0000","last_undegraded":"2024-08-29T15:27:42.744354+0000","last_fullsized":"2024-08-29T15:27:42.744354+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"285'14900","reported_seq":15067,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:06.316873+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:28:06.316873+0000","last_peered":"2024-08-29T15:28:06.316873+0000","last_clean":"2024-08-29T15:28:06.316873+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:28:06.316873+0000","last_undegraded":"2024-08-29T15:28:06.316873+0000","last_fullsized":"2024-08-29T15:28:06.316873+0000","mapping_epoch":218,"log_start":"180'4878","ondisk_log_start":"180'4878","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295448,"num_objects":860,"num_object_clones":0,"num_object_copies":2580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":860,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14900,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":434,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.729956+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:28:05.729956+0000","last_peered":"2024-08-29T15:28:05.729956+0000","last_clean":"2024-08-29T15:28:05.729956+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:28:05.729956+0000","last_undegraded":"2024-08-29T15:28:05.729956+0000","last_fullsized":"2024-08-29T15:28:05.729956+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":427,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728947+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:28:05.728947+0000","last_peered":"2024-08-29T15:28:05.728947+0000","last_clean":"2024-08-29T15:28:05.728947+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:28:05.728947+0000","last_undegraded":"2024-08-29T15:28:05.728947+0000","last_fullsized":"2024-08-29T15:28:05.728947+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"285'14362","reported_seq":14577,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.859299+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:28:05.859299+0000","last_peered":"2024-08-29T15:28:05.859299+0000","last_clean":"2024-08-29T15:28:05.859299+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:28:05.859299+0000","last_undegraded":"2024-08-29T15:28:05.859299+0000","last_fullsized":"2024-08-29T15:28:05.859299+0000","mapping_epoch":201,"log_start":"180'4300","ondisk_log_start":"180'4300","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55443456,"num_objects":846,"num_object_clones":0,"num_object_copies":2538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14362,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":523,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.801517+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:27:56.801517+0000","last_peered":"2024-08-29T15:27:56.801517+0000","last_clean":"2024-08-29T15:27:56.801517+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:27:56.801517+0000","last_undegraded":"2024-08-29T15:27:56.801517+0000","last_fullsized":"2024-08-29T15:27:56.801517+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":757,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726827+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:28:05.726827+0000","last_peered":"2024-08-29T15:28:05.726827+0000","last_clean":"2024-08-29T15:28:05.726827+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:28:05.726827+0000","last_undegraded":"2024-08-29T15:28:05.726827+0000","last_fullsized":"2024-08-29T15:28:05.726827+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":265,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:28:01.220557+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:28:01.220557+0000","last_peered":"2024-08-29T15:28:01.220557+0000","last_clean":"2024-08-29T15:28:01.220557+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:28:01.220557+0000","last_undegraded":"2024-08-29T15:28:01.220557+0000","last_fullsized":"2024-08-29T15:28:01.220557+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":126,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753915+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:27:42.753915+0000","last_peered":"2024-08-29T15:27:42.753915+0000","last_clean":"2024-08-29T15:27:42.753915+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:27:42.753915+0000","last_undegraded":"2024-08-29T15:27:42.753915+0000","last_fullsized":"2024-08-29T15:27:42.753915+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"285'14826","reported_seq":15275,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728264+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:28:05.728264+0000","last_peered":"2024-08-29T15:28:05.728264+0000","last_clean":"2024-08-29T15:28:05.728264+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:28:05.728264+0000","last_undegraded":"2024-08-29T15:28:05.728264+0000","last_fullsized":"2024-08-29T15:28:05.728264+0000","mapping_epoch":201,"log_start":"188'7753","ondisk_log_start":"188'7753","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56098816,"num_objects":856,"num_object_clones":0,"num_object_copies":2568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14826,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":899,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727727+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:28:05.727727+0000","last_peered":"2024-08-29T15:28:05.727727+0000","last_clean":"2024-08-29T15:28:05.727727+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:28:05.727727+0000","last_undegraded":"2024-08-29T15:28:05.727727+0000","last_fullsized":"2024-08-29T15:28:05.727727+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":435,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.801998+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:27:56.801998+0000","last_peered":"2024-08-29T15:27:56.801998+0000","last_clean":"2024-08-29T15:27:56.801998+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:27:56.801998+0000","last_undegraded":"2024-08-29T15:27:56.801998+0000","last_fullsized":"2024-08-29T15:27:56.801998+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":443,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.803960+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:27:56.803960+0000","last_peered":"2024-08-29T15:27:56.803960+0000","last_clean":"2024-08-29T15:27:56.803960+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:27:56.803960+0000","last_undegraded":"2024-08-29T15:27:56.803960+0000","last_fullsized":"2024-08-29T15:27:56.803960+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"285'14836","reported_seq":27976,"reported_epoch":285,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:06.516843+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:28:06.516843+0000","last_peered":"2024-08-29T15:28:06.516843+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:28:06.516843+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"194'8130","ondisk_log_start":"194'8130","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55967744,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1331,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14836,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6460,"num_bytes_recovered":423362560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"3,6","objects":54},{"shards":"3,7","objects":800}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":708,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727427+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:28:05.727427+0000","last_peered":"2024-08-29T15:28:05.727427+0000","last_clean":"2024-08-29T15:28:05.727427+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:28:05.727427+0000","last_undegraded":"2024-08-29T15:28:05.727427+0000","last_fullsized":"2024-08-29T15:28:05.727427+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":116,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753344+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:27:42.753344+0000","last_peered":"2024-08-29T15:27:42.753344+0000","last_clean":"2024-08-29T15:27:42.753344+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:27:42.753344+0000","last_undegraded":"2024-08-29T15:27:42.753344+0000","last_fullsized":"2024-08-29T15:27:42.753344+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":472,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728695+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:28:05.728695+0000","last_peered":"2024-08-29T15:28:05.728695+0000","last_clean":"2024-08-29T15:28:05.728695+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:28:05.728695+0000","last_undegraded":"2024-08-29T15:28:05.728695+0000","last_fullsized":"2024-08-29T15:28:05.728695+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"285'14882","reported_seq":15142,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:06.424437+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:28:06.424437+0000","last_peered":"2024-08-29T15:28:06.424437+0000","last_clean":"2024-08-29T15:28:06.424437+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:28:06.424437+0000","last_undegraded":"2024-08-29T15:28:06.424437+0000","last_fullsized":"2024-08-29T15:28:06.424437+0000","mapping_epoch":234,"log_start":"194'8197","ondisk_log_start":"194'8197","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54394880,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14882,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":225,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:28:01.220519+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:28:01.220519+0000","last_peered":"2024-08-29T15:28:01.220519+0000","last_clean":"2024-08-29T15:28:01.220519+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:28:01.220519+0000","last_undegraded":"2024-08-29T15:28:01.220519+0000","last_fullsized":"2024-08-29T15:28:01.220519+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.743206+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:27:42.743206+0000","last_peered":"2024-08-29T15:27:42.743206+0000","last_clean":"2024-08-29T15:27:42.743206+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:27:42.743206+0000","last_undegraded":"2024-08-29T15:27:42.743206+0000","last_fullsized":"2024-08-29T15:27:42.743206+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":132,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726798+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:28:05.726798+0000","last_peered":"2024-08-29T15:28:05.726798+0000","last_clean":"2024-08-29T15:28:05.726798+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:28:05.726798+0000","last_undegraded":"2024-08-29T15:28:05.726798+0000","last_fullsized":"2024-08-29T15:28:05.726798+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"284'14930","reported_seq":15187,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:04.038601+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:28:04.038601+0000","last_peered":"2024-08-29T15:28:04.038601+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:28:04.038601+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"187'6742","ondisk_log_start":"187'6742","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8188,"log_dups_size":0,"ondisk_log_size":8188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58720256,"num_objects":896,"num_object_clones":0,"num_object_copies":2688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14930,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-1527}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":157,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726715+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:28:05.726715+0000","last_peered":"2024-08-29T15:28:05.726715+0000","last_clean":"2024-08-29T15:28:05.726715+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:28:05.726715+0000","last_undegraded":"2024-08-29T15:28:05.726715+0000","last_fullsized":"2024-08-29T15:28:05.726715+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":864,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728367+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:28:05.728367+0000","last_peered":"2024-08-29T15:28:05.728367+0000","last_clean":"2024-08-29T15:28:05.728367+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:28:05.728367+0000","last_undegraded":"2024-08-29T15:28:05.728367+0000","last_fullsized":"2024-08-29T15:28:05.728367+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":441,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727966+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:28:05.727966+0000","last_peered":"2024-08-29T15:28:05.727966+0000","last_clean":"2024-08-29T15:28:05.727966+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:28:05.727966+0000","last_undegraded":"2024-08-29T15:28:05.727966+0000","last_fullsized":"2024-08-29T15:28:05.727966+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"285'14864","reported_seq":15679,"reported_epoch":285,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:05.729470+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:28:05.729470+0000","last_peered":"2024-08-29T15:28:05.729470+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:28:05.729470+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"193'7880","ondisk_log_start":"193'7880","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52690944,"num_objects":804,"num_object_clones":0,"num_object_copies":2412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":799,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14864,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":4},{"shards":"1,7","objects":-1347}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":446,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728228+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:28:05.728228+0000","last_peered":"2024-08-29T15:28:05.728228+0000","last_clean":"2024-08-29T15:28:05.728228+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:28:05.728228+0000","last_undegraded":"2024-08-29T15:28:05.728228+0000","last_fullsized":"2024-08-29T15:28:05.728228+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":713,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:50.003381+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:27:50.003381+0000","last_peered":"2024-08-29T15:27:50.003381+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:27:50.003381+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:27:50.003381+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":435,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.802552+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:27:56.802552+0000","last_peered":"2024-08-29T15:27:56.802552+0000","last_clean":"2024-08-29T15:27:56.802552+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:27:56.802552+0000","last_undegraded":"2024-08-29T15:27:56.802552+0000","last_fullsized":"2024-08-29T15:27:56.802552+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"284'14594","reported_seq":21764,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:28:04.767794+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:28:04.767794+0000","last_peered":"2024-08-29T15:28:04.767794+0000","last_clean":"2024-08-29T15:28:04.767794+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:28:04.767794+0000","last_undegraded":"2024-08-29T15:28:04.767794+0000","last_fullsized":"2024-08-29T15:28:04.767794+0000","mapping_epoch":234,"log_start":"187'6699","ondisk_log_start":"187'6699","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7895,"log_dups_size":0,"ondisk_log_size":7895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54657024,"num_objects":834,"num_object_clones":0,"num_object_copies":2502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14594,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":431,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.730263+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:28:05.730263+0000","last_peered":"2024-08-29T15:28:05.730263+0000","last_clean":"2024-08-29T15:28:05.730263+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:28:05.730263+0000","last_undegraded":"2024-08-29T15:28:05.730263+0000","last_fullsized":"2024-08-29T15:28:05.730263+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1054,"reported_epoch":285,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:05.727877+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:28:05.727877+0000","last_peered":"2024-08-29T15:28:05.727877+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:28:05.727877+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":429,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.729790+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:28:05.729790+0000","last_peered":"2024-08-29T15:28:05.729790+0000","last_clean":"2024-08-29T15:28:05.729790+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:28:05.729790+0000","last_undegraded":"2024-08-29T15:28:05.729790+0000","last_fullsized":"2024-08-29T15:28:05.729790+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"284'14738","reported_seq":28040,"reported_epoch":284,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:04.881590+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:28:04.881590+0000","last_peered":"2024-08-29T15:28:04.881590+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:28:04.881590+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"188'7212","ondisk_log_start":"188'7212","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7526,"log_dups_size":0,"ondisk_log_size":7526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57147392,"num_objects":872,"num_object_clones":0,"num_object_copies":2616,"num_objects_missing_on_primary":1915,"num_objects_missing":1915,"num_objects_degraded":808,"num_objects_misplaced":871,"num_objects_unfound":0,"num_objects_dirty":872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14738,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5815,"num_bytes_recovered":380960768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":63},{"shards":"1,7","objects":808},{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":425,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.803115+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:27:56.803115+0000","last_peered":"2024-08-29T15:27:56.803115+0000","last_clean":"2024-08-29T15:27:56.803115+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:27:56.803115+0000","last_undegraded":"2024-08-29T15:27:56.803115+0000","last_fullsized":"2024-08-29T15:27:56.803115+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":137,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:49.822464+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:27:49.822464+0000","last_peered":"2024-08-29T15:27:49.822464+0000","last_clean":"2024-08-29T15:27:49.822464+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:27:49.822464+0000","last_undegraded":"2024-08-29T15:27:49.822464+0000","last_fullsized":"2024-08-29T15:27:49.822464+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.742967+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:27:42.742967+0000","last_peered":"2024-08-29T15:27:42.742967+0000","last_clean":"2024-08-29T15:27:42.742967+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:27:42.742967+0000","last_undegraded":"2024-08-29T15:27:42.742967+0000","last_fullsized":"2024-08-29T15:27:42.742967+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.743471+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:27:42.743471+0000","last_peered":"2024-08-29T15:27:42.743471+0000","last_clean":"2024-08-29T15:27:42.743471+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:27:42.743471+0000","last_undegraded":"2024-08-29T15:27:42.743471+0000","last_fullsized":"2024-08-29T15:27:42.743471+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":62,"reported_epoch":283,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:42.743902+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:27:42.743902+0000","last_peered":"2024-08-29T15:27:42.743902+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:27:42.743902+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:27:42.743902+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":435,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727801+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:28:05.727801+0000","last_peered":"2024-08-29T15:28:05.727801+0000","last_clean":"2024-08-29T15:28:05.727801+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:28:05.727801+0000","last_undegraded":"2024-08-29T15:28:05.727801+0000","last_fullsized":"2024-08-29T15:28:05.727801+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":186,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:05.731374+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:28:05.731374+0000","last_peered":"2024-08-29T15:28:05.731374+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:28:05.731374+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:28:05.731374+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":154,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.787137+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:27:56.787137+0000","last_peered":"2024-08-29T15:27:56.787137+0000","last_clean":"2024-08-29T15:27:56.787137+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:27:56.787137+0000","last_undegraded":"2024-08-29T15:27:56.787137+0000","last_fullsized":"2024-08-29T15:27:56.787137+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":395,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726589+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:28:05.726589+0000","last_peered":"2024-08-29T15:28:05.726589+0000","last_clean":"2024-08-29T15:28:05.726589+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:28:05.726589+0000","last_undegraded":"2024-08-29T15:28:05.726589+0000","last_fullsized":"2024-08-29T15:28:05.726589+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":446,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.803402+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:27:56.803402+0000","last_peered":"2024-08-29T15:27:56.803402+0000","last_clean":"2024-08-29T15:27:56.803402+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:27:56.803402+0000","last_undegraded":"2024-08-29T15:27:56.803402+0000","last_fullsized":"2024-08-29T15:27:56.803402+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":447,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.729770+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:28:05.729770+0000","last_peered":"2024-08-29T15:28:05.729770+0000","last_clean":"2024-08-29T15:28:05.729770+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:28:05.729770+0000","last_undegraded":"2024-08-29T15:28:05.729770+0000","last_fullsized":"2024-08-29T15:28:05.729770+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":125,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.753889+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:27:42.753889+0000","last_peered":"2024-08-29T15:27:42.753889+0000","last_clean":"2024-08-29T15:27:42.753889+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:27:42.753889+0000","last_undegraded":"2024-08-29T15:27:42.753889+0000","last_fullsized":"2024-08-29T15:27:42.753889+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":301,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.801408+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:27:56.801408+0000","last_peered":"2024-08-29T15:27:56.801408+0000","last_clean":"2024-08-29T15:27:56.801408+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:27:56.801408+0000","last_undegraded":"2024-08-29T15:27:56.801408+0000","last_fullsized":"2024-08-29T15:27:56.801408+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":422,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.803884+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:27:56.803884+0000","last_peered":"2024-08-29T15:27:56.803884+0000","last_clean":"2024-08-29T15:27:56.803884+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:27:56.803884+0000","last_undegraded":"2024-08-29T15:27:56.803884+0000","last_fullsized":"2024-08-29T15:27:56.803884+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":427,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.729966+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:28:05.729966+0000","last_peered":"2024-08-29T15:28:05.729966+0000","last_clean":"2024-08-29T15:28:05.729966+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:28:05.729966+0000","last_undegraded":"2024-08-29T15:28:05.729966+0000","last_fullsized":"2024-08-29T15:28:05.729966+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":431,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727306+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:28:05.727306+0000","last_peered":"2024-08-29T15:28:05.727306+0000","last_clean":"2024-08-29T15:28:05.727306+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:28:05.727306+0000","last_undegraded":"2024-08-29T15:28:05.727306+0000","last_fullsized":"2024-08-29T15:28:05.727306+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.742918+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:27:42.742918+0000","last_peered":"2024-08-29T15:27:42.742918+0000","last_clean":"2024-08-29T15:27:42.742918+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:27:42.742918+0000","last_undegraded":"2024-08-29T15:27:42.742918+0000","last_fullsized":"2024-08-29T15:27:42.742918+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":451,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728333+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:28:05.728333+0000","last_peered":"2024-08-29T15:28:05.728333+0000","last_clean":"2024-08-29T15:28:05.728333+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:28:05.728333+0000","last_undegraded":"2024-08-29T15:28:05.728333+0000","last_fullsized":"2024-08-29T15:28:05.728333+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":413,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.802487+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:27:56.802487+0000","last_peered":"2024-08-29T15:27:56.802487+0000","last_clean":"2024-08-29T15:27:56.802487+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:27:56.802487+0000","last_undegraded":"2024-08-29T15:27:56.802487+0000","last_fullsized":"2024-08-29T15:27:56.802487+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.743127+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:27:42.743127+0000","last_peered":"2024-08-29T15:27:42.743127+0000","last_clean":"2024-08-29T15:27:42.743127+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:27:42.743127+0000","last_undegraded":"2024-08-29T15:27:42.743127+0000","last_fullsized":"2024-08-29T15:27:42.743127+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":178,"reported_epoch":284,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:27:56.786827+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:27:56.786827+0000","last_peered":"2024-08-29T15:27:56.786827+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:27:56.786827+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:27:56.786827+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.743002+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:27:42.743002+0000","last_peered":"2024-08-29T15:27:42.743002+0000","last_clean":"2024-08-29T15:27:42.743002+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:27:42.743002+0000","last_undegraded":"2024-08-29T15:27:42.743002+0000","last_fullsized":"2024-08-29T15:27:42.743002+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":184,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.787123+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:27:56.787123+0000","last_peered":"2024-08-29T15:27:56.787123+0000","last_clean":"2024-08-29T15:27:56.787123+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:27:56.787123+0000","last_undegraded":"2024-08-29T15:27:56.787123+0000","last_fullsized":"2024-08-29T15:27:56.787123+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":462,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726924+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:28:05.726924+0000","last_peered":"2024-08-29T15:28:05.726924+0000","last_clean":"2024-08-29T15:28:05.726924+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:28:05.726924+0000","last_undegraded":"2024-08-29T15:28:05.726924+0000","last_fullsized":"2024-08-29T15:28:05.726924+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":464,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728117+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:28:05.728117+0000","last_peered":"2024-08-29T15:28:05.728117+0000","last_clean":"2024-08-29T15:28:05.728117+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:28:05.728117+0000","last_undegraded":"2024-08-29T15:28:05.728117+0000","last_fullsized":"2024-08-29T15:28:05.728117+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":64,"reported_epoch":283,"state":"active+clean","last_fresh":"2024-08-29T15:27:42.742757+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:27:42.742757+0000","last_peered":"2024-08-29T15:27:42.742757+0000","last_clean":"2024-08-29T15:27:42.742757+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:27:42.742757+0000","last_undegraded":"2024-08-29T15:27:42.742757+0000","last_fullsized":"2024-08-29T15:27:42.742757+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":469,"reported_epoch":284,"state":"active+clean","last_fresh":"2024-08-29T15:27:56.803310+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:27:56.803310+0000","last_peered":"2024-08-29T15:27:56.803310+0000","last_clean":"2024-08-29T15:27:56.803310+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:27:56.803310+0000","last_undegraded":"2024-08-29T15:27:56.803310+0000","last_fullsized":"2024-08-29T15:27:56.803310+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":905576472,"num_objects":13819,"num_object_clones":0,"num_object_copies":41457,"num_objects_missing_on_primary":1915,"num_objects_missing":1915,"num_objects_degraded":19221,"num_objects_misplaced":5065,"num_objects_unfound":0,"num_objects_dirty":13819,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":237031,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26253,"num_bytes_recovered":1600520192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3898290176,"data_stored":3898277960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124208,"ondisk_log_size":124208,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234401,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":815960,"kb_used_data":459984,"kb_used_omap":1,"kb_used_meta":355966,"kb_avail":92929192,"statfs":{"total":95995035648,"available":95159492608,"internally_reserved":0,"allocated":471023616,"data_stored":469355003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":364509643},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496032,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":232532,"kb_used_data":16844,"kb_used_omap":1,"kb_used_meta":215678,"kb_avail":93512620,"statfs":{"total":95995035648,"available":95756922880,"internally_reserved":0,"allocated":17248256,"data_stored":15583443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":220854727},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426619,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":361048,"kb_used_data":279696,"kb_used_omap":8,"kb_used_meta":81335,"kb_avail":93384104,"statfs":{"total":95995035648,"available":95625322496,"internally_reserved":0,"allocated":286408704,"data_stored":284743709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":83287946},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019311,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":639736,"kb_used_data":293616,"kb_used_omap":1,"kb_used_meta":346110,"kb_avail":93105416,"statfs":{"total":95995035648,"available":95339945984,"internally_reserved":0,"allocated":300662784,"data_stored":299021220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":354417094},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435496,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":870744,"kb_used_data":546188,"kb_used_omap":4,"kb_used_meta":324539,"kb_avail":92874408,"statfs":{"total":95995035648,"available":95103393792,"internally_reserved":0,"allocated":559296512,"data_stored":557635457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":332328201},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903348,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487708,"kb_used_data":1319888,"kb_used_omap":10,"kb_used_meta":167797,"kb_avail":92257444,"statfs":{"total":95995035648,"available":94471622656,"internally_reserved":0,"allocated":1351565312,"data_stored":1349904193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":171824656},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762549,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":992260,"kb_used_data":712060,"kb_used_omap":7,"kb_used_meta":280184,"kb_avail":92752892,"statfs":{"total":95995035648,"available":94978961408,"internally_reserved":0,"allocated":729149440,"data_stored":727494793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":286908770},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926075,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590732,"kb_used_data":243908,"kb_used_omap":7,"kb_used_meta":346808,"kb_avail":93154420,"statfs":{"total":95995035648,"available":95390126080,"internally_reserved":0,"allocated":249761792,"data_stored":248099924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":355131535},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":239407104,"data_stored":239403032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":715001856,"data_stored":714997784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1347158016,"data_stored":1347158016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":554500096,"data_stored":554500096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":286720000,"data_stored":286720000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":282202112,"data_stored":282198040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12451840,"data_stored":12451840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":460849152,"data_stored":460849152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:28:11.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:11.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:11.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:11.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:11.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:11.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:11.751 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:11.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:11.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:11.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:11.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:11.911 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:11.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:11.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:11.931 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:11 smithi027 ceph-mon[93921]: pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 827 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 19004/39558 objects degraded (48.041%); 4934/39558 objects misplaced (12.473%); 6.5 MiB/s, 103 objects/s recovering 2024-08-29T15:28:11.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:11 smithi027 ceph-mon[98970]: pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 827 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 19004/39558 objects degraded (48.041%); 4934/39558 objects misplaced (12.473%); 6.5 MiB/s, 103 objects/s recovering 2024-08-29T15:28:12.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:11 smithi171 ceph-mon[90031]: pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 827 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 19004/39558 objects degraded (48.041%); 4934/39558 objects misplaced (12.473%); 6.5 MiB/s, 103 objects/s recovering 2024-08-29T15:28:12.042 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:12.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:12.175 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:12.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:12.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:12.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:12.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:12.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:12.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:28:12.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:12.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:12.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:28:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:12.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:12.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:12.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:12.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:12.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:12.913 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 19004/39558 objects degraded (48.041%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:12.913 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 19004/39558 objects degraded (48.041%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:12.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:12.955 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:12.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:12.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:12.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:12.990 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 19004/39558 objects degraded (48.041%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:13.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:13.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:13.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:13.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:13.250 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:13.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:13.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:13.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:13.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:13.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:13.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:13.523 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:13.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:13.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:13.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:13.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:13.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:13.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:13.776 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:13.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:13.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:13 smithi027 ceph-mon[93921]: pgmap v1000: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 738 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 18590/35307 objects degraded (52.652%); 4801/35307 objects misplaced (13.598%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:28:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:13.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:13 smithi027 ceph-mon[98970]: pgmap v1000: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 738 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 18590/35307 objects degraded (52.652%); 4801/35307 objects misplaced (13.598%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:28:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:13.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:13.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:13.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:13.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:13.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:14.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:13 smithi171 ceph-mon[90031]: pgmap v1000: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 738 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 18590/35307 objects degraded (52.652%); 4801/35307 objects misplaced (13.598%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:28:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:14.046 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:14.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:14.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:14.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:14.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:14.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:14.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:14.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:14.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:14.423 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:14.450 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:14.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:14.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:14.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:28:14.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:14.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:14.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:14.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:14.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:14.771 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:14.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:14.784 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:28:14.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:14.912 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:14.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:15.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:15.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:15.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4261984609' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:14 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4261984609' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:15.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:14 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:15.087 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4261984609' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:15.088 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:14 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:15.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:15.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:15.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:15.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:15.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:15.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:15.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:15.470 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:15.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:15.572 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:15.601 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:15.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:15.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:15.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:15.870 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:28:15.870 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:28:15.870 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:28:15.870 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:28:15.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:15.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:15.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:15.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[98970]: pgmap v1001: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 722 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 18590/34530 objects degraded (53.837%); 4801/34530 objects misplaced (13.904%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:28:15.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:15.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:28:15.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[98970]: osdmap e286: 8 total, 8 up, 8 in 2024-08-29T15:28:15.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[93921]: pgmap v1001: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 722 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 18590/34530 objects degraded (53.837%); 4801/34530 objects misplaced (13.904%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:28:15.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:15.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:28:15.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:15 smithi027 ceph-mon[93921]: osdmap e286: 8 total, 8 up, 8 in 2024-08-29T15:28:15.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:15 smithi171 ceph-mon[90031]: pgmap v1001: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 722 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 18590/34530 objects degraded (53.837%); 4801/34530 objects misplaced (13.904%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:28:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:15 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:16.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:15 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:28:16.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:15 smithi171 ceph-mon[90031]: osdmap e286: 8 total, 8 up, 8 in 2024-08-29T15:28:16.023 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:16.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:16.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:16.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:16.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:16.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:16.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:16.401 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:16.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:16.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:16.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:16.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:16.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:16.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:28:16.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:16.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:16.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:16.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:16.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:16.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:16.989 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:16.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:16.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:17.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:17.128 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:17.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:17.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:17.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:17.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:17.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:17.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:17.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:17.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:17.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:17.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:17.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:17.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:17.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:17.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:17.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:17.773 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:17.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:17.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:17.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:17.930 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:17.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:17.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:17 smithi171 ceph-mon[90031]: pgmap v1003: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 574 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 17771/27441 objects degraded (64.761%); 4454/27441 objects misplaced (16.231%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:28:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:17 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:18.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 17771/27441 objects degraded (64.761%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:18.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[98970]: pgmap v1003: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 574 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 17771/27441 objects degraded (64.761%); 4454/27441 objects misplaced (16.231%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:28:18.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 17771/27441 objects degraded (64.761%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:18.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[93921]: pgmap v1003: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 574 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 17771/27441 objects degraded (64.761%); 4454/27441 objects misplaced (16.231%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:28:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:18.073 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 17771/27441 objects degraded (64.761%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:18.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:18.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:18.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:18.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:18.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:18.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:18.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:18.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:18.359 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:18.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:18.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:18.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:18.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:18.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:18.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:18.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:18.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:18.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:18.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:18.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:18.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:18.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:28:18.847 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:28:18.847 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:28:18.847 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:28:18.847 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:28:18.847 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:28:18.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:18.902 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:18.926 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:18.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:19.019 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:28:19.019 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:28:19.028 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:19.054 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:19.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:19.063 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:28:19.063 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:28:19.064 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:28:19] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.51.0" 2024-08-29T15:28:19.103 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:28:19.104 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:28:19.142 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:28:19.143 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:28:19.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:19.183 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:28:19.184 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:28:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:19.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:19.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:19.216 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:28:19.216 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:28:19.252 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:28:19.252 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:28:19.294 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:28:19.294 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:28:19.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:19.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:19.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:19.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:19.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:19.499 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:19.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:19.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:28:19.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:19.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:19.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:19.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:19.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:19 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:19.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:19 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:19.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:19 smithi171 ceph-mon[90031]: pgmap v1004: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 574 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 17771/27441 objects degraded (64.761%); 4454/27441 objects misplaced (16.231%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:28:19.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1310161845' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:19.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:19 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:28:19.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:19 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:28:19.770 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:19 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:19.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:19.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:19.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:19.796 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:19.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:19.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:19.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:20.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:20.052 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[98970]: pgmap v1004: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 574 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 17771/27441 objects degraded (64.761%); 4454/27441 objects misplaced (16.231%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1310161845' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[93921]: pgmap v1004: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 574 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 17771/27441 objects degraded (64.761%); 4454/27441 objects misplaced (16.231%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:28:20.053 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1310161845' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:20.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:28:20.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:28:20.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:19 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:20.070 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:20.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:20.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:20.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:20.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:20.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:20.355 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:20.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:20.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:20.487 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:20.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:20.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:20.589 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:20.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:20.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:20.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:20.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:20.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:20.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:20.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:20.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:21.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:21.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:21.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:21.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:21.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:21.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:21.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:21.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:21.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:21.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:21.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:21.385 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:21.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:21.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:21.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:21.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:21.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:21.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:21.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:21.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:21.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:21.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:21.802 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:21 smithi027 ceph-mon[98970]: pgmap v1005: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 521 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 17429/24885 objects degraded (70.038%); 4287/24885 objects misplaced (17.227%); 8.1 MiB/s, 130 objects/s recovering 2024-08-29T15:28:21.804 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:21 smithi027 ceph-mon[93921]: pgmap v1005: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 521 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 17429/24885 objects degraded (70.038%); 4287/24885 objects misplaced (17.227%); 8.1 MiB/s, 130 objects/s recovering 2024-08-29T15:28:21.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:21.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:21.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:21.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:21.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:22.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:22.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:21 smithi171 ceph-mon[90031]: pgmap v1005: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 521 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 17429/24885 objects degraded (70.038%); 4287/24885 objects misplaced (17.227%); 8.1 MiB/s, 130 objects/s recovering 2024-08-29T15:28:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:28:22.112 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:28:22.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:22.160 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:22.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:22.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:22.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:22.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:22.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:22.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:22.326 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:28:22.329 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:28:22.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:22.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:22.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:22.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:22.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:22.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:22.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:22.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:22.728 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:22.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:22.824 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:28:22.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:22.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:28:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:22.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:22.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:23.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 17429/24885 objects degraded (70.038%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:22 smithi171 ceph-mon[90031]: from='client.46112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:23.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:23.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:23.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:23.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 17429/24885 objects degraded (70.038%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:23.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:22 smithi027 ceph-mon[98970]: from='client.46112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:23.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 17429/24885 objects degraded (70.038%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:23.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:22 smithi027 ceph-mon[93921]: from='client.46112 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:23.135 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:23.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:23.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:23.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:23.191 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:23.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:23.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:23.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:23.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:23.316 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1005,"stamp":"2024-08-29T15:28:20.686872+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":546510845,"num_objects":8295,"num_object_clones":0,"num_object_copies":24885,"num_objects_missing_on_primary":1470,"num_objects_missing":1470,"num_objects_degraded":17429,"num_objects_misplaced":4287,"num_objects_unfound":0,"num_objects_dirty":8295,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":243706,"num_write_kb":8042178,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27911,"num_bytes_recovered":1708002509,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125442,"ondisk_log_size":125442,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5127164,"kb_used_data":2991480,"kb_used_omap":43,"kb_used_meta":2135572,"kb_avail":744834052,"statfs":{"total":767960285184,"available":762710069248,"internally_reserved":0,"allocated":3063275520,"data_stored":3049905070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2186826220},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,2,2,2,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-320536576,"num_objects":-4891,"num_object_clones":0,"num_object_copies":-14673,"num_objects_missing_on_primary":-324,"num_objects_missing":-324,"num_objects_degraded":-1575,"num_objects_misplaced":-647,"num_objects_unfound":0,"num_objects_dirty":-4891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4891,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1301,"num_bytes_recovered":85262336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005463"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":459,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.781964+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:28:14.781964+0000","last_peered":"2024-08-29T15:28:14.781964+0000","last_clean":"2024-08-29T15:28:14.781964+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:28:14.781964+0000","last_undegraded":"2024-08-29T15:28:14.781964+0000","last_fullsized":"2024-08-29T15:28:14.781964+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":65,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:05.735090+0000","last_change":"2024-08-29T15:19:46.803811+0000","last_active":"2024-08-29T15:28:05.735090+0000","last_peered":"2024-08-29T15:28:05.735090+0000","last_clean":"2024-08-29T14:55:35.629441+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:28:05.735090+0000","last_undegraded":"2024-08-29T15:19:46.792587+0000","last_fullsized":"2024-08-29T15:28:05.735090+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","5","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":155,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726981+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:28:05.726981+0000","last_peered":"2024-08-29T15:28:05.726981+0000","last_clean":"2024-08-29T15:28:05.726981+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:28:05.726981+0000","last_undegraded":"2024-08-29T15:28:05.726981+0000","last_fullsized":"2024-08-29T15:28:05.726981+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":425,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.798534+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:28:14.798534+0000","last_peered":"2024-08-29T15:28:14.798534+0000","last_clean":"2024-08-29T15:28:14.798534+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:28:14.798534+0000","last_undegraded":"2024-08-29T15:28:14.798534+0000","last_fullsized":"2024-08-29T15:28:14.798534+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":424,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.785866+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:28:14.785866+0000","last_peered":"2024-08-29T15:28:14.785866+0000","last_clean":"2024-08-29T15:28:14.785866+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:28:14.785866+0000","last_undegraded":"2024-08-29T15:28:14.785866+0000","last_fullsized":"2024-08-29T15:28:14.785866+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734758+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:28:05.734758+0000","last_peered":"2024-08-29T15:28:05.734758+0000","last_clean":"2024-08-29T15:28:05.734758+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:28:05.734758+0000","last_undegraded":"2024-08-29T15:28:05.734758+0000","last_fullsized":"2024-08-29T15:28:05.734758+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":236,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794598+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:28:14.794598+0000","last_peered":"2024-08-29T15:28:14.794598+0000","last_clean":"2024-08-29T15:28:14.794598+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:28:14.794598+0000","last_undegraded":"2024-08-29T15:28:14.794598+0000","last_fullsized":"2024-08-29T15:28:14.794598+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":156,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.733265+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:28:05.733265+0000","last_peered":"2024-08-29T15:28:05.733265+0000","last_clean":"2024-08-29T15:28:05.733265+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:28:05.733265+0000","last_undegraded":"2024-08-29T15:28:05.733265+0000","last_fullsized":"2024-08-29T15:28:05.733265+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"286'15346","reported_seq":23411,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:16.351090+0000","last_change":"2024-08-29T15:19:46.817203+0000","last_active":"2024-08-29T15:28:16.351090+0000","last_peered":"2024-08-29T15:28:16.351090+0000","last_clean":"2024-08-29T15:16:27.749465+0000","last_became_active":"2024-08-29T15:18:35.010305+0000","last_became_peered":"2024-08-29T15:18:35.010305+0000","last_unstale":"2024-08-29T15:28:16.351090+0000","last_undegraded":"2024-08-29T15:18:34.984701+0000","last_fullsized":"2024-08-29T15:18:34.970168+0000","mapping_epoch":218,"log_start":"180'5300","ondisk_log_start":"180'5300","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:58.767350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15346,"num_write_kb":509184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4069,"num_bytes_recovered":261488640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,5","objects":-2359}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":64,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:05.738907+0000","last_change":"2024-08-29T15:19:46.805655+0000","last_active":"2024-08-29T15:28:05.738907+0000","last_peered":"2024-08-29T15:28:05.738907+0000","last_clean":"2024-08-29T15:19:43.268572+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:28:05.738907+0000","last_undegraded":"2024-08-29T15:19:46.799985+0000","last_fullsized":"2024-08-29T15:28:05.738907+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":173,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726462+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:28:05.726462+0000","last_peered":"2024-08-29T15:28:05.726462+0000","last_clean":"2024-08-29T15:28:05.726462+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:28:05.726462+0000","last_undegraded":"2024-08-29T15:28:05.726462+0000","last_fullsized":"2024-08-29T15:28:05.726462+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"286'14996","reported_seq":15248,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:16.503589+0000","last_change":"2024-08-29T15:15:18.757107+0000","last_active":"2024-08-29T15:28:16.503589+0000","last_peered":"2024-08-29T15:28:16.503589+0000","last_clean":"2024-08-29T15:28:16.503589+0000","last_became_active":"2024-08-29T15:15:18.756566+0000","last_became_peered":"2024-08-29T15:15:18.756566+0000","last_unstale":"2024-08-29T15:28:16.503589+0000","last_undegraded":"2024-08-29T15:28:16.503589+0000","last_fullsized":"2024-08-29T15:28:16.503589+0000","mapping_epoch":168,"log_start":"195'8328","ondisk_log_start":"195'8328","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:55:27.324392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36306944,"num_objects":554,"num_object_clones":0,"num_object_copies":1662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14996,"num_write_kb":497600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":64,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:05.738343+0000","last_change":"2024-08-29T15:19:46.809782+0000","last_active":"2024-08-29T15:28:05.738343+0000","last_peered":"2024-08-29T15:28:05.738343+0000","last_clean":"2024-08-29T15:17:58.576245+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:28:05.738343+0000","last_undegraded":"2024-08-29T15:19:46.797004+0000","last_fullsized":"2024-08-29T15:28:05.738343+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734520+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:28:05.734520+0000","last_peered":"2024-08-29T15:28:05.734520+0000","last_clean":"2024-08-29T15:28:05.734520+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:28:05.734520+0000","last_undegraded":"2024-08-29T15:28:05.734520+0000","last_fullsized":"2024-08-29T15:28:05.734520+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"286'15110","reported_seq":15332,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:16.474296+0000","last_change":"2024-08-29T15:18:35.132212+0000","last_active":"2024-08-29T15:28:16.474296+0000","last_peered":"2024-08-29T15:28:16.474296+0000","last_clean":"2024-08-29T15:16:27.788234+0000","last_became_active":"2024-08-29T15:18:35.121364+0000","last_became_peered":"2024-08-29T15:18:35.121364+0000","last_unstale":"2024-08-29T15:28:16.474296+0000","last_undegraded":"2024-08-29T15:18:35.001998+0000","last_fullsized":"2024-08-29T15:18:34.970223+0000","mapping_epoch":218,"log_start":"195'8374","ondisk_log_start":"195'8374","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39452672,"num_objects":602,"num_object_clones":0,"num_object_copies":1806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15110,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1875}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":241,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.786004+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:28:14.786004+0000","last_peered":"2024-08-29T15:28:14.786004+0000","last_clean":"2024-08-29T15:28:14.786004+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:28:14.786004+0000","last_undegraded":"2024-08-29T15:28:14.786004+0000","last_fullsized":"2024-08-29T15:28:14.786004+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.733732+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:28:05.733732+0000","last_peered":"2024-08-29T15:28:05.733732+0000","last_clean":"2024-08-29T15:28:05.733732+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:28:05.733732+0000","last_undegraded":"2024-08-29T15:28:05.733732+0000","last_fullsized":"2024-08-29T15:28:05.733732+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"286'15321","reported_seq":20794,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:19.097084+0000","last_change":"2024-08-29T15:18:54.480853+0000","last_active":"2024-08-29T15:28:19.097084+0000","last_peered":"2024-08-29T15:28:19.097084+0000","last_clean":"2024-08-29T15:16:27.793624+0000","last_became_active":"2024-08-29T15:18:35.107150+0000","last_became_peered":"2024-08-29T15:18:35.107150+0000","last_unstale":"2024-08-29T15:28:19.097084+0000","last_undegraded":"2024-08-29T15:18:54.480622+0000","last_fullsized":"2024-08-29T15:18:34.975614+0000","mapping_epoch":218,"log_start":"187'7162","ondisk_log_start":"187'7162","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8159,"log_dups_size":0,"ondisk_log_size":8159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29949952,"num_objects":457,"num_object_clones":0,"num_object_copies":1371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15321,"num_write_kb":504896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":158662656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":457}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":173,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727625+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:28:05.727625+0000","last_peered":"2024-08-29T15:28:05.727625+0000","last_clean":"2024-08-29T15:28:05.727625+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:28:05.727625+0000","last_undegraded":"2024-08-29T15:28:05.727625+0000","last_fullsized":"2024-08-29T15:28:05.727625+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":439,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.785677+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:28:14.785677+0000","last_peered":"2024-08-29T15:28:14.785677+0000","last_clean":"2024-08-29T15:28:14.785677+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:28:14.785677+0000","last_undegraded":"2024-08-29T15:28:14.785677+0000","last_fullsized":"2024-08-29T15:28:14.785677+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"286'15350","reported_seq":19550,"reported_epoch":286,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:19.841674+0000","last_change":"2024-08-29T15:19:47.853949+0000","last_active":"2024-08-29T15:28:19.841674+0000","last_peered":"2024-08-29T15:28:19.841674+0000","last_clean":"2024-08-29T15:16:27.779584+0000","last_became_active":"2024-08-29T15:19:47.803743+0000","last_became_peered":"2024-08-29T15:19:47.803743+0000","last_unstale":"2024-08-29T15:28:19.841674+0000","last_undegraded":"2024-08-29T15:19:47.796574+0000","last_fullsized":"2024-08-29T15:28:19.841674+0000","mapping_epoch":234,"log_start":"188'7433","ondisk_log_start":"188'7433","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7917,"log_dups_size":0,"ondisk_log_size":7917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:26:23.866257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30408704,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1647,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15350,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2013,"num_bytes_recovered":131923968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4","objects":464}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":433,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.784665+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:28:14.784665+0000","last_peered":"2024-08-29T15:28:14.784665+0000","last_clean":"2024-08-29T15:28:14.784665+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:28:14.784665+0000","last_undegraded":"2024-08-29T15:28:14.784665+0000","last_fullsized":"2024-08-29T15:28:14.784665+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734459+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:28:05.734459+0000","last_peered":"2024-08-29T15:28:05.734459+0000","last_clean":"2024-08-29T15:28:05.734459+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:28:05.734459+0000","last_undegraded":"2024-08-29T15:28:05.734459+0000","last_fullsized":"2024-08-29T15:28:05.734459+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"286'15302","reported_seq":19421,"reported_epoch":286,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:16.510252+0000","last_change":"2024-08-29T15:19:47.877810+0000","last_active":"2024-08-29T15:28:16.510252+0000","last_peered":"2024-08-29T15:28:16.510252+0000","last_clean":"2024-08-29T15:16:27.779276+0000","last_became_active":"2024-08-29T15:19:47.810723+0000","last_became_peered":"2024-08-29T15:19:47.810723+0000","last_unstale":"2024-08-29T15:28:16.510252+0000","last_undegraded":"2024-08-29T15:19:47.801122+0000","last_fullsized":"2024-08-29T15:28:16.510252+0000","mapping_epoch":234,"log_start":"195'8568","ondisk_log_start":"195'8568","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6734,"log_dups_size":0,"ondisk_log_size":6734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35389440,"num_objects":540,"num_object_clones":0,"num_object_copies":1620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15302,"num_write_kb":506944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1925,"num_bytes_recovered":126156800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","4","6"],"object_location_counts":[{"shards":"3,4","objects":540}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":130,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.733485+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:28:05.733485+0000","last_peered":"2024-08-29T15:28:05.733485+0000","last_clean":"2024-08-29T15:28:05.733485+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:28:05.733485+0000","last_undegraded":"2024-08-29T15:28:05.733485+0000","last_fullsized":"2024-08-29T15:28:05.733485+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.738134+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:28:05.738134+0000","last_peered":"2024-08-29T15:28:05.738134+0000","last_clean":"2024-08-29T15:28:05.738134+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:28:05.738134+0000","last_undegraded":"2024-08-29T15:28:05.738134+0000","last_fullsized":"2024-08-29T15:28:05.738134+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"286'15119","reported_seq":15572,"reported_epoch":286,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:28:16.530982+0000","last_change":"2024-08-29T15:18:34.450720+0000","last_active":"2024-08-29T15:28:16.530982+0000","last_peered":"2024-08-29T15:28:16.530982+0000","last_clean":"2024-08-29T15:16:58.030079+0000","last_became_active":"2024-08-29T15:17:41.722529+0000","last_became_peered":"2024-08-29T15:17:41.722529+0000","last_unstale":"2024-08-29T15:28:16.530982+0000","last_undegraded":"2024-08-29T15:28:16.530982+0000","last_fullsized":"2024-08-29T15:28:16.530982+0000","mapping_epoch":202,"log_start":"195'8367","ondisk_log_start":"195'8367","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35323904,"num_objects":539,"num_object_clones":0,"num_object_copies":1617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":539,"num_objects_unfound":0,"num_objects_dirty":539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15119,"num_write_kb":501056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":539}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734602+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:28:05.734602+0000","last_peered":"2024-08-29T15:28:05.734602+0000","last_clean":"2024-08-29T15:28:05.734602+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:28:05.734602+0000","last_undegraded":"2024-08-29T15:28:05.734602+0000","last_fullsized":"2024-08-29T15:28:05.734602+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":272,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.738552+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:28:05.738552+0000","last_peered":"2024-08-29T15:28:05.738552+0000","last_clean":"2024-08-29T15:28:05.738552+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:28:05.738552+0000","last_undegraded":"2024-08-29T15:28:05.738552+0000","last_fullsized":"2024-08-29T15:28:05.738552+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"286'15181","reported_seq":15349,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:16.229959+0000","last_change":"2024-08-29T15:18:35.003005+0000","last_active":"2024-08-29T15:28:16.229959+0000","last_peered":"2024-08-29T15:28:16.229959+0000","last_clean":"2024-08-29T15:28:16.229959+0000","last_became_active":"2024-08-29T15:18:34.985091+0000","last_became_peered":"2024-08-29T15:18:34.985091+0000","last_unstale":"2024-08-29T15:28:16.229959+0000","last_undegraded":"2024-08-29T15:28:16.229959+0000","last_fullsized":"2024-08-29T15:28:16.229959+0000","mapping_epoch":218,"log_start":"180'5178","ondisk_log_start":"180'5178","created":168,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10003,"log_dups_size":0,"ondisk_log_size":10003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:35:22.022468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879832,"num_objects":579,"num_object_clones":0,"num_object_copies":1737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":579,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15181,"num_write_kb":504257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":436,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.782735+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:28:14.782735+0000","last_peered":"2024-08-29T15:28:14.782735+0000","last_clean":"2024-08-29T15:28:14.782735+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:28:14.782735+0000","last_undegraded":"2024-08-29T15:28:14.782735+0000","last_fullsized":"2024-08-29T15:28:14.782735+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":429,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794680+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:28:14.794680+0000","last_peered":"2024-08-29T15:28:14.794680+0000","last_clean":"2024-08-29T15:28:14.794680+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:28:14.794680+0000","last_undegraded":"2024-08-29T15:28:14.794680+0000","last_fullsized":"2024-08-29T15:28:14.794680+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"286'14663","reported_seq":14879,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:16.328763+0000","last_change":"2024-08-29T15:18:02.014160+0000","last_active":"2024-08-29T15:28:16.328763+0000","last_peered":"2024-08-29T15:28:16.328763+0000","last_clean":"2024-08-29T15:28:16.328763+0000","last_became_active":"2024-08-29T15:17:40.714727+0000","last_became_peered":"2024-08-29T15:17:40.714727+0000","last_unstale":"2024-08-29T15:28:16.328763+0000","last_undegraded":"2024-08-29T15:28:16.328763+0000","last_fullsized":"2024-08-29T15:28:16.328763+0000","mapping_epoch":201,"log_start":"180'4600","ondisk_log_start":"180'4600","created":168,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":10063,"log_dups_size":0,"ondisk_log_size":10063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:28:15.989362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35717120,"num_objects":545,"num_object_clones":0,"num_object_copies":1635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14663,"num_write_kb":486656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":528,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.793461+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:28:14.793461+0000","last_peered":"2024-08-29T15:28:14.793461+0000","last_clean":"2024-08-29T15:28:14.793461+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:28:14.793461+0000","last_undegraded":"2024-08-29T15:28:14.793461+0000","last_fullsized":"2024-08-29T15:28:14.793461+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":758,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:06.326554+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:28:06.326554+0000","last_peered":"2024-08-29T15:28:06.326554+0000","last_clean":"2024-08-29T15:28:06.326554+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:28:06.326554+0000","last_undegraded":"2024-08-29T15:28:06.326554+0000","last_fullsized":"2024-08-29T15:28:06.326554+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":266,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.728078+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:28:05.728078+0000","last_peered":"2024-08-29T15:28:05.728078+0000","last_clean":"2024-08-29T15:28:05.728078+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:28:05.728078+0000","last_undegraded":"2024-08-29T15:28:05.728078+0000","last_fullsized":"2024-08-29T15:28:05.728078+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":128,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734558+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:28:05.734558+0000","last_peered":"2024-08-29T15:28:05.734558+0000","last_clean":"2024-08-29T15:28:05.734558+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:28:05.734558+0000","last_undegraded":"2024-08-29T15:28:05.734558+0000","last_fullsized":"2024-08-29T15:28:05.734558+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"286'15097","reported_seq":15548,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:15.753290+0000","last_change":"2024-08-29T15:18:33.532763+0000","last_active":"2024-08-29T15:28:15.753290+0000","last_peered":"2024-08-29T15:28:15.753290+0000","last_clean":"2024-08-29T15:28:15.753290+0000","last_became_active":"2024-08-29T15:17:40.737033+0000","last_became_peered":"2024-08-29T15:17:40.737033+0000","last_unstale":"2024-08-29T15:28:15.753290+0000","last_undegraded":"2024-08-29T15:28:15.753290+0000","last_fullsized":"2024-08-29T15:28:15.753290+0000","mapping_epoch":201,"log_start":"194'8053","ondisk_log_start":"194'8053","created":168,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:27:21.286983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38338560,"num_objects":585,"num_object_clones":0,"num_object_copies":1755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15097,"num_write_kb":501824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":902,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.787029+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:28:14.787029+0000","last_peered":"2024-08-29T15:28:14.787029+0000","last_clean":"2024-08-29T15:28:14.787029+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:28:14.787029+0000","last_undegraded":"2024-08-29T15:28:14.787029+0000","last_fullsized":"2024-08-29T15:28:14.787029+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":440,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.786048+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:28:14.786048+0000","last_peered":"2024-08-29T15:28:14.786048+0000","last_clean":"2024-08-29T15:28:14.786048+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:28:14.786048+0000","last_undegraded":"2024-08-29T15:28:14.786048+0000","last_fullsized":"2024-08-29T15:28:14.786048+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":447,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.786098+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:28:14.786098+0000","last_peered":"2024-08-29T15:28:14.786098+0000","last_clean":"2024-08-29T15:28:14.786098+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:28:14.786098+0000","last_undegraded":"2024-08-29T15:28:14.786098+0000","last_fullsized":"2024-08-29T15:28:14.786098+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"286'15165","reported_seq":28967,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:16.536170+0000","last_change":"2024-08-29T15:18:37.863522+0000","last_active":"2024-08-29T15:28:16.536170+0000","last_peered":"2024-08-29T15:28:16.536170+0000","last_clean":"2024-08-29T15:16:27.779432+0000","last_became_active":"2024-08-29T15:18:37.799770+0000","last_became_peered":"2024-08-29T15:18:37.799770+0000","last_unstale":"2024-08-29T15:28:16.536170+0000","last_undegraded":"2024-08-29T15:18:37.792055+0000","last_fullsized":"2024-08-29T15:18:37.783511+0000","mapping_epoch":221,"log_start":"195'8430","ondisk_log_start":"195'8430","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34406400,"num_objects":525,"num_object_clones":0,"num_object_copies":1575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1001,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15165,"num_write_kb":502080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6790,"num_bytes_recovered":444989440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,6],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"3,6","objects":55},{"shards":"3,7","objects":470}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":710,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.777630+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:28:14.777630+0000","last_peered":"2024-08-29T15:28:14.777630+0000","last_clean":"2024-08-29T15:28:14.777630+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:28:14.777630+0000","last_undegraded":"2024-08-29T15:28:14.777630+0000","last_fullsized":"2024-08-29T15:28:14.777630+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":118,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.732955+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:28:05.732955+0000","last_peered":"2024-08-29T15:28:05.732955+0000","last_clean":"2024-08-29T15:28:05.732955+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:28:05.732955+0000","last_undegraded":"2024-08-29T15:28:05.732955+0000","last_fullsized":"2024-08-29T15:28:05.732955+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":474,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794462+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:28:14.794462+0000","last_peered":"2024-08-29T15:28:14.794462+0000","last_clean":"2024-08-29T15:28:14.794462+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:28:14.794462+0000","last_undegraded":"2024-08-29T15:28:14.794462+0000","last_fullsized":"2024-08-29T15:28:14.794462+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"286'15191","reported_seq":15453,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:16.491511+0000","last_change":"2024-08-29T15:19:47.821394+0000","last_active":"2024-08-29T15:28:16.491511+0000","last_peered":"2024-08-29T15:28:16.491511+0000","last_clean":"2024-08-29T15:28:16.491511+0000","last_became_active":"2024-08-29T15:19:47.800356+0000","last_became_peered":"2024-08-29T15:19:47.800356+0000","last_unstale":"2024-08-29T15:28:16.491511+0000","last_undegraded":"2024-08-29T15:28:16.491511+0000","last_fullsized":"2024-08-29T15:28:16.491511+0000","mapping_epoch":234,"log_start":"195'8497","ondisk_log_start":"195'8497","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:49.837648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34144256,"num_objects":521,"num_object_clones":0,"num_object_copies":1563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15191,"num_write_kb":502784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":226,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.726487+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:28:05.726487+0000","last_peered":"2024-08-29T15:28:05.726487+0000","last_clean":"2024-08-29T15:28:05.726487+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:28:05.726487+0000","last_undegraded":"2024-08-29T15:28:05.726487+0000","last_fullsized":"2024-08-29T15:28:05.726487+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:09.862008+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:28:09.862008+0000","last_peered":"2024-08-29T15:28:09.862008+0000","last_clean":"2024-08-29T15:28:09.862008+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:28:09.862008+0000","last_undegraded":"2024-08-29T15:28:09.862008+0000","last_fullsized":"2024-08-29T15:28:09.862008+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":133,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:06.326572+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:28:06.326572+0000","last_peered":"2024-08-29T15:28:06.326572+0000","last_clean":"2024-08-29T15:28:06.326572+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:28:06.326572+0000","last_undegraded":"2024-08-29T15:28:06.326572+0000","last_fullsized":"2024-08-29T15:28:06.326572+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.0","version":"286'15366","reported_seq":15625,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:19.050547+0000","last_change":"2024-08-29T15:18:35.139004+0000","last_active":"2024-08-29T15:28:19.050547+0000","last_peered":"2024-08-29T15:28:19.050547+0000","last_clean":"2024-08-29T15:16:27.780096+0000","last_became_active":"2024-08-29T15:18:35.123607+0000","last_became_peered":"2024-08-29T15:18:35.123607+0000","last_unstale":"2024-08-29T15:28:19.050547+0000","last_undegraded":"2024-08-29T15:18:35.011192+0000","last_fullsized":"2024-08-29T15:18:34.978569+0000","mapping_epoch":218,"log_start":"187'7242","ondisk_log_start":"187'7242","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":8124,"log_dups_size":0,"ondisk_log_size":8124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30146560,"num_objects":460,"num_object_clones":0,"num_object_copies":1380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15366,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-1963}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":158,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:06.326608+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:28:06.326608+0000","last_peered":"2024-08-29T15:28:06.326608+0000","last_clean":"2024-08-29T15:28:06.326608+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:28:06.326608+0000","last_undegraded":"2024-08-29T15:28:06.326608+0000","last_fullsized":"2024-08-29T15:28:06.326608+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":866,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794915+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:28:14.794915+0000","last_peered":"2024-08-29T15:28:14.794915+0000","last_clean":"2024-08-29T15:28:14.794915+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:28:14.794915+0000","last_undegraded":"2024-08-29T15:28:14.794915+0000","last_fullsized":"2024-08-29T15:28:14.794915+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":443,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794460+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:28:14.794460+0000","last_peered":"2024-08-29T15:28:14.794460+0000","last_clean":"2024-08-29T15:28:14.794460+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:28:14.794460+0000","last_undegraded":"2024-08-29T15:28:14.794460+0000","last_fullsized":"2024-08-29T15:28:14.794460+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"286'15144","reported_seq":15961,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:15.734246+0000","last_change":"2024-08-29T15:18:36.366082+0000","last_active":"2024-08-29T15:28:15.734246+0000","last_peered":"2024-08-29T15:28:15.734246+0000","last_clean":"2024-08-29T15:16:27.769596+0000","last_became_active":"2024-08-29T15:18:34.239056+0000","last_became_peered":"2024-08-29T15:18:34.239056+0000","last_unstale":"2024-08-29T15:28:15.734246+0000","last_undegraded":"2024-08-29T15:18:34.145373+0000","last_fullsized":"2024-08-29T15:18:33.978992+0000","mapping_epoch":217,"log_start":"194'8080","ondisk_log_start":"194'8080","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34340864,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":522,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15144,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":271,"num_bytes_recovered":17760256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":2},{"shards":"1,7","objects":-1627}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":448,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794813+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:28:14.794813+0000","last_peered":"2024-08-29T15:28:14.794813+0000","last_clean":"2024-08-29T15:28:14.794813+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:28:14.794813+0000","last_undegraded":"2024-08-29T15:28:14.794813+0000","last_fullsized":"2024-08-29T15:28:14.794813+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":715,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:05.736803+0000","last_change":"2024-08-29T15:19:46.805756+0000","last_active":"2024-08-29T15:28:05.736803+0000","last_peered":"2024-08-29T15:28:05.736803+0000","last_clean":"2024-08-29T15:19:43.180113+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:28:05.736803+0000","last_undegraded":"2024-08-29T15:19:46.793006+0000","last_fullsized":"2024-08-29T15:28:05.736803+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":440,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.785582+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:28:14.785582+0000","last_peered":"2024-08-29T15:28:14.785582+0000","last_clean":"2024-08-29T15:28:14.785582+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:28:14.785582+0000","last_undegraded":"2024-08-29T15:28:14.785582+0000","last_fullsized":"2024-08-29T15:28:14.785582+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"286'15037","reported_seq":22210,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:19.860094+0000","last_change":"2024-08-29T15:19:47.819417+0000","last_active":"2024-08-29T15:28:19.860094+0000","last_peered":"2024-08-29T15:28:19.860094+0000","last_clean":"2024-08-29T15:28:19.860094+0000","last_became_active":"2024-08-29T15:19:47.801874+0000","last_became_peered":"2024-08-29T15:19:47.801874+0000","last_unstale":"2024-08-29T15:28:19.860094+0000","last_undegraded":"2024-08-29T15:28:19.860094+0000","last_fullsized":"2024-08-29T15:28:19.860094+0000","mapping_epoch":234,"log_start":"187'7099","ondisk_log_start":"187'7099","created":168,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:58:21.883693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25624576,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15037,"num_write_kb":493696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3494,"num_bytes_recovered":150077440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":433,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.805956+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:28:14.805956+0000","last_peered":"2024-08-29T15:28:14.805956+0000","last_clean":"2024-08-29T15:28:14.805956+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:28:14.805956+0000","last_undegraded":"2024-08-29T15:28:14.805956+0000","last_fullsized":"2024-08-29T15:28:14.805956+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1056,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:14.787069+0000","last_change":"2024-08-29T15:19:47.803854+0000","last_active":"2024-08-29T15:28:14.787069+0000","last_peered":"2024-08-29T15:28:14.787069+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:28:14.787069+0000","last_undegraded":"2024-08-29T15:19:47.795161+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":431,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.781428+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:28:14.781428+0000","last_peered":"2024-08-29T15:28:14.781428+0000","last_clean":"2024-08-29T15:28:14.781428+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:28:14.781428+0000","last_undegraded":"2024-08-29T15:28:14.781428+0000","last_fullsized":"2024-08-29T15:28:14.781428+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"286'15195","reported_seq":29873,"reported_epoch":286,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:28:19.960889+0000","last_change":"2024-08-29T15:18:54.543071+0000","last_active":"2024-08-29T15:28:19.960889+0000","last_peered":"2024-08-29T15:28:19.960889+0000","last_clean":"2024-08-29T15:16:27.788588+0000","last_became_active":"2024-08-29T15:18:54.234120+0000","last_became_peered":"2024-08-29T15:18:54.234120+0000","last_unstale":"2024-08-29T15:28:19.960889+0000","last_undegraded":"2024-08-29T15:18:47.528053+0000","last_fullsized":"2024-08-29T15:18:34.971803+0000","mapping_epoch":218,"log_start":"188'7612","ondisk_log_start":"188'7612","created":168,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:15:17.731027+0000","last_clean_scrub_stamp":"2024-08-29T15:15:17.731027+0000","objects_scrubbed":0,"log_size":7583,"log_dups_size":0,"ondisk_log_size":7583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:44:30.602617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27197440,"num_objects":415,"num_object_clones":0,"num_object_copies":1245,"num_objects_missing_on_primary":1462,"num_objects_missing":1462,"num_objects_degraded":355,"num_objects_misplaced":413,"num_objects_unfound":0,"num_objects_dirty":415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15195,"num_write_kb":499520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6273,"num_bytes_recovered":410976256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,6,7","objects":58},{"shards":"1,7","objects":355},{"shards":"2,6","objects":2}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":429,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.785755+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:28:14.785755+0000","last_peered":"2024-08-29T15:28:14.785755+0000","last_clean":"2024-08-29T15:28:14.785755+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:28:14.785755+0000","last_undegraded":"2024-08-29T15:28:14.785755+0000","last_fullsized":"2024-08-29T15:28:14.785755+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":139,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.733254+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:28:05.733254+0000","last_peered":"2024-08-29T15:28:05.733254+0000","last_clean":"2024-08-29T15:28:05.733254+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:28:05.733254+0000","last_undegraded":"2024-08-29T15:28:05.733254+0000","last_fullsized":"2024-08-29T15:28:05.733254+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":67,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:09.861960+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:28:09.861960+0000","last_peered":"2024-08-29T15:28:09.861960+0000","last_clean":"2024-08-29T15:28:09.861960+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:28:09.861960+0000","last_undegraded":"2024-08-29T15:28:09.861960+0000","last_fullsized":"2024-08-29T15:28:09.861960+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.735244+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:28:05.735244+0000","last_peered":"2024-08-29T15:28:05.735244+0000","last_clean":"2024-08-29T15:28:05.735244+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:28:05.735244+0000","last_undegraded":"2024-08-29T15:28:05.735244+0000","last_fullsized":"2024-08-29T15:28:05.735244+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":64,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:05.737955+0000","last_change":"2024-08-29T15:19:46.806434+0000","last_active":"2024-08-29T15:28:05.737955+0000","last_peered":"2024-08-29T15:28:05.737955+0000","last_clean":"2024-08-29T15:14:33.716767+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:28:05.737955+0000","last_undegraded":"2024-08-29T15:19:46.791183+0000","last_fullsized":"2024-08-29T15:28:05.737955+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":437,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794297+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:28:14.794297+0000","last_peered":"2024-08-29T15:28:14.794297+0000","last_clean":"2024-08-29T15:28:14.794297+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:28:14.794297+0000","last_undegraded":"2024-08-29T15:28:14.794297+0000","last_fullsized":"2024-08-29T15:28:14.794297+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":188,"reported_epoch":286,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:14.783898+0000","last_change":"2024-08-29T15:19:46.806701+0000","last_active":"2024-08-29T15:28:14.783898+0000","last_peered":"2024-08-29T15:28:14.783898+0000","last_clean":"2024-08-29T15:19:43.183009+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:28:14.783898+0000","last_undegraded":"2024-08-29T15:19:46.790094+0000","last_fullsized":"2024-08-29T15:28:14.783898+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":155,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727894+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:28:05.727894+0000","last_peered":"2024-08-29T15:28:05.727894+0000","last_clean":"2024-08-29T15:28:05.727894+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:28:05.727894+0000","last_undegraded":"2024-08-29T15:28:05.727894+0000","last_fullsized":"2024-08-29T15:28:05.727894+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":397,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.777411+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:28:14.777411+0000","last_peered":"2024-08-29T15:28:14.777411+0000","last_clean":"2024-08-29T15:28:14.777411+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:28:14.777411+0000","last_undegraded":"2024-08-29T15:28:14.777411+0000","last_fullsized":"2024-08-29T15:28:14.777411+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":450,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.786304+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:28:14.786304+0000","last_peered":"2024-08-29T15:28:14.786304+0000","last_clean":"2024-08-29T15:28:14.786304+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:28:14.786304+0000","last_undegraded":"2024-08-29T15:28:14.786304+0000","last_fullsized":"2024-08-29T15:28:14.786304+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":449,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.781339+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:28:14.781339+0000","last_peered":"2024-08-29T15:28:14.781339+0000","last_clean":"2024-08-29T15:28:14.781339+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:28:14.781339+0000","last_undegraded":"2024-08-29T15:28:14.781339+0000","last_fullsized":"2024-08-29T15:28:14.781339+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":127,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.733326+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:28:05.733326+0000","last_peered":"2024-08-29T15:28:05.733326+0000","last_clean":"2024-08-29T15:28:05.733326+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:28:05.733326+0000","last_undegraded":"2024-08-29T15:28:05.733326+0000","last_fullsized":"2024-08-29T15:28:05.733326+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":305,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.793361+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:28:14.793361+0000","last_peered":"2024-08-29T15:28:14.793361+0000","last_clean":"2024-08-29T15:28:14.793361+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:28:14.793361+0000","last_undegraded":"2024-08-29T15:28:14.793361+0000","last_fullsized":"2024-08-29T15:28:14.793361+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":426,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.786017+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:28:14.786017+0000","last_peered":"2024-08-29T15:28:14.786017+0000","last_clean":"2024-08-29T15:28:14.786017+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:28:14.786017+0000","last_undegraded":"2024-08-29T15:28:14.786017+0000","last_fullsized":"2024-08-29T15:28:14.786017+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":429,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.794034+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:28:14.794034+0000","last_peered":"2024-08-29T15:28:14.794034+0000","last_clean":"2024-08-29T15:28:14.794034+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:28:14.794034+0000","last_undegraded":"2024-08-29T15:28:14.794034+0000","last_fullsized":"2024-08-29T15:28:14.794034+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":433,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.783748+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:28:14.783748+0000","last_peered":"2024-08-29T15:28:14.783748+0000","last_clean":"2024-08-29T15:28:14.783748+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:28:14.783748+0000","last_undegraded":"2024-08-29T15:28:14.783748+0000","last_fullsized":"2024-08-29T15:28:14.783748+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734672+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:28:05.734672+0000","last_peered":"2024-08-29T15:28:05.734672+0000","last_clean":"2024-08-29T15:28:05.734672+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:28:05.734672+0000","last_undegraded":"2024-08-29T15:28:05.734672+0000","last_fullsized":"2024-08-29T15:28:05.734672+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":453,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.795018+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:28:14.795018+0000","last_peered":"2024-08-29T15:28:14.795018+0000","last_clean":"2024-08-29T15:28:14.795018+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:28:14.795018+0000","last_undegraded":"2024-08-29T15:28:14.795018+0000","last_fullsized":"2024-08-29T15:28:14.795018+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":417,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.785546+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:28:14.785546+0000","last_peered":"2024-08-29T15:28:14.785546+0000","last_clean":"2024-08-29T15:28:14.785546+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:28:14.785546+0000","last_undegraded":"2024-08-29T15:28:14.785546+0000","last_fullsized":"2024-08-29T15:28:14.785546+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734898+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:28:05.734898+0000","last_peered":"2024-08-29T15:28:05.734898+0000","last_clean":"2024-08-29T15:28:05.734898+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:28:05.734898+0000","last_undegraded":"2024-08-29T15:28:05.734898+0000","last_fullsized":"2024-08-29T15:28:05.734898+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":179,"reported_epoch":285,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:28:05.727515+0000","last_change":"2024-08-29T15:19:46.804528+0000","last_active":"2024-08-29T15:28:05.727515+0000","last_peered":"2024-08-29T15:28:05.727515+0000","last_clean":"2024-08-29T15:19:43.181970+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:28:05.727515+0000","last_undegraded":"2024-08-29T15:19:46.797964+0000","last_fullsized":"2024-08-29T15:28:05.727515+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734760+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:28:05.734760+0000","last_peered":"2024-08-29T15:28:05.734760+0000","last_clean":"2024-08-29T15:28:05.734760+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:28:05.734760+0000","last_undegraded":"2024-08-29T15:28:05.734760+0000","last_fullsized":"2024-08-29T15:28:05.734760+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":185,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.727058+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:28:05.727058+0000","last_peered":"2024-08-29T15:28:05.727058+0000","last_clean":"2024-08-29T15:28:05.727058+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:28:05.727058+0000","last_undegraded":"2024-08-29T15:28:05.727058+0000","last_fullsized":"2024-08-29T15:28:05.727058+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":464,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.777712+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:28:14.777712+0000","last_peered":"2024-08-29T15:28:14.777712+0000","last_clean":"2024-08-29T15:28:14.777712+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:28:14.777712+0000","last_undegraded":"2024-08-29T15:28:14.777712+0000","last_fullsized":"2024-08-29T15:28:14.777712+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":466,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.778356+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:28:14.778356+0000","last_peered":"2024-08-29T15:28:14.778356+0000","last_clean":"2024-08-29T15:28:14.778356+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:28:14.778356+0000","last_undegraded":"2024-08-29T15:28:14.778356+0000","last_fullsized":"2024-08-29T15:28:14.778356+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":66,"reported_epoch":285,"state":"active+clean","last_fresh":"2024-08-29T15:28:05.734274+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:28:05.734274+0000","last_peered":"2024-08-29T15:28:05.734274+0000","last_clean":"2024-08-29T15:28:05.734274+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:28:05.734274+0000","last_undegraded":"2024-08-29T15:28:05.734274+0000","last_fullsized":"2024-08-29T15:28:05.734274+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":474,"reported_epoch":286,"state":"active+clean","last_fresh":"2024-08-29T15:28:14.786205+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:28:14.786205+0000","last_peered":"2024-08-29T15:28:14.786205+0000","last_clean":"2024-08-29T15:28:14.786205+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:28:14.786205+0000","last_undegraded":"2024-08-29T15:28:14.786205+0000","last_fullsized":"2024-08-29T15:28:14.786205+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":541720600,"num_objects":8267,"num_object_clones":0,"num_object_copies":24801,"num_objects_missing_on_primary":1462,"num_objects_missing":1462,"num_objects_degraded":17400,"num_objects_misplaced":4287,"num_objects_unfound":0,"num_objects_dirty":8267,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":242583,"num_write_kb":8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27802,"num_bytes_recovered":1702035456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2960142336,"data_stored":2960130120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124260,"ondisk_log_size":124260,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234404,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576824,"kb_used_data":224944,"kb_used_omap":1,"kb_used_meta":351870,"kb_avail":93168328,"statfs":{"total":95995035648,"available":95404367872,"internally_reserved":0,"allocated":230342656,"data_stored":228658747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":360315339},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496035,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":228020,"kb_used_data":16428,"kb_used_omap":1,"kb_used_meta":211582,"kb_avail":93517132,"statfs":{"total":95995035648,"available":95761543168,"internally_reserved":0,"allocated":16822272,"data_stored":15142163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":216660423},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426621,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281128,"kb_used_data":194016,"kb_used_omap":8,"kb_used_meta":87095,"kb_avail":93464024,"statfs":{"total":95995035648,"available":95707160576,"internally_reserved":0,"allocated":198672384,"data_stored":196999741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":89186186},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019314,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":487192,"kb_used_data":149264,"kb_used_omap":1,"kb_used_meta":337918,"kb_avail":93257960,"statfs":{"total":95995035648,"available":95496151040,"internally_reserved":0,"allocated":152846336,"data_stored":151189476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":346028486},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435498,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":708328,"kb_used_data":369884,"kb_used_omap":4,"kb_used_meta":338427,"kb_avail":93036824,"statfs":{"total":95995035648,"available":95269707776,"internally_reserved":0,"allocated":378761216,"data_stored":377092513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":346549513},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903350,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1463336,"kb_used_data":1281632,"kb_used_omap":10,"kb_used_meta":181685,"kb_avail":92281816,"statfs":{"total":95995035648,"available":94496579584,"internally_reserved":0,"allocated":1312391168,"data_stored":1310722401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":186045968},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762551,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916692,"kb_used_data":628172,"kb_used_omap":7,"kb_used_meta":288504,"kb_avail":92828460,"statfs":{"total":95995035648,"available":95056343040,"internally_reserved":0,"allocated":643248128,"data_stored":641585833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":295428450},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926078,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":465644,"kb_used_data":127140,"kb_used_omap":7,"kb_used_meta":338488,"kb_avail":93279508,"statfs":{"total":95995035648,"available":95518216192,"internally_reserved":0,"allocated":130191360,"data_stored":128514196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":346611855},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":120786944,"data_stored":120782872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":622596096,"data_stored":622592024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1308426240,"data_stored":1308426240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":354484224,"data_stored":354484224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":140247040,"data_stored":140247040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":183701504,"data_stored":183697432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":11993088,"data_stored":11993088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":217907200,"data_stored":217907200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:28:23.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:28:23.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:23.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:23.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:23.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:23.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:23.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:23.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:23.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:23.724 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:23.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:23.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:23.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:23.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:23.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:23.969 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:23 smithi027 ceph-mon[98970]: pgmap v1006: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 388 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 16859/18477 objects degraded (91.243%); 4086/18477 objects misplaced (22.114%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:28:23.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:23 smithi027 ceph-mon[93921]: pgmap v1006: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 388 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 16859/18477 objects degraded (91.243%); 4086/18477 objects misplaced (22.114%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:28:23.998 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:23 smithi171 ceph-mon[90031]: pgmap v1006: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 388 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 16859/18477 objects degraded (91.243%); 4086/18477 objects misplaced (22.114%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:28:24.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:24.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:24.236 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:24.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:24.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:24.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:24.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:24.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:24.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.557 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:24.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:24.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:24.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:24.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:24.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:24.853 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:24.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:24.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:24.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:24.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:25.048 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:28:25.048 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:28:25.061 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:25.065 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:25.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:25.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:25.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:25.249 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:25.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:25.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:25.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:25.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:25.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:25.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:25.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:25.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:25.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:25.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:25.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:25.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:25.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:25.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:28:25.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:25.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:25.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:25.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:25.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:25.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:26.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:25 smithi171 ceph-mon[90031]: pgmap v1007: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 364 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 16859/17337 objects degraded (97.243%); 4086/17337 objects misplaced (23.568%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:28:26.034 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:25 smithi027 ceph-mon[98970]: pgmap v1007: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 364 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 16859/17337 objects degraded (97.243%); 4086/17337 objects misplaced (23.568%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:28:26.034 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:25 smithi027 ceph-mon[93921]: pgmap v1007: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 364 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 16859/17337 objects degraded (97.243%); 4086/17337 objects misplaced (23.568%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:28:26.069 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:26.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:26.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:26.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:26.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:26.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:26.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:26.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:26.388 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:26.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:26.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:26.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:26.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:26.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:26.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:26.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:26.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:26.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:26.785 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:26.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:26.855 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:28:26.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:26.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:26.923 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:26.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:27.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:27.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:27.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:27.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1569057147' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:28:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:26 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:28:27.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1569057147' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:28:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:26 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:28:27.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:27.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:27.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:27.208 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1569057147' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:28:27.208 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:26 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:28:27.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:27.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:27.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:27.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:27.472 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:27.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:27.573 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:27.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:27.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:27.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:27.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:27.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:27.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:27.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[93921]: pgmap v1008: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 195 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 15932/9216 objects degraded (172.873%); 3720/9216 objects misplaced (40.365%); 11 MiB/s, 180 objects/s recovering 2024-08-29T15:28:27.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[93921]: osdmap e287: 8 total, 8 up, 8 in 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 15932/9216 objects degraded (172.873%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[98970]: pgmap v1008: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 195 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 15932/9216 objects degraded (172.873%); 3720/9216 objects misplaced (40.365%); 11 MiB/s, 180 objects/s recovering 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[98970]: osdmap e287: 8 total, 8 up, 8 in 2024-08-29T15:28:27.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 15932/9216 objects degraded (172.873%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:27.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:27.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:27.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:27.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:28.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:28.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:28.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:28.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:28.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:27 smithi171 ceph-mon[90031]: pgmap v1008: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 195 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 15932/9216 objects degraded (172.873%); 3720/9216 objects misplaced (40.365%); 11 MiB/s, 180 objects/s recovering 2024-08-29T15:28:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:27 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:27 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:28:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:27 smithi171 ceph-mon[90031]: osdmap e287: 8 total, 8 up, 8 in 2024-08-29T15:28:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 15932/9216 objects degraded (172.873%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:28.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:28.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:28.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:28.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:28.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:28.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:28.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:28.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:28.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:28.858 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:28.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:28.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:28.992 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:28.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:28.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:29.009 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:29.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:29.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:29.094 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:29.126 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:29.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:29.228 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:29.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:29.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:29.306 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:28:29.307 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:28:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:28:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:28:29] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.51.0" 2024-08-29T15:28:29.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:29.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:29.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:29.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:29.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:29.533 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:29.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:29.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:29.665 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:29.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:29.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:29.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:29.822 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:29.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:29.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:29.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:29.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:30.048 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:29 smithi171 ceph-mon[90031]: pgmap v1010: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 195 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 15932/9216 objects degraded (172.873%); 3720/9216 objects misplaced (40.365%); 9.3 MiB/s, 149 objects/s recovering 2024-08-29T15:28:30.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:28:30.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:29 smithi027 ceph-mon[98970]: pgmap v1010: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 195 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 15932/9216 objects degraded (172.873%); 3720/9216 objects misplaced (40.365%); 9.3 MiB/s, 149 objects/s recovering 2024-08-29T15:28:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:29 smithi027 ceph-mon[93921]: pgmap v1010: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 195 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 15932/9216 objects degraded (172.873%); 3720/9216 objects misplaced (40.365%); 9.3 MiB/s, 149 objects/s recovering 2024-08-29T15:28:30.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:30.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:30.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:30.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:30.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:30.295 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Removed 125424 objects 2024-08-29T15:28:30.295 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Clean up completed and total clean up time :687.686 2024-08-29T15:28:30.319 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:28:30.356 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:30.379 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:30.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:30.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:30.503 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:30.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:30.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:30.605 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:30.627 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:30.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:30.728 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:30.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:30.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:30.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:30.877 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:30.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:30.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:31.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:31.105 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:31.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:31.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:28:31.179 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:28:31.196 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:28:31.196 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:28:31.213 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:28:31.213 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:28:31.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:31.230 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:28:31.230 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:28:31.248 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:28:31.248 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:28:31.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:31.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:31.264 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:28:31.265 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:28:31.281 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:28:31.282 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:28:31.298 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:28:31.298 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:28:31.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:31.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:31.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:31.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:31.503 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:31.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:31.604 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:31.626 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:31.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:31.691 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:28:31.691 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15522/5991 objects degraded (259.089%), 16 pgs degraded, 8 pgs undersized","count":24},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1493,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":287,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":71},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":1997,"data_bytes":133765117,"bytes_used":4273549312,"bytes_avail":763686735872,"bytes_total":767960285184,"degraded_objects":15522,"degraded_total":5991,"degraded_ratio":2.5908863294942415,"misplaced_objects":3500,"misplaced_total":5991,"misplaced_ratio":0.58420964780504092,"recovering_objects_per_sec":170,"recovering_bytes_per_sec":10546910,"recovering_keys_per_sec":0,"num_objects_recovered":1702,"num_bytes_recovered":105512960,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":629},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:28:31.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:31.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:31.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:28:31.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:31.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:31.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:31.992 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:32.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:32.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:32.031 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:31 smithi027 ceph-mon[98970]: pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 128 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 15522/5991 objects degraded (259.089%); 3500/5991 objects misplaced (58.421%); 10 MiB/s, 170 objects/s recovering 2024-08-29T15:28:32.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3675653703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:32.032 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/191632676' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:28:32.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:31 smithi027 ceph-mon[93921]: pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 128 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 15522/5991 objects degraded (259.089%); 3500/5991 objects misplaced (58.421%); 10 MiB/s, 170 objects/s recovering 2024-08-29T15:28:32.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3675653703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:32.032 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/191632676' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:28:32.084 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:31 smithi171 ceph-mon[90031]: pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 128 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 15522/5991 objects degraded (259.089%); 3500/5991 objects misplaced (58.421%); 10 MiB/s, 170 objects/s recovering 2024-08-29T15:28:32.084 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3675653703' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:32.084 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/191632676' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:28:32.090 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:28:32.090 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:238695 6.a 6:524191cb:::benchmark_data_smithi027_169445_object113268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:09.787344+0000", 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 22.300221615000002, 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040391942, 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.091 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 238695 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.787344+0000", 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.787344+0000", 2024-08-29T15:28:32.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.787343+0000", 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.787348+0000", 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6059999999999996e-06 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.787360+0000", 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1731e-05 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.787366+0000", 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3470000000000004e-06 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.791012+0000", 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036456309999999999 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.791130+0000", 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000118495 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.791239+0000", 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010872800000000001 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.799836+0000", 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0085973950000000007 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.802198+0000", 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0023617220000000001 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.826496+0000", 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024298396 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.826545+0000", 2024-08-29T15:28:32.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8551000000000003e-05 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:09.827736+0000", 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0011911490000000001 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:239552 6.7 6:e70a5873:::benchmark_data_smithi027_169445_object114125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:11.722970+0000", 2024-08-29T15:28:32.097 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 20.364595980000001, 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042355519000000001, 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 239552 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.722970+0000", 2024-08-29T15:28:32.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.722970+0000", 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.722968+0000", 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.722977+0000", 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9709999999999993e-06 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.722995+0000", 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8085e-05 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.723006+0000", 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1079e-05 2024-08-29T15:28:32.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.723052+0000", 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6294000000000001e-05 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.723144+0000", 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1797000000000002e-05 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.723242+0000", 2024-08-29T15:28:32.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8123999999999998e-05 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.726674+0000", 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034316210000000002 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.727611+0000", 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00093689700000000001 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.765231+0000", 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.037620114000000003 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.765284+0000", 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3656999999999998e-05 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:11.765325+0000", 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0810999999999999e-05 2024-08-29T15:28:32.103 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240731 6.7 6:ed500a9e:::benchmark_data_smithi027_169445_object115304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.071343+0000", 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 18.016222290999998, 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.047718495, 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.104 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240731 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071343+0000", 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071343+0000", 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071343+0000", 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071347+0000", 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9060000000000001e-06 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071358+0000", 2024-08-29T15:28:32.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1049999999999999e-05 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071371+0000", 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2272999999999999e-05 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.092546+0000", 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021175295 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.092727+0000", 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000181192 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.092848+0000", 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000120569 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.109051+0000", 2024-08-29T15:28:32.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016202946999999999 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.118796+0000", 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0097453660000000001 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.118972+0000", 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000176336 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119024+0000", 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1063999999999998e-05 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119062+0000", 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8343000000000002e-05 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.110 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240732 6.7 6:eec3e861:::benchmark_data_smithi027_169445_object115305:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.071692+0000", 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 18.015873619000001, 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.056128277999999997, 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240732 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.111 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071692+0000", 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071692+0000", 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071691+0000", 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071696+0000", 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0640000000000001e-06 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071714+0000", 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7394999999999999e-05 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.071720+0000", 2024-08-29T15:28:32.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2040000000000002e-06 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.098686+0000", 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026966343 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.098988+0000", 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00030133699999999999 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.099351+0000", 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00036328700000000001 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.109194+0000", 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0098430719999999996 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.118831+0000", 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.009637372 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.127500+0000", 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086689839999999994 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.127666+0000", 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016559699999999999 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.127820+0000", 2024-08-29T15:28:32.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015441700000000001 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240733 6.7 6:ee8bf952:::benchmark_data_smithi027_169445_object115306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.072526+0000", 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 18.015039291000001, 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066965819999999995, 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.117 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240733 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072526+0000", 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072526+0000", 2024-08-29T15:28:32.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072525+0000", 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072531+0000", 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5799999999999999e-06 2024-08-29T15:28:32.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:32.121 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.121 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.121 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072541+0000", 2024-08-29T15:28:32.121 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0453e-05 2024-08-29T15:28:32.121 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.121 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072548+0000", 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4400000000000002e-06 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.104570+0000", 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032022238000000001 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.104758+0000", 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018819900000000001 2024-08-29T15:28:32.122 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.104904+0000", 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014563600000000001 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.118915+0000", 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014011288 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.123 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119079+0000", 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000163528 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.139379+0000", 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020299762999999998 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.139461+0000", 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2114999999999996e-05 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.124 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.139492+0000", 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.154e-05 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240750 6.7 6:e7d3b0c8:::benchmark_data_smithi027_169445_object115323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.099518+0000", 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.988047932000001, 2024-08-29T15:28:32.125 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042355489000000003, 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240750 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.099518+0000", 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.126 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.099518+0000", 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.099516+0000", 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.099524+0000", 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7829999999999999e-06 2024-08-29T15:28:32.127 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.099536+0000", 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1841999999999999e-05 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.099543+0000", 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3679999999999996e-06 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.128 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.109291+0000", 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0097476090000000008 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.109584+0000", 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00029329300000000003 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.109947+0000", 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00036337300000000001 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.129 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119085+0000", 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0091380460000000004 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.127354+0000", 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0082690650000000008 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.141693+0000", 2024-08-29T15:28:32.130 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014338564 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.141789+0000", 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6112000000000004e-05 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.141873+0000", 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4135000000000007e-05 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.131 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240752 6.7 6:eed7d311:::benchmark_data_smithi027_169445_object115325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.100131+0000", 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.987434490999998, 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041880756999999998, 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.132 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240752 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.100131+0000", 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.100131+0000", 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.133 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.100130+0000", 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.100136+0000", 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6439999999999997e-06 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.100153+0000", 2024-08-29T15:28:32.134 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7059999999999999e-05 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.100160+0000", 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4050000000000003e-06 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.114382+0000", 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014221958 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.135 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.114509+0000", 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000126914 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.114639+0000", 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000129469 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.127859+0000", 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013220021 2024-08-29T15:28:32.136 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.134351+0000", 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006492622 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.141943+0000", 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075916380000000004 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.137 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.141979+0000", 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6625000000000003e-05 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.142012+0000", 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2456999999999998e-05 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.138 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240753 6.7 6:eac595be:::benchmark_data_smithi027_169445_object115326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.101742+0000", 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.985823256, 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040614984, 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240753 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.139 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.101742+0000", 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.101742+0000", 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.101741+0000", 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.101748+0000", 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6080000000000001e-06 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.101766+0000", 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8303000000000001e-05 2024-08-29T15:28:32.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.101774+0000", 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9440000000000005e-06 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119104+0000", 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017329884 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119188+0000", 2024-08-29T15:28:32.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4503000000000001e-05 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119267+0000", 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8875000000000006e-05 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.127879+0000", 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0086115749999999998 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.134381+0000", 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0065024599999999998 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.142103+0000", 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0077216419999999999 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.142217+0000", 2024-08-29T15:28:32.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000114036 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.142357+0000", 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000140366 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240994 6.a 6:58b7de4c:::benchmark_data_smithi027_169445_object115567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.486495+0000", 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.601071051000002, 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.044790976000000003, 2024-08-29T15:28:32.145 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240994 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486495+0000", 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.146 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486495+0000", 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486493+0000", 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486500+0000", 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.049e-06 2024-08-29T15:28:32.147 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486511+0000", 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1219e-05 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486516+0000", 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3619999999999997e-06 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.148 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486565+0000", 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8563000000000002e-05 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486739+0000", 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000174022 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.486917+0000", 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017792 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.490735+0000", 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038188520000000002 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.493071+0000", 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002335831 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.529778+0000", 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036706766000000002 2024-08-29T15:28:32.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.529845+0000", 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6855999999999998e-05 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.531286+0000", 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014406709999999999 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240995 6.a 6:56e0e263:::benchmark_data_smithi027_169445_object115568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.494025+0000", 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.593540229999999, 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038283902000000002, 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240995 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.152 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494025+0000", 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494025+0000", 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494024+0000", 2024-08-29T15:28:32.153 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494030+0000", 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1739999999999997e-06 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494043+0000", 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2500000000000001e-05 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.154 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494050+0000", 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0119999999999999e-06 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494086+0000", 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6628999999999998e-05 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494193+0000", 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106401 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.494295+0000", 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000102778 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.498695+0000", 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043992959999999996 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.500169+0000", 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001474515 2024-08-29T15:28:32.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.531335+0000", 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031165846000000001 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.531368+0000", 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2645000000000003e-05 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.532309+0000", 2024-08-29T15:28:32.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00094165000000000004 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:241139 6.7 6:e2b3e38d:::benchmark_data_smithi027_169445_object115712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.771016+0000", 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.316549464000001, 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041961581999999997, 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.158 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 241139 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771016+0000", 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771016+0000", 2024-08-29T15:28:32.159 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771014+0000", 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771023+0000", 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1300000000000007e-06 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.160 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771040+0000", 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6475000000000001e-05 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771048+0000", 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.225e-06 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771094+0000", 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6016999999999997e-05 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.161 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771184+0000", 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0060000000000002e-05 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771305+0000", 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000120877 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.782498+0000", 2024-08-29T15:28:32.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011193409 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.785879+0000", 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003380751 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.812871+0000", 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026992153000000001 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.812943+0000", 2024-08-29T15:28:32.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1613999999999996e-05 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.812978+0000", 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4767000000000001e-05 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:241141 6.7 6:e5595b30:::benchmark_data_smithi027_169445_object115714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.771059+0000", 2024-08-29T15:28:32.164 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 17.316506921999999, 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041999061999999997, 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 241141 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771059+0000", 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.165 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771059+0000", 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771058+0000", 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.166 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771062+0000", 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.326e-06 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771068+0000", 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8639999999999998e-06 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.771074+0000", 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3789999999999997e-06 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.167 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.776148+0000", 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050746480000000002 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.776302+0000", 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015359400000000001 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.777014+0000", 2024-08-29T15:28:32.168 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00071159399999999996 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.782504+0000", 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0054899809999999997 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.786058+0000", 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003553914 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.813013+0000", 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026955921000000001 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.813039+0000", 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5103e-05 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.813058+0000", 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9215000000000002e-05 2024-08-29T15:28:32.170 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:242062 6.7 6:e24efc32:::benchmark_data_smithi027_169445_object116635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:16.380816+0000", 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.706749393999999, 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040000485000000002, 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.171 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242062 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.380816+0000", 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.380816+0000", 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.172 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.380810+0000", 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999938 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.380823+0000", 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2822e-05 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.380840+0000", 2024-08-29T15:28:32.173 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6592999999999999e-05 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.380847+0000", 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9140000000000008e-06 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.380899+0000", 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.168e-05 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.381018+0000", 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119116 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.381152+0000", 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000134191 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.385756+0000", 2024-08-29T15:28:32.175 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046040150000000004 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.387524+0000", 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001767668 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.420640+0000", 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033116275000000001 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.420774+0000", 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013334800000000001 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.420817+0000", 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2997999999999997e-05 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:242136 6.a 6:5f125f53:::benchmark_data_smithi027_169445_object116709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:16.486258+0000", 2024-08-29T15:28:32.177 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.601307970000001, 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049162550999999999, 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242136 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.486258+0000", 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.178 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.486258+0000", 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.486256+0000", 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.486263+0000", 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.041e-06 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.486279+0000", 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6317999999999999e-05 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.486286+0000", 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3640000000000001e-06 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.489960+0000", 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036738729999999998 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.490102+0000", 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014226700000000001 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.490252+0000", 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000149617 2024-08-29T15:28:32.181 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.498144+0000", 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078922090000000007 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.515208+0000", 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017063367999999999 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.533940+0000", 2024-08-29T15:28:32.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018732586999999998 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.534023+0000", 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2742000000000002e-05 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.535420+0000", 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013973049999999999 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.183 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:242137 6.a 6:555f9d08:::benchmark_data_smithi027_169445_object116710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:16.491614+0000", 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.595952130000001, 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.044900019999999999, 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242137 2024-08-29T15:28:32.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.491614+0000", 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.491614+0000", 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.491612+0000", 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.491619+0000", 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.985e-06 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.491633+0000", 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4555e-05 2024-08-29T15:28:32.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.491641+0000", 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5310000000000004e-06 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.493582+0000", 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00194133 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.493739+0000", 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015656599999999999 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.493844+0000", 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000105179 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.506517+0000", 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012673415 2024-08-29T15:28:32.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.515241+0000", 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087233439999999992 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.535479+0000", 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020238602000000001 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.535516+0000", 2024-08-29T15:28:32.189 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6934000000000002e-05 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.536514+0000", 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00099721399999999996 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:242142 6.a 6:5ad7ccee:::benchmark_data_smithi027_169445_object116715:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:16.494685+0000", 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.592880426000001, 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042570872000000003, 2024-08-29T15:28:32.190 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242142 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.494685+0000", 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.494685+0000", 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.494684+0000", 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.494690+0000", 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2210000000000002e-06 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.494717+0000", 2024-08-29T15:28:32.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7200999999999999e-05 2024-08-29T15:28:32.193 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.494726+0000", 2024-08-29T15:28:32.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3380000000000005e-06 2024-08-29T15:28:32.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.498165+0000", 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034395459999999999 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.498331+0000", 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016560899999999999 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.498488+0000", 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000157229 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.506535+0000", 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0080475259999999993 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.515273+0000", 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087375749999999992 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.536573+0000", 2024-08-29T15:28:32.196 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021300423999999998 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.536629+0000", 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6036999999999999e-05 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.537256+0000", 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00062661699999999999 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:242146 6.a 6:5bf6c1f7:::benchmark_data_smithi027_169445_object116719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:16.499557+0000", 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.588008645, 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038352672999999997, 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 242146 2024-08-29T15:28:32.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.499557+0000", 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.499557+0000", 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.499555+0000", 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.499561+0000", 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.285e-06 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.499575+0000", 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3312999999999999e-05 2024-08-29T15:28:32.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.499582+0000", 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.985e-06 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.502018+0000", 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0024364970000000001 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.502209+0000", 2024-08-29T15:28:32.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019118600000000001 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.502382+0000", 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017313800000000001 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.511344+0000", 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0089617299999999993 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.523532+0000", 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012187402999999999 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.537311+0000", 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013779837 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.537363+0000", 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1848000000000002e-05 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:16.537910+0000", 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00054638199999999999 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:243473 6.7 6:edff5081:::benchmark_data_smithi027_169445_object118046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 13.220036457000001, 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062441970999999999, 2024-08-29T15:28:32.204 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 243473 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867532+0000", 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1389999999999999e-06 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867539+0000", 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9530000000000002e-06 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867544+0000", 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5389999999999998e-06 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.871915+0000", 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043704920000000001 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.872038+0000", 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000123638 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.872149+0000", 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000110525 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.877166+0000", 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050165799999999997 2024-08-29T15:28:32.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.878416+0000", 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001250082 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929870+0000", 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051454518999999997 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929919+0000", 2024-08-29T15:28:32.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9165000000000001e-05 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929971+0000", 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1830999999999999e-05 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:243497 6.d 6:b7a24a48:::benchmark_data_smithi027_169445_object118070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:18.928962+0000", 2024-08-29T15:28:32.210 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 13.158603278999999, 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.043408611999999999, 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 243497 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.928962+0000", 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.928962+0000", 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.928961+0000", 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.928968+0000", 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4789999999999999e-06 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.928982+0000", 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.344e-05 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.928990+0000", 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4670000000000004e-06 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929044+0000", 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3990000000000003e-05 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929174+0000", 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000130259 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929294+0000", 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000120066 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.936323+0000", 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070289999999999997 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.939803+0000", 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034794439999999999 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.972303+0000", 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032500316000000001 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.972345+0000", 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2219999999999999e-05 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.972371+0000", 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5561000000000001e-05 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:243500 6.d 6:b0cbeaf7:::benchmark_data_smithi027_169445_object118073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:28:32.216 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:18.931375+0000", 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 13.156190523999999, 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041140208999999997, 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 243500 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:28:32.217 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.931375+0000", 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.931375+0000", 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.931374+0000", 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.931380+0000", 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4919999999999998e-06 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.931392+0000", 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2139e-05 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.931399+0000", 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3210000000000001e-06 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.932959+0000", 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015598700000000001 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.933090+0000", 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013171 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.933220+0000", 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013005699999999999 2024-08-29T15:28:32.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.936442+0000", 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003221481 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.951832+0000", 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01539018 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:28:32.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.972423+0000", 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020590765 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.972475+0000", 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2210999999999999e-05 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.972515+0000", 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0293000000000002e-05 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.222 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:28:32.223 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:28:32.223 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:28:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:32.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:32.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:32.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:32.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:32.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:32.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:32.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:32.525 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-08-29T15:28:32.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-08-29T15:28:32.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:32.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:32.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:32.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:32.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:32.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:32.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:32.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:32.966 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-29T15:28:32.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:28:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:32.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:32.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 15522/5991 objects degraded (259.089%), 16 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 15522/5991 objects degraded (259.089%), 16 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:33.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:33.084 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:33.104 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:33.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:33.107 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:33.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:33.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:33.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 15522/5991 objects degraded (259.089%), 16 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:33.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:33.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:33.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:33.455 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:33.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:33.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:33.600 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:33.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:33.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:33.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:33.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:33.848 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:33.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:33.973 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:33.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:33.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:33 smithi171 ceph-mon[90031]: pgmap v1012: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 14 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 15094/549 objects degraded (2749.362%); 3345/549 objects misplaced (609.290%); 9.1 MiB/s, 155 objects/s recovering 2024-08-29T15:28:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:33 smithi027 ceph-mon[98970]: pgmap v1012: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 14 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 15094/549 objects degraded (2749.362%); 3345/549 objects misplaced (609.290%); 9.1 MiB/s, 155 objects/s recovering 2024-08-29T15:28:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:33 smithi027 ceph-mon[93921]: pgmap v1012: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 14 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 15094/549 objects degraded (2749.362%); 3345/549 objects misplaced (609.290%); 9.1 MiB/s, 155 objects/s recovering 2024-08-29T15:28:34.076 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:34.097 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:34.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:34.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:34.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:34.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:34.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:28:34.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:34.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:34.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:34.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:34.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:34.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:34.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:34.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:34.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:28:34.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:34.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:34.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:34.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:34.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:34.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:34.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:35.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:35.089 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:35.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:35.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:35.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:35.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:35.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:35.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:35.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:35.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:35.543 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:35.544 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:35.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:35.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:35.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:35.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:35.815 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:35.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:35.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:35.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:35.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:35.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:36.033 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:35 smithi027 ceph-mon[93921]: pgmap v1013: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.8 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 15094/192 objects degraded (7861.458%); 3345/192 objects misplaced (1742.188%); 9.1 MiB/s, 155 objects/s recovering 2024-08-29T15:28:36.033 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:35 smithi027 ceph-mon[98970]: pgmap v1013: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.8 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 15094/192 objects degraded (7861.458%); 3345/192 objects misplaced (1742.188%); 9.1 MiB/s, 155 objects/s recovering 2024-08-29T15:28:36.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:36.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:36.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:36.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:35 smithi171 ceph-mon[90031]: pgmap v1013: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 9 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 6.8 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 15094/192 objects degraded (7861.458%); 3345/192 objects misplaced (1742.188%); 9.1 MiB/s, 155 objects/s recovering 2024-08-29T15:28:36.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:36.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:36.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:36.337 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:36.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:36.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:36.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:36.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:36.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:36.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:36.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:36.811 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:28:36.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:36.839 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:36.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:36.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:36.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:36 smithi027 ceph-mon[98970]: osdmap e288: 8 total, 8 up, 8 in 2024-08-29T15:28:36.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1094010876' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:36.954 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:36 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:36.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:36 smithi027 ceph-mon[93921]: osdmap e288: 8 total, 8 up, 8 in 2024-08-29T15:28:36.954 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1094010876' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:36.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:36 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:36.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:36.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:37.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:37.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:37.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:37.192 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:37.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:36 smithi171 ceph-mon[90031]: osdmap e288: 8 total, 8 up, 8 in 2024-08-29T15:28:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1094010876' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:36 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:28:37.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:37.342 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:37.444 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:37.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:37.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:37.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:37.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:37.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:37.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:37.717 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:37.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:37.803 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:37.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:37.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:37.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:37.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:37.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[93921]: pgmap v1015: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 14116/84 objects degraded (16804.762%); 3335/84 objects misplaced (3970.238%); 4.6 MiB/s, 192 objects/s recovering 2024-08-29T15:28:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:28:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:28:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[93921]: osdmap e289: 8 total, 8 up, 8 in 2024-08-29T15:28:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 14116/84 objects degraded (16804.762%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:38.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[98970]: pgmap v1015: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 14116/84 objects degraded (16804.762%); 3335/84 objects misplaced (3970.238%); 4.6 MiB/s, 192 objects/s recovering 2024-08-29T15:28:38.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:28:38.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:28:38.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[98970]: osdmap e289: 8 total, 8 up, 8 in 2024-08-29T15:28:38.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 14116/84 objects degraded (16804.762%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:37 smithi171 ceph-mon[90031]: pgmap v1015: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 14116/84 objects degraded (16804.762%); 3335/84 objects misplaced (3970.238%); 4.6 MiB/s, 192 objects/s recovering 2024-08-29T15:28:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:37 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:28:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:37 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:28:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:37 smithi171 ceph-mon[90031]: osdmap e289: 8 total, 8 up, 8 in 2024-08-29T15:28:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 14116/84 objects degraded (16804.762%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-29T15:28:38.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:38.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:38.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:38.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:38.224 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:38.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:38.326 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:38.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:38.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:38.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:38.475 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:38.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:38.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:38.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:38.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:38.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:38.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:38.830 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_1' removed 2024-08-29T15:28:38.869 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:38.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:38.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:38.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:39.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:38 smithi027 ceph-mon[98970]: osdmap e290: 8 total, 8 up, 8 in 2024-08-29T15:28:39.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1456004232' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:28:39.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:38 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:28:39.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:38 smithi027 ceph-mon[93921]: osdmap e290: 8 total, 8 up, 8 in 2024-08-29T15:28:39.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1456004232' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:28:39.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:38 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:28:39.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:39.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:39.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:39.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:39.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:38 smithi171 ceph-mon[90031]: osdmap e290: 8 total, 8 up, 8 in 2024-08-29T15:28:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1456004232' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:28:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:38 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:28:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:28:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:28:39] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.51.0" 2024-08-29T15:28:39.347 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:39.368 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:39.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:39.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:39.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:39.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:39.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:39.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:39.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:39.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:39.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:28:39.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:39.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:39.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:39.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:39.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:39.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:39.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:39 smithi027 ceph-mon[98970]: pgmap v1018: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 14116/84 objects degraded (16804.762%); 3335/84 objects misplaced (3970.238%); 277 KiB/s, 184 objects/s recovering 2024-08-29T15:28:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:39 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:28:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:39 smithi027 ceph-mon[98970]: osdmap e291: 8 total, 8 up, 8 in 2024-08-29T15:28:40.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:39 smithi027 ceph-mon[93921]: pgmap v1018: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 14116/84 objects degraded (16804.762%); 3335/84 objects misplaced (3970.238%); 277 KiB/s, 184 objects/s recovering 2024-08-29T15:28:40.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:39 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:28:40.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:39 smithi027 ceph-mon[93921]: osdmap e291: 8 total, 8 up, 8 in 2024-08-29T15:28:40.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:40.115 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:40.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:40.239 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:39 smithi171 ceph-mon[90031]: pgmap v1018: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 25 op/s; 14116/84 objects degraded (16804.762%); 3335/84 objects misplaced (3970.238%); 277 KiB/s, 184 objects/s recovering 2024-08-29T15:28:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:39 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:28:40.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:39 smithi171 ceph-mon[90031]: osdmap e291: 8 total, 8 up, 8 in 2024-08-29T15:28:40.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:40.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:40.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:40.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:40.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.488 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:40.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:40.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:40.612 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:40.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:40.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:40.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:40.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:40.838 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:40.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:40.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:40.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:40.987 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:40.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:40.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:41.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:41.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:41.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:41.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:40 smithi027 ceph-mon[93921]: osdmap e292: 8 total, 8 up, 8 in 2024-08-29T15:28:41.175 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:40 smithi027 ceph-mon[98970]: osdmap e292: 8 total, 8 up, 8 in 2024-08-29T15:28:41.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:41.235 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:41.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:41.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:40 smithi171 ceph-mon[90031]: osdmap e292: 8 total, 8 up, 8 in 2024-08-29T15:28:41.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:41.364 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:41.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:41.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:41.466 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:41.480 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-29T15:28:41.481 INFO:tasks.radosbench:Beginning radosbench... 2024-08-29T15:28:41.481 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-08-29T15:28:41.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-08-29T15:28:41.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:41.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:41.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:41.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:41.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:41.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:41.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:41.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:41.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:41.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:41.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:41.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:41.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:42.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:41 smithi027 ceph-mon[93921]: pgmap v1021: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 0 B/s, 2 keys/s, 0 objects/s recovering 2024-08-29T15:28:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:41 smithi027 ceph-mon[93921]: osdmap e293: 8 total, 8 up, 8 in 2024-08-29T15:28:42.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:41 smithi027 ceph-mon[98970]: pgmap v1021: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 0 B/s, 2 keys/s, 0 objects/s recovering 2024-08-29T15:28:42.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:41 smithi027 ceph-mon[98970]: osdmap e293: 8 total, 8 up, 8 in 2024-08-29T15:28:42.101 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:42.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:42.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:42.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:42.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:41 smithi171 ceph-mon[90031]: pgmap v1021: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10/84 objects degraded (11.905%); 0 B/s, 2 keys/s, 0 objects/s recovering 2024-08-29T15:28:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:41 smithi171 ceph-mon[90031]: osdmap e293: 8 total, 8 up, 8 in 2024-08-29T15:28:42.309 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:28:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:28:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:28:42.310 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:28:42.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:42.376 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:42.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:42.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:42.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:42.602 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:42.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:42.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:42.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:42.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:42.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:42.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:42.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:43.005 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:43.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:43.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:28:43.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:28:43.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:43.133 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:43.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:43.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:43.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:43.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:28:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:28:43.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:43.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:43.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:43.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:43.511 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:43.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:43.570 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:28:43.570 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:28:43.574 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:43.599 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:28:43.599 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:28:43.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:43.623 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:28:43.623 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:28:43.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:43.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:43.641 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:28:43.641 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:28:43.657 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:28:43.658 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:28:43.674 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:28:43.674 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:28:43.691 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:28:43.691 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:28:43.709 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:28:43.709 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:28:43.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:43.767 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:43.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:43.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:43.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:43.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:43.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:43.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[93921]: pgmap v1023: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-08-29T15:28:43.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2024-08-29T15:28:43.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:28:43.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1382954959' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:43.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:43.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:43.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[98970]: pgmap v1023: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-08-29T15:28:43.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2024-08-29T15:28:43.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:28:43.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1382954959' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:43.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:43.896 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:43.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:44.018 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:44.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:44.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:44.152 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:44.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:43 smithi171 ceph-mon[90031]: pgmap v1023: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-08-29T15:28:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:43 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded, 1 pg undersized) 2024-08-29T15:28:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:43 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:28:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1382954959' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:44.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:44.374 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:44.400 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:44.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:44.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:44.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:44.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:44.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:44.739 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:28:44.739 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:28:44.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:44.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:44.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:44.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:44.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:44.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:45.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:45.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:45.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:45.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:45.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:45.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:45.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:45.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:45.393 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:45.418 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:45.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:45.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:45.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:45.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:45.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:45.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:45.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:45.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:45.931 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:45.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:46.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:46.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:46.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:45 smithi027 ceph-mon[98970]: pgmap v1024: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-08-29T15:28:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:45 smithi027 ceph-mon[98970]: from='client.36216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:45 smithi027 ceph-mon[93921]: pgmap v1024: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-08-29T15:28:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:45 smithi027 ceph-mon[93921]: from='client.36216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:46.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:46.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:46.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:45 smithi171 ceph-mon[90031]: pgmap v1024: 73 pgs: 1 peering, 72 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-08-29T15:28:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:45 smithi171 ceph-mon[90031]: from='client.36216 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:28:46.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:46.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:46.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:46.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:46.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:46.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:46.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:46.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:46.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:46.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:46.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:46.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:46.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:46.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:46.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:46.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:46.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:47.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:47.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:47.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:47.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:47.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:47.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:47.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:47.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:47.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:47.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:47.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:47.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:47.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:47.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:28:47.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:47.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:47.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:47.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:47.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:47.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:47.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:47 smithi171 ceph-mon[90031]: pgmap v1025: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 521 KiB/s, 1 keys/s, 3 objects/s recovering 2024-08-29T15:28:48.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:48.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:48.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:48.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:48.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:48.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:47 smithi027 ceph-mon[98970]: pgmap v1025: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 521 KiB/s, 1 keys/s, 3 objects/s recovering 2024-08-29T15:28:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:47 smithi027 ceph-mon[93921]: pgmap v1025: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 521 KiB/s, 1 keys/s, 3 objects/s recovering 2024-08-29T15:28:48.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:48.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:48.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:48.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:48.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:48.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:48.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:48.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:48.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:48.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:48.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:48.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:48.998 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:48.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:49.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:49.115 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:49.130 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:49.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:49.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:49.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:49.257 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:49.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:49.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:28:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:28:49] "GET /metrics HTTP/1.1" 200 34174 "" "Prometheus/2.51.0" 2024-08-29T15:28:49.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:49.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:49.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:49.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:49.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:49.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:49.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:49.616 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:49.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:49.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:49.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:49.655 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1024,"stamp":"2024-08-29T15:28:44.694864+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6365,"num_read_kb":13822,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":10166259,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2683484,"kb_used_data":462364,"kb_used_omap":43,"kb_used_meta":2220948,"kb_avail":747277732,"statfs":{"total":767960285184,"available":765212397568,"internally_reserved":0,"allocated":473460736,"data_stored":459763052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2274251244},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":25,"apply_latency_ms":25,"commit_latency_ns":25000000,"apply_latency_ns":25000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":-89,"num_objects_missing":-89,"num_objects_degraded":-14116,"num_objects_misplaced":-3335,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-250850,"num_write_kb":-8027137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-30831,"num_bytes_recovered":-1821502772,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001677"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":472,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833145+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:28:40.833145+0000","last_peered":"2024-08-29T15:28:40.833145+0000","last_clean":"2024-08-29T15:28:40.833145+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:28:40.833145+0000","last_undegraded":"2024-08-29T15:28:40.833145+0000","last_fullsized":"2024-08-29T15:28:40.833145+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":79,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011997+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:28:39.011997+0000","last_peered":"2024-08-29T15:28:39.011997+0000","last_clean":"2024-08-29T15:28:39.011997+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:28:39.011997+0000","last_undegraded":"2024-08-29T15:28:39.011997+0000","last_fullsized":"2024-08-29T15:28:39.011997+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":162,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828398+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:28:39.828398+0000","last_peered":"2024-08-29T15:28:39.828398+0000","last_clean":"2024-08-29T15:28:39.828398+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:28:39.828398+0000","last_undegraded":"2024-08-29T15:28:39.828398+0000","last_fullsized":"2024-08-29T15:28:39.828398+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":439,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004446+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:28:41.004446+0000","last_peered":"2024-08-29T15:28:41.004446+0000","last_clean":"2024-08-29T15:28:41.004446+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:28:41.004446+0000","last_undegraded":"2024-08-29T15:28:41.004446+0000","last_fullsized":"2024-08-29T15:28:41.004446+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":438,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004643+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:28:41.004643+0000","last_peered":"2024-08-29T15:28:41.004643+0000","last_clean":"2024-08-29T15:28:41.004643+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:28:41.004643+0000","last_undegraded":"2024-08-29T15:28:41.004643+0000","last_fullsized":"2024-08-29T15:28:41.004643+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011912+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:28:39.011912+0000","last_peered":"2024-08-29T15:28:39.011912+0000","last_clean":"2024-08-29T15:28:39.011912+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:28:39.011912+0000","last_undegraded":"2024-08-29T15:28:39.011912+0000","last_fullsized":"2024-08-29T15:28:39.011912+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830716+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:28:39.830716+0000","last_peered":"2024-08-29T15:28:39.830716+0000","last_clean":"2024-08-29T15:28:39.830716+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:28:39.830716+0000","last_undegraded":"2024-08-29T15:28:39.830716+0000","last_fullsized":"2024-08-29T15:28:39.830716+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":163,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:38.827685+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:28:38.827685+0000","last_peered":"2024-08-29T15:28:38.827685+0000","last_clean":"2024-08-29T15:28:38.827685+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:28:38.827685+0000","last_undegraded":"2024-08-29T15:28:38.827685+0000","last_fullsized":"2024-08-29T15:28:38.827685+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":76,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012373+0000","last_change":"2024-08-29T15:28:35.524176+0000","last_active":"2024-08-29T15:28:39.012373+0000","last_peered":"2024-08-29T15:28:39.012373+0000","last_clean":"2024-08-29T15:28:39.012373+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:28:39.012373+0000","last_undegraded":"2024-08-29T15:28:39.012373+0000","last_fullsized":"2024-08-29T15:28:39.012373+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":181,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827819+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:28:39.827819+0000","last_peered":"2024-08-29T15:28:39.827819+0000","last_clean":"2024-08-29T15:28:39.827819+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:28:39.827819+0000","last_undegraded":"2024-08-29T15:28:39.827819+0000","last_fullsized":"2024-08-29T15:28:39.827819+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":76,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012385+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:28:39.012385+0000","last_peered":"2024-08-29T15:28:39.012385+0000","last_clean":"2024-08-29T15:28:39.012385+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:28:39.012385+0000","last_undegraded":"2024-08-29T15:28:39.012385+0000","last_fullsized":"2024-08-29T15:28:39.012385+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":73,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011756+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:28:39.011756+0000","last_peered":"2024-08-29T15:28:39.011756+0000","last_clean":"2024-08-29T15:28:39.011756+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:28:39.011756+0000","last_undegraded":"2024-08-29T15:28:39.011756+0000","last_fullsized":"2024-08-29T15:28:39.011756+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":255,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004559+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:28:41.004559+0000","last_peered":"2024-08-29T15:28:41.004559+0000","last_clean":"2024-08-29T15:28:41.004559+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:28:41.004559+0000","last_undegraded":"2024-08-29T15:28:41.004559+0000","last_fullsized":"2024-08-29T15:28:41.004559+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":118,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:38.828408+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:28:38.828408+0000","last_peered":"2024-08-29T15:28:38.828408+0000","last_clean":"2024-08-29T15:28:38.828408+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:28:38.828408+0000","last_undegraded":"2024-08-29T15:28:38.828408+0000","last_fullsized":"2024-08-29T15:28:38.828408+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":180,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828441+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:28:39.828441+0000","last_peered":"2024-08-29T15:28:39.828441+0000","last_clean":"2024-08-29T15:28:39.828441+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:28:39.828441+0000","last_undegraded":"2024-08-29T15:28:39.828441+0000","last_fullsized":"2024-08-29T15:28:39.828441+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":453,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004675+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:28:41.004675+0000","last_peered":"2024-08-29T15:28:41.004675+0000","last_clean":"2024-08-29T15:28:41.004675+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:28:41.004675+0000","last_undegraded":"2024-08-29T15:28:41.004675+0000","last_fullsized":"2024-08-29T15:28:41.004675+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":446,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833089+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:28:40.833089+0000","last_peered":"2024-08-29T15:28:40.833089+0000","last_clean":"2024-08-29T15:28:40.833089+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:28:40.833089+0000","last_undegraded":"2024-08-29T15:28:40.833089+0000","last_fullsized":"2024-08-29T15:28:40.833089+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012096+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:28:39.012096+0000","last_peered":"2024-08-29T15:28:39.012096+0000","last_clean":"2024-08-29T15:28:39.012096+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:28:39.012096+0000","last_undegraded":"2024-08-29T15:28:39.012096+0000","last_fullsized":"2024-08-29T15:28:39.012096+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":136,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:38.828275+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:28:38.828275+0000","last_peered":"2024-08-29T15:28:38.828275+0000","last_clean":"2024-08-29T15:28:38.828275+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:28:38.828275+0000","last_undegraded":"2024-08-29T15:28:38.828275+0000","last_fullsized":"2024-08-29T15:28:38.828275+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012300+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:28:39.012300+0000","last_peered":"2024-08-29T15:28:39.012300+0000","last_clean":"2024-08-29T15:28:39.012300+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:28:39.012300+0000","last_undegraded":"2024-08-29T15:28:39.012300+0000","last_fullsized":"2024-08-29T15:28:39.012300+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011798+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:28:39.011798+0000","last_peered":"2024-08-29T15:28:39.011798+0000","last_clean":"2024-08-29T15:28:39.011798+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:28:39.011798+0000","last_undegraded":"2024-08-29T15:28:39.011798+0000","last_fullsized":"2024-08-29T15:28:39.011798+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":278,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012828+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:28:39.012828+0000","last_peered":"2024-08-29T15:28:39.012828+0000","last_clean":"2024-08-29T15:28:39.012828+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:28:39.012828+0000","last_undegraded":"2024-08-29T15:28:39.012828+0000","last_fullsized":"2024-08-29T15:28:39.012828+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":450,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832981+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:28:40.832981+0000","last_peered":"2024-08-29T15:28:40.832981+0000","last_clean":"2024-08-29T15:28:40.832981+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:28:40.832981+0000","last_undegraded":"2024-08-29T15:28:40.832981+0000","last_fullsized":"2024-08-29T15:28:40.832981+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":441,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830806+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:28:39.830806+0000","last_peered":"2024-08-29T15:28:39.830806+0000","last_clean":"2024-08-29T15:28:39.830806+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:28:39.830806+0000","last_undegraded":"2024-08-29T15:28:39.830806+0000","last_fullsized":"2024-08-29T15:28:39.830806+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":542,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004415+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:28:41.004415+0000","last_peered":"2024-08-29T15:28:41.004415+0000","last_clean":"2024-08-29T15:28:41.004415+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:28:41.004415+0000","last_undegraded":"2024-08-29T15:28:41.004415+0000","last_fullsized":"2024-08-29T15:28:41.004415+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":765,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828113+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:28:39.828113+0000","last_peered":"2024-08-29T15:28:39.828113+0000","last_clean":"2024-08-29T15:28:39.828113+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:28:39.828113+0000","last_undegraded":"2024-08-29T15:28:39.828113+0000","last_fullsized":"2024-08-29T15:28:39.828113+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":273,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827931+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:28:39.827931+0000","last_peered":"2024-08-29T15:28:39.827931+0000","last_clean":"2024-08-29T15:28:39.827931+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:28:39.827931+0000","last_undegraded":"2024-08-29T15:28:39.827931+0000","last_fullsized":"2024-08-29T15:28:39.827931+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":134,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:38.828628+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:28:38.828628+0000","last_peered":"2024-08-29T15:28:38.828628+0000","last_clean":"2024-08-29T15:28:38.828628+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:28:38.828628+0000","last_undegraded":"2024-08-29T15:28:38.828628+0000","last_fullsized":"2024-08-29T15:28:38.828628+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":914,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830192+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:28:39.830192+0000","last_peered":"2024-08-29T15:28:39.830192+0000","last_clean":"2024-08-29T15:28:39.830192+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:28:39.830192+0000","last_undegraded":"2024-08-29T15:28:39.830192+0000","last_fullsized":"2024-08-29T15:28:39.830192+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":454,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004390+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:28:41.004390+0000","last_peered":"2024-08-29T15:28:41.004390+0000","last_clean":"2024-08-29T15:28:41.004390+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:28:41.004390+0000","last_undegraded":"2024-08-29T15:28:41.004390+0000","last_fullsized":"2024-08-29T15:28:41.004390+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":461,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004771+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:28:41.004771+0000","last_peered":"2024-08-29T15:28:41.004771+0000","last_clean":"2024-08-29T15:28:41.004771+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:28:41.004771+0000","last_undegraded":"2024-08-29T15:28:41.004771+0000","last_fullsized":"2024-08-29T15:28:41.004771+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":723,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833076+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:28:40.833076+0000","last_peered":"2024-08-29T15:28:40.833076+0000","last_clean":"2024-08-29T15:28:40.833076+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:28:40.833076+0000","last_undegraded":"2024-08-29T15:28:40.833076+0000","last_fullsized":"2024-08-29T15:28:40.833076+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":125,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:38.827465+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:28:38.827465+0000","last_peered":"2024-08-29T15:28:38.827465+0000","last_clean":"2024-08-29T15:28:38.827465+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:28:38.827465+0000","last_undegraded":"2024-08-29T15:28:38.827465+0000","last_fullsized":"2024-08-29T15:28:38.827465+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":486,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830559+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:28:39.830559+0000","last_peered":"2024-08-29T15:28:39.830559+0000","last_clean":"2024-08-29T15:28:39.830559+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:28:39.830559+0000","last_undegraded":"2024-08-29T15:28:39.830559+0000","last_fullsized":"2024-08-29T15:28:39.830559+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":233,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827884+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:28:39.827884+0000","last_peered":"2024-08-29T15:28:39.827884+0000","last_clean":"2024-08-29T15:28:39.827884+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:28:39.827884+0000","last_undegraded":"2024-08-29T15:28:39.827884+0000","last_fullsized":"2024-08-29T15:28:39.827884+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011947+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:28:39.011947+0000","last_peered":"2024-08-29T15:28:39.011947+0000","last_clean":"2024-08-29T15:28:39.011947+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:28:39.011947+0000","last_undegraded":"2024-08-29T15:28:39.011947+0000","last_fullsized":"2024-08-29T15:28:39.011947+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":140,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828287+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:28:39.828287+0000","last_peered":"2024-08-29T15:28:39.828287+0000","last_clean":"2024-08-29T15:28:39.828287+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:28:39.828287+0000","last_undegraded":"2024-08-29T15:28:39.828287+0000","last_fullsized":"2024-08-29T15:28:39.828287+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":165,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828378+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:28:39.828378+0000","last_peered":"2024-08-29T15:28:39.828378+0000","last_clean":"2024-08-29T15:28:39.828378+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:28:39.828378+0000","last_undegraded":"2024-08-29T15:28:39.828378+0000","last_fullsized":"2024-08-29T15:28:39.828378+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":879,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830893+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:28:39.830893+0000","last_peered":"2024-08-29T15:28:39.830893+0000","last_clean":"2024-08-29T15:28:39.830893+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:28:39.830893+0000","last_undegraded":"2024-08-29T15:28:39.830893+0000","last_fullsized":"2024-08-29T15:28:39.830893+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":456,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830531+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:28:39.830531+0000","last_peered":"2024-08-29T15:28:39.830531+0000","last_clean":"2024-08-29T15:28:39.830531+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:28:39.830531+0000","last_undegraded":"2024-08-29T15:28:39.830531+0000","last_fullsized":"2024-08-29T15:28:39.830531+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":461,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830367+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:28:39.830367+0000","last_peered":"2024-08-29T15:28:39.830367+0000","last_clean":"2024-08-29T15:28:39.830367+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:28:39.830367+0000","last_undegraded":"2024-08-29T15:28:39.830367+0000","last_fullsized":"2024-08-29T15:28:39.830367+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":744,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.047252+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:28:39.047252+0000","last_peered":"2024-08-29T15:28:39.047252+0000","last_clean":"2024-08-29T15:28:39.047252+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:28:39.047252+0000","last_undegraded":"2024-08-29T15:28:39.047252+0000","last_fullsized":"2024-08-29T15:28:39.047252+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":454,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004670+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:28:41.004670+0000","last_peered":"2024-08-29T15:28:41.004670+0000","last_clean":"2024-08-29T15:28:41.004670+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:28:41.004670+0000","last_undegraded":"2024-08-29T15:28:41.004670+0000","last_fullsized":"2024-08-29T15:28:41.004670+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":447,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833410+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:28:40.833410+0000","last_peered":"2024-08-29T15:28:40.833410+0000","last_clean":"2024-08-29T15:28:40.833410+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:28:40.833410+0000","last_undegraded":"2024-08-29T15:28:40.833410+0000","last_fullsized":"2024-08-29T15:28:40.833410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1089,"reported_epoch":291,"state":"peering","last_fresh":"2024-08-29T15:28:39.089477+0000","last_change":"2024-08-29T15:28:39.830636+0000","last_active":"2024-08-29T15:28:39.830636+0000","last_peered":"2024-08-29T15:28:39.089477+0000","last_clean":"2024-08-29T15:19:43.281167+0000","last_became_active":"2024-08-29T15:19:47.803300+0000","last_became_peered":"2024-08-29T15:19:47.803300+0000","last_unstale":"2024-08-29T15:28:39.089477+0000","last_undegraded":"2024-08-29T15:28:39.089477+0000","last_fullsized":"2024-08-29T15:19:47.794702+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":444,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832870+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:28:40.832870+0000","last_peered":"2024-08-29T15:28:40.832870+0000","last_clean":"2024-08-29T15:28:40.832870+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:28:40.832870+0000","last_undegraded":"2024-08-29T15:28:40.832870+0000","last_fullsized":"2024-08-29T15:28:40.832870+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":443,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004653+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:28:41.004653+0000","last_peered":"2024-08-29T15:28:41.004653+0000","last_clean":"2024-08-29T15:28:41.004653+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:28:41.004653+0000","last_undegraded":"2024-08-29T15:28:41.004653+0000","last_fullsized":"2024-08-29T15:28:41.004653+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":145,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:38.828751+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:28:38.828751+0000","last_peered":"2024-08-29T15:28:38.828751+0000","last_clean":"2024-08-29T15:28:38.828751+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:28:38.828751+0000","last_undegraded":"2024-08-29T15:28:38.828751+0000","last_fullsized":"2024-08-29T15:28:38.828751+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011820+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:28:39.011820+0000","last_peered":"2024-08-29T15:28:39.011820+0000","last_clean":"2024-08-29T15:28:39.011820+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:28:39.011820+0000","last_undegraded":"2024-08-29T15:28:39.011820+0000","last_fullsized":"2024-08-29T15:28:39.011820+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012003+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:28:39.012003+0000","last_peered":"2024-08-29T15:28:39.012003+0000","last_clean":"2024-08-29T15:28:39.012003+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:28:39.012003+0000","last_undegraded":"2024-08-29T15:28:39.012003+0000","last_fullsized":"2024-08-29T15:28:39.012003+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":82,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012128+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:28:39.012128+0000","last_peered":"2024-08-29T15:28:39.012128+0000","last_clean":"2024-08-29T15:28:39.012128+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:28:39.012128+0000","last_undegraded":"2024-08-29T15:28:39.012128+0000","last_fullsized":"2024-08-29T15:28:39.012128+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":449,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830331+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:28:39.830331+0000","last_peered":"2024-08-29T15:28:39.830331+0000","last_clean":"2024-08-29T15:28:39.830331+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:28:39.830331+0000","last_undegraded":"2024-08-29T15:28:39.830331+0000","last_fullsized":"2024-08-29T15:28:39.830331+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":210,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833296+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:28:40.833296+0000","last_peered":"2024-08-29T15:28:40.833296+0000","last_clean":"2024-08-29T15:28:40.833296+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:28:40.833296+0000","last_undegraded":"2024-08-29T15:28:40.833296+0000","last_fullsized":"2024-08-29T15:28:40.833296+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":162,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828028+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:28:39.828028+0000","last_peered":"2024-08-29T15:28:39.828028+0000","last_clean":"2024-08-29T15:28:39.828028+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:28:39.828028+0000","last_undegraded":"2024-08-29T15:28:39.828028+0000","last_fullsized":"2024-08-29T15:28:39.828028+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":410,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832911+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:28:40.832911+0000","last_peered":"2024-08-29T15:28:40.832911+0000","last_clean":"2024-08-29T15:28:40.832911+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:28:40.832911+0000","last_undegraded":"2024-08-29T15:28:40.832911+0000","last_fullsized":"2024-08-29T15:28:40.832911+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":464,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004825+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:28:41.004825+0000","last_peered":"2024-08-29T15:28:41.004825+0000","last_clean":"2024-08-29T15:28:41.004825+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:28:41.004825+0000","last_undegraded":"2024-08-29T15:28:41.004825+0000","last_fullsized":"2024-08-29T15:28:41.004825+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":462,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832865+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:28:40.832865+0000","last_peered":"2024-08-29T15:28:40.832865+0000","last_clean":"2024-08-29T15:28:40.832865+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:28:40.832865+0000","last_undegraded":"2024-08-29T15:28:40.832865+0000","last_fullsized":"2024-08-29T15:28:40.832865+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":133,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:38.828830+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:28:38.828830+0000","last_peered":"2024-08-29T15:28:38.828830+0000","last_clean":"2024-08-29T15:28:38.828830+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:28:38.828830+0000","last_undegraded":"2024-08-29T15:28:38.828830+0000","last_fullsized":"2024-08-29T15:28:38.828830+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":320,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004405+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:28:41.004405+0000","last_peered":"2024-08-29T15:28:41.004405+0000","last_clean":"2024-08-29T15:28:41.004405+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:28:41.004405+0000","last_undegraded":"2024-08-29T15:28:41.004405+0000","last_fullsized":"2024-08-29T15:28:41.004405+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":440,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004739+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:28:41.004739+0000","last_peered":"2024-08-29T15:28:41.004739+0000","last_clean":"2024-08-29T15:28:41.004739+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:28:41.004739+0000","last_undegraded":"2024-08-29T15:28:41.004739+0000","last_fullsized":"2024-08-29T15:28:41.004739+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":441,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830256+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:28:39.830256+0000","last_peered":"2024-08-29T15:28:39.830256+0000","last_clean":"2024-08-29T15:28:39.830256+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:28:39.830256+0000","last_undegraded":"2024-08-29T15:28:39.830256+0000","last_fullsized":"2024-08-29T15:28:39.830256+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":446,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833161+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:28:40.833161+0000","last_peered":"2024-08-29T15:28:40.833161+0000","last_clean":"2024-08-29T15:28:40.833161+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:28:40.833161+0000","last_undegraded":"2024-08-29T15:28:40.833161+0000","last_fullsized":"2024-08-29T15:28:40.833161+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.012032+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:28:39.012032+0000","last_peered":"2024-08-29T15:28:39.012032+0000","last_clean":"2024-08-29T15:28:39.012032+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:28:39.012032+0000","last_undegraded":"2024-08-29T15:28:39.012032+0000","last_fullsized":"2024-08-29T15:28:39.012032+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":465,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.830813+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:28:39.830813+0000","last_peered":"2024-08-29T15:28:39.830813+0000","last_clean":"2024-08-29T15:28:39.830813+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:28:39.830813+0000","last_undegraded":"2024-08-29T15:28:39.830813+0000","last_fullsized":"2024-08-29T15:28:39.830813+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":431,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004327+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:28:41.004327+0000","last_peered":"2024-08-29T15:28:41.004327+0000","last_clean":"2024-08-29T15:28:41.004327+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:28:41.004327+0000","last_undegraded":"2024-08-29T15:28:41.004327+0000","last_fullsized":"2024-08-29T15:28:41.004327+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011907+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:28:39.011907+0000","last_peered":"2024-08-29T15:28:39.011907+0000","last_clean":"2024-08-29T15:28:39.011907+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:28:39.011907+0000","last_undegraded":"2024-08-29T15:28:39.011907+0000","last_fullsized":"2024-08-29T15:28:39.011907+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":192,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828397+0000","last_change":"2024-08-29T15:28:35.519733+0000","last_active":"2024-08-29T15:28:39.828397+0000","last_peered":"2024-08-29T15:28:39.828397+0000","last_clean":"2024-08-29T15:28:39.828397+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:28:39.828397+0000","last_undegraded":"2024-08-29T15:28:39.828397+0000","last_fullsized":"2024-08-29T15:28:39.828397+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011998+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:28:39.011998+0000","last_peered":"2024-08-29T15:28:39.011998+0000","last_clean":"2024-08-29T15:28:39.011998+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:28:39.011998+0000","last_undegraded":"2024-08-29T15:28:39.011998+0000","last_fullsized":"2024-08-29T15:28:39.011998+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":192,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828466+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:28:39.828466+0000","last_peered":"2024-08-29T15:28:39.828466+0000","last_clean":"2024-08-29T15:28:39.828466+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:28:39.828466+0000","last_undegraded":"2024-08-29T15:28:39.828466+0000","last_fullsized":"2024-08-29T15:28:39.828466+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":478,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833178+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:28:40.833178+0000","last_peered":"2024-08-29T15:28:40.833178+0000","last_clean":"2024-08-29T15:28:40.833178+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:28:40.833178+0000","last_undegraded":"2024-08-29T15:28:40.833178+0000","last_fullsized":"2024-08-29T15:28:40.833178+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":479,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833342+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:28:40.833342+0000","last_peered":"2024-08-29T15:28:40.833342+0000","last_clean":"2024-08-29T15:28:40.833342+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:28:40.833342+0000","last_undegraded":"2024-08-29T15:28:40.833342+0000","last_fullsized":"2024-08-29T15:28:40.833342+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":72,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.011687+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:28:39.011687+0000","last_peered":"2024-08-29T15:28:39.011687+0000","last_clean":"2024-08-29T15:28:39.011687+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:28:39.011687+0000","last_undegraded":"2024-08-29T15:28:39.011687+0000","last_fullsized":"2024-08-29T15:28:39.011687+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":488,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004690+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:28:41.004690+0000","last_peered":"2024-08-29T15:28:41.004690+0000","last_clean":"2024-08-29T15:28:41.004690+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:28:41.004690+0000","last_undegraded":"2024-08-29T15:28:41.004690+0000","last_fullsized":"2024-08-29T15:28:41.004690+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":6528,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234408,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":372704,"kb_used_data":8524,"kb_used_omap":1,"kb_used_meta":364158,"kb_avail":93372448,"statfs":{"total":95995035648,"available":95613386752,"internally_reserved":0,"allocated":8728576,"data_stored":7002267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":372898251},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496039,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":242248,"kb_used_data":4408,"kb_used_omap":1,"kb_used_meta":237822,"kb_avail":93502904,"statfs":{"total":95995035648,"available":95746973696,"internally_reserved":0,"allocated":4513792,"data_stored":2799498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243530183},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426626,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":103248,"kb_used_data":3840,"kb_used_omap":8,"kb_used_meta":99383,"kb_avail":93641904,"statfs":{"total":95995035648,"available":95889309696,"internally_reserved":0,"allocated":3932160,"data_stored":2221077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":101769098},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019318,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":349948,"kb_used_data":3816,"kb_used_omap":1,"kb_used_meta":346110,"kb_avail":93395204,"statfs":{"total":95995035648,"available":95636688896,"internally_reserved":0,"allocated":3907584,"data_stored":2200503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":354417094},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435503,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":346448,"kb_used_data":3840,"kb_used_omap":4,"kb_used_meta":342587,"kb_avail":93398704,"statfs":{"total":95995035648,"available":95640272896,"internally_reserved":0,"allocated":3932160,"data_stored":2220909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":350809353},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903355,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":207752,"kb_used_data":3836,"kb_used_omap":10,"kb_used_meta":203893,"kb_avail":93537400,"statfs":{"total":95995035648,"available":95782297600,"internally_reserved":0,"allocated":3928064,"data_stored":2220255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":208786960},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762556,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":709952,"kb_used_data":425584,"kb_used_omap":7,"kb_used_meta":284344,"kb_avail":93035200,"statfs":{"total":95995035648,"available":95268044800,"internally_reserved":0,"allocated":435798016,"data_stored":434093819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":291168610},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926083,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":351184,"kb_used_data":8516,"kb_used_omap":7,"kb_used_meta":342648,"kb_avail":93393968,"statfs":{"total":95995035648,"available":95635423232,"internally_reserved":0,"allocated":8720384,"data_stored":7004724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":350871695},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:28:49.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:49.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:49.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:49.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:49.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:49.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:49.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:50.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:50.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:49 smithi027 ceph-mon[93921]: pgmap v1026: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 462 KiB/s, 1 keys/s, 2 objects/s recovering 2024-08-29T15:28:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:49 smithi027 ceph-mon[98970]: pgmap v1026: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 462 KiB/s, 1 keys/s, 2 objects/s recovering 2024-08-29T15:28:50.070 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:50.127 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:50.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:50.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:50.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:49 smithi171 ceph-mon[90031]: pgmap v1026: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 462 KiB/s, 1 keys/s, 2 objects/s recovering 2024-08-29T15:28:50.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:50.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:50.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:50.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:50.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:50.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:50.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:50.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:50.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:50.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:50.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:50.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:50.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:50.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:51.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:51.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:51.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:51.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:51.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:51.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:51.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:51.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:51.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:51.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:51.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:51.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:51.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:51.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:51.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:51.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:51.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:51.892 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:51.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:51.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:51.937 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:51 smithi027 ceph-mon[98970]: pgmap v1027: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:28:51.938 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:51 smithi027 ceph-mon[93921]: pgmap v1027: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:28:52.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:52.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:52.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:52.172 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:51 smithi171 ceph-mon[90031]: pgmap v1027: 73 pgs: 73 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:28:52.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:52.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:52.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:52.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:52.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:52.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:52.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:52.653 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:52.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:28:52.678 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:52.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:52.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:52.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:52.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:52.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:52.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:53.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:53.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:53.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:53.083 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:28:53.083 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:28:53.100 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:28:53.100 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:28:53.116 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:28:53.116 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:28:53.132 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:28:53.133 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:28:53.150 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:28:53.150 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:28:53.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:53.167 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:28:53.167 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:28:53.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:53.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:53.185 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:28:53.185 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:28:53.202 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:28:53.202 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:28:53.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:53.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:53.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:53.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:53.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:53.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:53.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:53.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:53.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:53.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:53.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:53.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:53.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:53.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:53.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:54.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:54.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:53 smithi027 ceph-mon[98970]: pgmap v1028: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:54.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/460554882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:54.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:53 smithi027 ceph-mon[93921]: pgmap v1028: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:54.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/460554882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:54.067 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:54.082 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:28:54.082 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:28:54.082 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:28:54.082 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:28:54.083 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:28:54.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:54.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:54.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:53 smithi171 ceph-mon[90031]: pgmap v1028: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/460554882' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:28:54.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:54.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:54.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:54.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:54.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:54.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:54.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:54.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:54.698 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:54.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:54.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:54.824 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:54.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:54.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:54.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:54.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:54.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:55.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.077 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:55.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:55.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:55.201 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:55.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:55.325 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:55.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:55.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:55.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:55.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:55.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:55.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:55.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:55.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:55.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:55.827 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:55.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:55.952 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:55.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:55.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:56.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:56.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:56.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:56.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:56.204 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:56.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:55 smithi171 ceph-mon[90031]: pgmap v1029: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:56.306 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:55 smithi027 ceph-mon[93921]: pgmap v1029: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:55 smithi027 ceph-mon[98970]: pgmap v1029: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:56.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:56.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:56.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:56.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:56.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:28:56.560 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:56.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:56.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:56.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:56.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:56.731 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:56.766 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:28:56.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:56.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:56.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:56.912 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:28:56.917 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_2' created 2024-08-29T15:28:56.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:56.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:28:56.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:56.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:57.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:57.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:57.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:57.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2868557478' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1900937537' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-29T15:28:57.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:56 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-29T15:28:57.315 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2868557478' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:57.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1900937537' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-29T15:28:57.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:56 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-29T15:28:57.316 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2868557478' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:28:57.316 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1900937537' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-29T15:28:57.316 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:56 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-29T15:28:57.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:57.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:57.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:57.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:57.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:57.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:57.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:57.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:57.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:57.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:57.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:57.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:57.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:28:57.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:28:57.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[93921]: pgmap v1030: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:57.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:57.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2868557478' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:28:57.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-29T15:28:57.911 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[93921]: osdmap e294: 8 total, 8 up, 8 in 2024-08-29T15:28:57.912 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[98970]: pgmap v1030: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:57.912 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:57.912 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2868557478' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:28:57.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:57.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:57.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:57.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:58.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:58.099 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:58.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:58.223 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:57 smithi171 ceph-mon[90031]: pgmap v1030: 73 pgs: 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:28:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:57 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:28:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2868557478' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:28:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:57 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-29T15:28:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:57 smithi171 ceph-mon[90031]: osdmap e294: 8 total, 8 up, 8 in 2024-08-29T15:28:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-29T15:28:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:57 smithi027 ceph-mon[98970]: osdmap e294: 8 total, 8 up, 8 in 2024-08-29T15:28:58.325 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:58.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:58.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:28:58.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:28:58.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:58.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:58.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:58.726 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:58.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:58.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:58.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:58.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:58.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:58.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:58.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:59.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:28:59.104 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:59.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:28:59.110 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:28:59.111 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:28:59.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:59.228 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:59.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:58 smithi171 ceph-mon[90031]: osdmap e295: 8 total, 8 up, 8 in 2024-08-29T15:28:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:28:59] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-08-29T15:28:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:58 smithi027 ceph-mon[98970]: osdmap e295: 8 total, 8 up, 8 in 2024-08-29T15:28:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:58 smithi027 ceph-mon[93921]: osdmap e295: 8 total, 8 up, 8 in 2024-08-29T15:28:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:28:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:28:59.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:28:59.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:28:59.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:28:59.475 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:28:59.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:28:59.577 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:28:59.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:28:59.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:59.730 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:59.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:28:59.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:28:59.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:59.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:28:59.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-08-29T15:28:59.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:28:59.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:28:59.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:28:59.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:00.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:00.104 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:00.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:00.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:00.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:00.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:00.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:59 smithi171 ceph-mon[90031]: pgmap v1033: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:59 smithi171 ceph-mon[90031]: osdmap e296: 8 total, 8 up, 8 in 2024-08-29T15:29:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:59 smithi171 ceph-mon[90031]: from='client.36240 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:28:59 smithi171 ceph-mon[90031]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:29:00.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[98970]: pgmap v1033: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:00.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[98970]: osdmap e296: 8 total, 8 up, 8 in 2024-08-29T15:29:00.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[98970]: from='client.36240 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:00.296 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[98970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:29:00.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[93921]: pgmap v1033: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:00.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[93921]: osdmap e296: 8 total, 8 up, 8 in 2024-08-29T15:29:00.296 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[93921]: from='client.36240 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:00.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:28:59 smithi027 ceph-mon[93921]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:29:00.332 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:00.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:00.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:00.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:00.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:00.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:00.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:00.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:00.611 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:00.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:00.677 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:00.699 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1034,"stamp":"2024-08-29T15:28:58.920754+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6365,"num_read_kb":13822,"num_write":1123,"num_write_kb":15041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":10166259,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1182,"ondisk_log_size":1182,"up":222,"acting":222,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":221,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2658848,"kb_used_data":437748,"kb_used_omap":43,"kb_used_meta":2220948,"kb_avail":747302368,"statfs":{"total":767960285184,"available":765237624832,"internally_reserved":0,"allocated":448253952,"data_stored":434523566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2274251244},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.223847"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":472,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833145+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:28:40.833145+0000","last_peered":"2024-08-29T15:28:40.833145+0000","last_clean":"2024-08-29T15:28:40.833145+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:28:40.833145+0000","last_undegraded":"2024-08-29T15:28:40.833145+0000","last_fullsized":"2024-08-29T15:28:40.833145+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":80,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828156+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:28:39.828156+0000","last_peered":"2024-08-29T15:28:39.828156+0000","last_clean":"2024-08-29T15:28:39.828156+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:28:39.828156+0000","last_undegraded":"2024-08-29T15:28:39.828156+0000","last_fullsized":"2024-08-29T15:28:39.828156+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":162,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828398+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:28:39.828398+0000","last_peered":"2024-08-29T15:28:39.828398+0000","last_clean":"2024-08-29T15:28:39.828398+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:28:39.828398+0000","last_undegraded":"2024-08-29T15:28:39.828398+0000","last_fullsized":"2024-08-29T15:28:39.828398+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":439,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004446+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:28:41.004446+0000","last_peered":"2024-08-29T15:28:41.004446+0000","last_clean":"2024-08-29T15:28:41.004446+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:28:41.004446+0000","last_undegraded":"2024-08-29T15:28:41.004446+0000","last_fullsized":"2024-08-29T15:28:41.004446+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":438,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004643+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:28:41.004643+0000","last_peered":"2024-08-29T15:28:41.004643+0000","last_clean":"2024-08-29T15:28:41.004643+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:28:41.004643+0000","last_undegraded":"2024-08-29T15:28:41.004643+0000","last_fullsized":"2024-08-29T15:28:41.004643+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828055+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:28:39.828055+0000","last_peered":"2024-08-29T15:28:39.828055+0000","last_clean":"2024-08-29T15:28:39.828055+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:28:39.828055+0000","last_undegraded":"2024-08-29T15:28:39.828055+0000","last_fullsized":"2024-08-29T15:28:39.828055+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.835002+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:28:40.835002+0000","last_peered":"2024-08-29T15:28:40.835002+0000","last_clean":"2024-08-29T15:28:40.835002+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:28:40.835002+0000","last_undegraded":"2024-08-29T15:28:40.835002+0000","last_fullsized":"2024-08-29T15:28:40.835002+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":164,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.832033+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:28:39.832033+0000","last_peered":"2024-08-29T15:28:39.832033+0000","last_clean":"2024-08-29T15:28:39.832033+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:28:39.832033+0000","last_undegraded":"2024-08-29T15:28:39.832033+0000","last_fullsized":"2024-08-29T15:28:39.832033+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":78,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833653+0000","last_change":"2024-08-29T15:28:35.524176+0000","last_active":"2024-08-29T15:28:40.833653+0000","last_peered":"2024-08-29T15:28:40.833653+0000","last_clean":"2024-08-29T15:28:40.833653+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:28:40.833653+0000","last_undegraded":"2024-08-29T15:28:40.833653+0000","last_fullsized":"2024-08-29T15:28:40.833653+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":181,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827819+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:28:39.827819+0000","last_peered":"2024-08-29T15:28:39.827819+0000","last_clean":"2024-08-29T15:28:39.827819+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:28:39.827819+0000","last_undegraded":"2024-08-29T15:28:39.827819+0000","last_fullsized":"2024-08-29T15:28:39.827819+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":78,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833983+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:28:40.833983+0000","last_peered":"2024-08-29T15:28:40.833983+0000","last_clean":"2024-08-29T15:28:40.833983+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:28:40.833983+0000","last_undegraded":"2024-08-29T15:28:40.833983+0000","last_fullsized":"2024-08-29T15:28:40.833983+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":74,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827892+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:28:39.827892+0000","last_peered":"2024-08-29T15:28:39.827892+0000","last_clean":"2024-08-29T15:28:39.827892+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:28:39.827892+0000","last_undegraded":"2024-08-29T15:28:39.827892+0000","last_fullsized":"2024-08-29T15:28:39.827892+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":255,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004559+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:28:41.004559+0000","last_peered":"2024-08-29T15:28:41.004559+0000","last_clean":"2024-08-29T15:28:41.004559+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:28:41.004559+0000","last_undegraded":"2024-08-29T15:28:41.004559+0000","last_fullsized":"2024-08-29T15:28:41.004559+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.832077+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:28:39.832077+0000","last_peered":"2024-08-29T15:28:39.832077+0000","last_clean":"2024-08-29T15:28:39.832077+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:28:39.832077+0000","last_undegraded":"2024-08-29T15:28:39.832077+0000","last_fullsized":"2024-08-29T15:28:39.832077+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833952+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:28:40.833952+0000","last_peered":"2024-08-29T15:28:40.833952+0000","last_clean":"2024-08-29T15:28:40.833952+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:28:40.833952+0000","last_undegraded":"2024-08-29T15:28:40.833952+0000","last_fullsized":"2024-08-29T15:28:40.833952+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":453,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004675+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:28:41.004675+0000","last_peered":"2024-08-29T15:28:41.004675+0000","last_clean":"2024-08-29T15:28:41.004675+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:28:41.004675+0000","last_undegraded":"2024-08-29T15:28:41.004675+0000","last_fullsized":"2024-08-29T15:28:41.004675+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":446,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833089+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:28:40.833089+0000","last_peered":"2024-08-29T15:28:40.833089+0000","last_clean":"2024-08-29T15:28:40.833089+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:28:40.833089+0000","last_undegraded":"2024-08-29T15:28:40.833089+0000","last_fullsized":"2024-08-29T15:28:40.833089+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":74,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833157+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:28:40.833157+0000","last_peered":"2024-08-29T15:28:40.833157+0000","last_clean":"2024-08-29T15:28:40.833157+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:28:40.833157+0000","last_undegraded":"2024-08-29T15:28:40.833157+0000","last_fullsized":"2024-08-29T15:28:40.833157+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":1,"reported_epoch":294,"state":"creating+peering","last_fresh":"2024-08-29T15:28:56.917048+0000","last_change":"2024-08-29T15:28:56.917048+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.917048+0000","last_undegraded":"2024-08-29T15:28:56.917048+0000","last_fullsized":"2024-08-29T15:28:56.917048+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":137,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.832001+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:28:39.832001+0000","last_peered":"2024-08-29T15:28:39.832001+0000","last_clean":"2024-08-29T15:28:39.832001+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:28:39.832001+0000","last_undegraded":"2024-08-29T15:28:39.832001+0000","last_fullsized":"2024-08-29T15:28:39.832001+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":74,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833546+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:28:40.833546+0000","last_peered":"2024-08-29T15:28:40.833546+0000","last_clean":"2024-08-29T15:28:40.833546+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:28:40.833546+0000","last_undegraded":"2024-08-29T15:28:40.833546+0000","last_fullsized":"2024-08-29T15:28:40.833546+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":74,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827975+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:28:39.827975+0000","last_peered":"2024-08-29T15:28:39.827975+0000","last_clean":"2024-08-29T15:28:39.827975+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:28:39.827975+0000","last_undegraded":"2024-08-29T15:28:39.827975+0000","last_fullsized":"2024-08-29T15:28:39.827975+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":279,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828903+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:28:39.828903+0000","last_peered":"2024-08-29T15:28:39.828903+0000","last_clean":"2024-08-29T15:28:39.828903+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:28:39.828903+0000","last_undegraded":"2024-08-29T15:28:39.828903+0000","last_fullsized":"2024-08-29T15:28:39.828903+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":450,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832981+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:28:40.832981+0000","last_peered":"2024-08-29T15:28:40.832981+0000","last_clean":"2024-08-29T15:28:40.832981+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:28:40.832981+0000","last_undegraded":"2024-08-29T15:28:40.832981+0000","last_fullsized":"2024-08-29T15:28:40.832981+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":443,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.835031+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:28:40.835031+0000","last_peered":"2024-08-29T15:28:40.835031+0000","last_clean":"2024-08-29T15:28:40.835031+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:28:40.835031+0000","last_undegraded":"2024-08-29T15:28:40.835031+0000","last_fullsized":"2024-08-29T15:28:40.835031+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":542,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004415+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:28:41.004415+0000","last_peered":"2024-08-29T15:28:41.004415+0000","last_clean":"2024-08-29T15:28:41.004415+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:28:41.004415+0000","last_undegraded":"2024-08-29T15:28:41.004415+0000","last_fullsized":"2024-08-29T15:28:41.004415+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":765,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828113+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:28:39.828113+0000","last_peered":"2024-08-29T15:28:39.828113+0000","last_clean":"2024-08-29T15:28:39.828113+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:28:39.828113+0000","last_undegraded":"2024-08-29T15:28:39.828113+0000","last_fullsized":"2024-08-29T15:28:39.828113+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":273,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827931+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:28:39.827931+0000","last_peered":"2024-08-29T15:28:39.827931+0000","last_clean":"2024-08-29T15:28:39.827931+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:28:39.827931+0000","last_undegraded":"2024-08-29T15:28:39.827931+0000","last_fullsized":"2024-08-29T15:28:39.827931+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":135,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.832205+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:28:39.832205+0000","last_peered":"2024-08-29T15:28:39.832205+0000","last_clean":"2024-08-29T15:28:39.832205+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:28:39.832205+0000","last_undegraded":"2024-08-29T15:28:39.832205+0000","last_fullsized":"2024-08-29T15:28:39.832205+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":916,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834444+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:28:40.834444+0000","last_peered":"2024-08-29T15:28:40.834444+0000","last_clean":"2024-08-29T15:28:40.834444+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:28:40.834444+0000","last_undegraded":"2024-08-29T15:28:40.834444+0000","last_fullsized":"2024-08-29T15:28:40.834444+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":454,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004390+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:28:41.004390+0000","last_peered":"2024-08-29T15:28:41.004390+0000","last_clean":"2024-08-29T15:28:41.004390+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:28:41.004390+0000","last_undegraded":"2024-08-29T15:28:41.004390+0000","last_fullsized":"2024-08-29T15:28:41.004390+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":461,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004771+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:28:41.004771+0000","last_peered":"2024-08-29T15:28:41.004771+0000","last_clean":"2024-08-29T15:28:41.004771+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:28:41.004771+0000","last_undegraded":"2024-08-29T15:28:41.004771+0000","last_fullsized":"2024-08-29T15:28:41.004771+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":723,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833076+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:28:40.833076+0000","last_peered":"2024-08-29T15:28:40.833076+0000","last_clean":"2024-08-29T15:28:40.833076+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:28:40.833076+0000","last_undegraded":"2024-08-29T15:28:40.833076+0000","last_fullsized":"2024-08-29T15:28:40.833076+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":126,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.831847+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:28:39.831847+0000","last_peered":"2024-08-29T15:28:39.831847+0000","last_clean":"2024-08-29T15:28:39.831847+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:28:39.831847+0000","last_undegraded":"2024-08-29T15:28:39.831847+0000","last_fullsized":"2024-08-29T15:28:39.831847+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":488,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834938+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:28:40.834938+0000","last_peered":"2024-08-29T15:28:40.834938+0000","last_clean":"2024-08-29T15:28:40.834938+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:28:40.834938+0000","last_undegraded":"2024-08-29T15:28:40.834938+0000","last_fullsized":"2024-08-29T15:28:40.834938+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":233,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827884+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:28:39.827884+0000","last_peered":"2024-08-29T15:28:39.827884+0000","last_clean":"2024-08-29T15:28:39.827884+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:28:39.827884+0000","last_undegraded":"2024-08-29T15:28:39.827884+0000","last_fullsized":"2024-08-29T15:28:39.827884+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":74,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828093+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:28:39.828093+0000","last_peered":"2024-08-29T15:28:39.828093+0000","last_clean":"2024-08-29T15:28:39.828093+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:28:39.828093+0000","last_undegraded":"2024-08-29T15:28:39.828093+0000","last_fullsized":"2024-08-29T15:28:39.828093+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":140,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828287+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:28:39.828287+0000","last_peered":"2024-08-29T15:28:39.828287+0000","last_clean":"2024-08-29T15:28:39.828287+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:28:39.828287+0000","last_undegraded":"2024-08-29T15:28:39.828287+0000","last_fullsized":"2024-08-29T15:28:39.828287+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":165,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828378+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:28:39.828378+0000","last_peered":"2024-08-29T15:28:39.828378+0000","last_clean":"2024-08-29T15:28:39.828378+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:28:39.828378+0000","last_undegraded":"2024-08-29T15:28:39.828378+0000","last_fullsized":"2024-08-29T15:28:39.828378+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":881,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834762+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:28:40.834762+0000","last_peered":"2024-08-29T15:28:40.834762+0000","last_clean":"2024-08-29T15:28:40.834762+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:28:40.834762+0000","last_undegraded":"2024-08-29T15:28:40.834762+0000","last_fullsized":"2024-08-29T15:28:40.834762+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":458,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834610+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:28:40.834610+0000","last_peered":"2024-08-29T15:28:40.834610+0000","last_clean":"2024-08-29T15:28:40.834610+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:28:40.834610+0000","last_undegraded":"2024-08-29T15:28:40.834610+0000","last_fullsized":"2024-08-29T15:28:40.834610+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":463,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834753+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:28:40.834753+0000","last_peered":"2024-08-29T15:28:40.834753+0000","last_clean":"2024-08-29T15:28:40.834753+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:28:40.834753+0000","last_undegraded":"2024-08-29T15:28:40.834753+0000","last_fullsized":"2024-08-29T15:28:40.834753+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":745,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.951343+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:28:39.951343+0000","last_peered":"2024-08-29T15:28:39.951343+0000","last_clean":"2024-08-29T15:28:39.951343+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:28:39.951343+0000","last_undegraded":"2024-08-29T15:28:39.951343+0000","last_fullsized":"2024-08-29T15:28:39.951343+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":454,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004670+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:28:41.004670+0000","last_peered":"2024-08-29T15:28:41.004670+0000","last_clean":"2024-08-29T15:28:41.004670+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:28:41.004670+0000","last_undegraded":"2024-08-29T15:28:41.004670+0000","last_fullsized":"2024-08-29T15:28:41.004670+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":447,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833410+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:28:40.833410+0000","last_peered":"2024-08-29T15:28:40.833410+0000","last_clean":"2024-08-29T15:28:40.833410+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:28:40.833410+0000","last_undegraded":"2024-08-29T15:28:40.833410+0000","last_fullsized":"2024-08-29T15:28:40.833410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1099,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.836972+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:28:40.836972+0000","last_peered":"2024-08-29T15:28:40.836972+0000","last_clean":"2024-08-29T15:28:40.836972+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:28:40.836972+0000","last_undegraded":"2024-08-29T15:28:40.836972+0000","last_fullsized":"2024-08-29T15:28:40.836972+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":444,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832870+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:28:40.832870+0000","last_peered":"2024-08-29T15:28:40.832870+0000","last_clean":"2024-08-29T15:28:40.832870+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:28:40.832870+0000","last_undegraded":"2024-08-29T15:28:40.832870+0000","last_fullsized":"2024-08-29T15:28:40.832870+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-29T15:28:56.906146+0000","last_change":"2024-08-29T15:28:56.906146+0000","last_active":"2024-08-29T15:28:56.906146+0000","last_peered":"2024-08-29T15:28:56.906146+0000","last_clean":"2024-08-29T15:28:56.906146+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-29T15:28:56.906146+0000","last_undegraded":"2024-08-29T15:28:56.906146+0000","last_fullsized":"2024-08-29T15:28:56.906146+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":443,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004653+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:28:41.004653+0000","last_peered":"2024-08-29T15:28:41.004653+0000","last_clean":"2024-08-29T15:28:41.004653+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:28:41.004653+0000","last_undegraded":"2024-08-29T15:28:41.004653+0000","last_fullsized":"2024-08-29T15:28:41.004653+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":146,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.832183+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:28:39.832183+0000","last_peered":"2024-08-29T15:28:39.832183+0000","last_clean":"2024-08-29T15:28:39.832183+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:28:39.832183+0000","last_undegraded":"2024-08-29T15:28:39.832183+0000","last_fullsized":"2024-08-29T15:28:39.832183+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":74,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828003+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:28:39.828003+0000","last_peered":"2024-08-29T15:28:39.828003+0000","last_clean":"2024-08-29T15:28:39.828003+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:28:39.828003+0000","last_undegraded":"2024-08-29T15:28:39.828003+0000","last_fullsized":"2024-08-29T15:28:39.828003+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828179+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:28:39.828179+0000","last_peered":"2024-08-29T15:28:39.828179+0000","last_clean":"2024-08-29T15:28:39.828179+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:28:39.828179+0000","last_undegraded":"2024-08-29T15:28:39.828179+0000","last_fullsized":"2024-08-29T15:28:39.828179+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":83,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828643+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:28:39.828643+0000","last_peered":"2024-08-29T15:28:39.828643+0000","last_clean":"2024-08-29T15:28:39.828643+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:28:39.828643+0000","last_undegraded":"2024-08-29T15:28:39.828643+0000","last_fullsized":"2024-08-29T15:28:39.828643+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":452,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834576+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:28:40.834576+0000","last_peered":"2024-08-29T15:28:40.834576+0000","last_clean":"2024-08-29T15:28:40.834576+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:28:40.834576+0000","last_undegraded":"2024-08-29T15:28:40.834576+0000","last_fullsized":"2024-08-29T15:28:40.834576+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":210,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833296+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:28:40.833296+0000","last_peered":"2024-08-29T15:28:40.833296+0000","last_clean":"2024-08-29T15:28:40.833296+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:28:40.833296+0000","last_undegraded":"2024-08-29T15:28:40.833296+0000","last_fullsized":"2024-08-29T15:28:40.833296+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":162,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828028+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:28:39.828028+0000","last_peered":"2024-08-29T15:28:39.828028+0000","last_clean":"2024-08-29T15:28:39.828028+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:28:39.828028+0000","last_undegraded":"2024-08-29T15:28:39.828028+0000","last_fullsized":"2024-08-29T15:28:39.828028+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":410,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832911+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:28:40.832911+0000","last_peered":"2024-08-29T15:28:40.832911+0000","last_clean":"2024-08-29T15:28:40.832911+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:28:40.832911+0000","last_undegraded":"2024-08-29T15:28:40.832911+0000","last_fullsized":"2024-08-29T15:28:40.832911+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":464,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004825+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:28:41.004825+0000","last_peered":"2024-08-29T15:28:41.004825+0000","last_clean":"2024-08-29T15:28:41.004825+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:28:41.004825+0000","last_undegraded":"2024-08-29T15:28:41.004825+0000","last_fullsized":"2024-08-29T15:28:41.004825+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":462,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.832865+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:28:40.832865+0000","last_peered":"2024-08-29T15:28:40.832865+0000","last_clean":"2024-08-29T15:28:40.832865+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:28:40.832865+0000","last_undegraded":"2024-08-29T15:28:40.832865+0000","last_fullsized":"2024-08-29T15:28:40.832865+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.832195+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:28:39.832195+0000","last_peered":"2024-08-29T15:28:39.832195+0000","last_clean":"2024-08-29T15:28:39.832195+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:28:39.832195+0000","last_undegraded":"2024-08-29T15:28:39.832195+0000","last_fullsized":"2024-08-29T15:28:39.832195+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":320,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004405+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:28:41.004405+0000","last_peered":"2024-08-29T15:28:41.004405+0000","last_clean":"2024-08-29T15:28:41.004405+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:28:41.004405+0000","last_undegraded":"2024-08-29T15:28:41.004405+0000","last_fullsized":"2024-08-29T15:28:41.004405+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":440,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004739+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:28:41.004739+0000","last_peered":"2024-08-29T15:28:41.004739+0000","last_clean":"2024-08-29T15:28:41.004739+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:28:41.004739+0000","last_undegraded":"2024-08-29T15:28:41.004739+0000","last_fullsized":"2024-08-29T15:28:41.004739+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":443,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834458+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:28:40.834458+0000","last_peered":"2024-08-29T15:28:40.834458+0000","last_clean":"2024-08-29T15:28:40.834458+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:28:40.834458+0000","last_undegraded":"2024-08-29T15:28:40.834458+0000","last_fullsized":"2024-08-29T15:28:40.834458+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":447,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833204+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:28:40.833204+0000","last_peered":"2024-08-29T15:28:40.833204+0000","last_clean":"2024-08-29T15:28:40.833204+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:28:40.833204+0000","last_undegraded":"2024-08-29T15:28:40.833204+0000","last_fullsized":"2024-08-29T15:28:40.833204+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828141+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:28:39.828141+0000","last_peered":"2024-08-29T15:28:39.828141+0000","last_clean":"2024-08-29T15:28:39.828141+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:28:39.828141+0000","last_undegraded":"2024-08-29T15:28:39.828141+0000","last_fullsized":"2024-08-29T15:28:39.828141+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":467,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.834866+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:28:40.834866+0000","last_peered":"2024-08-29T15:28:40.834866+0000","last_clean":"2024-08-29T15:28:40.834866+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:28:40.834866+0000","last_undegraded":"2024-08-29T15:28:40.834866+0000","last_fullsized":"2024-08-29T15:28:40.834866+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":431,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004327+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:28:41.004327+0000","last_peered":"2024-08-29T15:28:41.004327+0000","last_clean":"2024-08-29T15:28:41.004327+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:28:41.004327+0000","last_undegraded":"2024-08-29T15:28:41.004327+0000","last_fullsized":"2024-08-29T15:28:41.004327+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828076+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:28:39.828076+0000","last_peered":"2024-08-29T15:28:39.828076+0000","last_clean":"2024-08-29T15:28:39.828076+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:28:39.828076+0000","last_undegraded":"2024-08-29T15:28:39.828076+0000","last_fullsized":"2024-08-29T15:28:39.828076+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":192,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828397+0000","last_change":"2024-08-29T15:28:35.519733+0000","last_active":"2024-08-29T15:28:39.828397+0000","last_peered":"2024-08-29T15:28:39.828397+0000","last_clean":"2024-08-29T15:28:39.828397+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:28:39.828397+0000","last_undegraded":"2024-08-29T15:28:39.828397+0000","last_fullsized":"2024-08-29T15:28:39.828397+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828215+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:28:39.828215+0000","last_peered":"2024-08-29T15:28:39.828215+0000","last_clean":"2024-08-29T15:28:39.828215+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:28:39.828215+0000","last_undegraded":"2024-08-29T15:28:39.828215+0000","last_fullsized":"2024-08-29T15:28:39.828215+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":192,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.828466+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:28:39.828466+0000","last_peered":"2024-08-29T15:28:39.828466+0000","last_clean":"2024-08-29T15:28:39.828466+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:28:39.828466+0000","last_undegraded":"2024-08-29T15:28:39.828466+0000","last_fullsized":"2024-08-29T15:28:39.828466+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":478,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833178+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:28:40.833178+0000","last_peered":"2024-08-29T15:28:40.833178+0000","last_clean":"2024-08-29T15:28:40.833178+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:28:40.833178+0000","last_undegraded":"2024-08-29T15:28:40.833178+0000","last_fullsized":"2024-08-29T15:28:40.833178+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":479,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:40.833342+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:28:40.833342+0000","last_peered":"2024-08-29T15:28:40.833342+0000","last_clean":"2024-08-29T15:28:40.833342+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:28:40.833342+0000","last_undegraded":"2024-08-29T15:28:40.833342+0000","last_fullsized":"2024-08-29T15:28:40.833342+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-08-29T15:28:39.827819+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:28:39.827819+0000","last_peered":"2024-08-29T15:28:39.827819+0000","last_clean":"2024-08-29T15:28:39.827819+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:28:39.827819+0000","last_undegraded":"2024-08-29T15:28:39.827819+0000","last_fullsized":"2024-08-29T15:28:39.827819+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":488,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-08-29T15:28:41.004690+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:28:41.004690+0000","last_peered":"2024-08-29T15:28:41.004690+0000","last_clean":"2024-08-29T15:28:41.004690+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:28:41.004690+0000","last_undegraded":"2024-08-29T15:28:41.004690+0000","last_fullsized":"2024-08-29T15:28:41.004690+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":1},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":6528,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234411,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":372708,"kb_used_data":8536,"kb_used_omap":1,"kb_used_meta":364158,"kb_avail":93372444,"statfs":{"total":95995035648,"available":95613382656,"internally_reserved":0,"allocated":8740864,"data_stored":7010381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":372898251},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496042,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":242268,"kb_used_data":4432,"kb_used_omap":1,"kb_used_meta":237822,"kb_avail":93502884,"statfs":{"total":95995035648,"available":95746953216,"internally_reserved":0,"allocated":4538368,"data_stored":2815747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243530183},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426629,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":103248,"kb_used_data":3840,"kb_used_omap":8,"kb_used_meta":99383,"kb_avail":93641904,"statfs":{"total":95995035648,"available":95889309696,"internally_reserved":0,"allocated":3932160,"data_stored":2221077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":101769098},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019321,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":354084,"kb_used_data":7952,"kb_used_omap":1,"kb_used_meta":346110,"kb_avail":93391068,"statfs":{"total":95995035648,"available":95632453632,"internally_reserved":0,"allocated":8142848,"data_stored":6419740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":354417094},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435506,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":346448,"kb_used_data":3840,"kb_used_omap":4,"kb_used_meta":342587,"kb_avail":93398704,"statfs":{"total":95995035648,"available":95640272896,"internally_reserved":0,"allocated":3932160,"data_stored":2220909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":350809353},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903358,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":207752,"kb_used_data":3836,"kb_used_omap":10,"kb_used_meta":203893,"kb_avail":93537400,"statfs":{"total":95995035648,"available":95782297600,"internally_reserved":0,"allocated":3928064,"data_stored":2220255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":208786960},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762559,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":681156,"kb_used_data":396796,"kb_used_omap":7,"kb_used_meta":284344,"kb_avail":93063996,"statfs":{"total":95995035648,"available":95297531904,"internally_reserved":0,"allocated":406319104,"data_stored":404610733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":291168610},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926085,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":351184,"kb_used_data":8516,"kb_used_omap":7,"kb_used_meta":342648,"kb_avail":93393968,"statfs":{"total":95995035648,"available":95635423232,"internally_reserved":0,"allocated":8720384,"data_stored":7004724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":350871695},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:29:00.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph status --format=json 2024-08-29T15:29:00.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:00.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:00.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:00.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:00.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:00.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:00.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:00.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:01.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:01.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:01.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:01.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:01.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:01.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:01.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:01.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.407 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:01.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:01.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:29:01.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:01.530 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:01.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:01.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.656 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:01.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:01.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:01.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:01.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:01.908 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:01.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:02.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:02.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:02.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:02.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:02.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:02.166 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:01 smithi027 ceph-mon[98970]: pgmap v1035: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:02.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:01 smithi027 ceph-mon[93921]: pgmap v1035: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:02.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:01 smithi171 ceph-mon[90031]: pgmap v1035: 89 pgs: 1 creating+peering, 5 unknown, 83 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:02.288 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:02.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:02.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:02.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:02.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:02.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:02.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:02.590 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:02.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:02.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:02.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:02.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:02.828 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:02.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:02.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:02.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:03.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:03.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:03.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:03.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:03.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:03.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:03.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:03.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:03.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:03.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:03.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:03.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:03.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:03.608 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:29:03.608 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:29:03.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:29:03.608 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:29:03.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.608 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(6.3 285/218 [PushOp(6:c2314182:::benchmark_data_smithi027_169445_object115231:head, version: 188'7198, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:c2314182:::benchmark_data_smithi027_169445_object115231:head@188'7198, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-29T15:29:03.608 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:13.902494+0000", 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.700491743999997, 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039204324999999998, 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.902494+0000", 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.902494+0000", 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.902498+0000", 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3469999999999999e-06 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.902550+0000", 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2023e-05 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.902564+0000", 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4266e-05 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.902577+0000", 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2822e-05 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.937735+0000", 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.035158214 2024-08-29T15:29:03.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.937744+0000", 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1409999999999994e-06 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:13.941699+0000", 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039545120000000003 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:240851 6.0 e285/218 6:0b8369ad:::benchmark_data_smithi027_169445_object115424:head v 285'15213, mlcod=181'5490)", 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.255417+0000", 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.347569341000003, 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046651352, 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.255417+0000", 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.255417+0000", 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.255418+0000", 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.206e-06 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.255426+0000", 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0010000000000001e-06 2024-08-29T15:29:03.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.255441+0000", 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4834000000000001e-05 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.255449+0000", 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2889999999999998e-06 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.257939+0000", 2024-08-29T15:29:03.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024904689999999999 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.257960+0000", 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0375999999999999e-05 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.302002+0000", 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.044042087000000001 2024-08-29T15:29:03.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.302068+0000", 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6089999999999999e-05 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:240991 6.8 e285/218 6:17ee3221:::benchmark_data_smithi027_169445_object115564:head v 285'15037, mlcod=181'5379)", 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.500931+0000", 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.102054586000001, 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.054691989000000003, 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.500931+0000", 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.500931+0000", 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.500932+0000", 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.61e-07 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.500939+0000", 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0620000000000003e-06 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.500950+0000", 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0651e-05 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.500956+0000", 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2960000000000004e-06 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.506146+0000", 2024-08-29T15:29:03.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051893690000000001 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.506190+0000", 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4768999999999998e-05 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.555493+0000", 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.049303154000000002 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.555623+0000", 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129927 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:241001 6.8 e285/218 6:1589059d:::benchmark_data_smithi027_169445_object115574:head v 285'15038, mlcod=181'5379)", 2024-08-29T15:29:03.623 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.508119+0000", 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.094867422999997, 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.071909872, 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.508119+0000", 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.508119+0000", 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.508120+0000", 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.063e-06 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.508127+0000", 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8450000000000003e-06 2024-08-29T15:29:03.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.508140+0000", 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2734999999999999e-05 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.508149+0000", 2024-08-29T15:29:03.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3009999999999995e-06 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.520158+0000", 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012008911000000001 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.520189+0000", 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0960999999999997e-05 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.579971+0000", 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.059781784999999997 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580028+0000", 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7271000000000001e-05 2024-08-29T15:29:03.629 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:241006 6.8 e285/218 6:1c4b8be3:::benchmark_data_smithi027_169445_object115579:head v 285'15039, mlcod=181'5379)", 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.519453+0000", 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.083532748000003, 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.060605820999999997, 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.519453+0000", 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.519453+0000", 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.519454+0000", 2024-08-29T15:29:03.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1699999999999999e-07 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.519459+0000", 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6030000000000004e-06 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.519466+0000", 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2679999999999997e-06 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.519471+0000", 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.724e-06 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.530194+0000", 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010722656000000001 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.530220+0000", 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6205999999999999e-05 2024-08-29T15:29:03.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580033+0000", 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.049812848 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580059+0000", 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6098999999999999e-05 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.634 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:241008 6.8 e285/218 6:17ebefcb:::benchmark_data_smithi027_169445_object115581:head v 285'15040, mlcod=181'5379)", 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.526207+0000", 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.076779356999999, 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053884478, 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.526207+0000", 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.526207+0000", 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.526207+0000", 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2900000000000003e-07 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.526212+0000", 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7979999999999996e-06 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.526222+0000", 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7960000000000007e-06 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.526232+0000", 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0603e-05 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540749+0000", 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014517039000000001 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540802+0000", 2024-08-29T15:29:03.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2598e-05 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580061+0000", 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.03925915 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580091+0000", 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9865e-05 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:241012 6.8 e285/218 6:187141bb:::benchmark_data_smithi027_169445_object115585:head v 285'15041, mlcod=181'5379)", 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.534019+0000", 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.068967151000003, 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046107954, 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.534019+0000", 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.534019+0000", 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.534020+0000", 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8200000000000008e-07 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.534027+0000", 2024-08-29T15:29:03.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1960000000000002e-06 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.534044+0000", 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6778000000000001e-05 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.534052+0000", 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6759999999999996e-06 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.547521+0000", 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013468443 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.547547+0000", 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5834999999999999e-05 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580096+0000", 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.032548964 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580127+0000", 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1080000000000001e-05 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:241019 6.0 e285/218 6:0a6a5058:::benchmark_data_smithi027_169445_object115592:head v 285'15218, mlcod=181'5490)", 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.540279+0000", 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 49.062707271999997, 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039964261000000001, 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.644 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540279+0000", 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540279+0000", 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540280+0000", 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7999999999999993e-07 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540286+0000", 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7710000000000001e-06 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540298+0000", 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1574e-05 2024-08-29T15:29:03.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.540305+0000", 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0110000000000003e-06 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.555657+0000", 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015352387 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.555684+0000", 2024-08-29T15:29:03.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6338000000000001e-05 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580158+0000", 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024474302999999999 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.580243+0000", 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4896999999999997e-05 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.648 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:241121 6.0 e285/218 6:0cc4bd1c:::benchmark_data_smithi027_169445_object115694:head v 285'15225, mlcod=181'5490)", 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.741086+0000", 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 48.861900132999999, 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.037267293, 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.649 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.741086+0000", 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.741086+0000", 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.741087+0000", 2024-08-29T15:29:03.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1349999999999999e-06 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.741095+0000", 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1990000000000006e-06 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.741121+0000", 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5751000000000001e-05 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.741133+0000", 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2537999999999999e-05 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.744640+0000", 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035068230000000001 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.744669+0000", 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8430000000000001e-05 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.764364+0000", 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019695570999999999 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.778353+0000", 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013988845999999999 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.653 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:241256 6.1 e286/217 6:88fa083a:::benchmark_data_smithi027_169445_object115829:head v 286'15113, mlcod=185'5683)", 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.955225+0000", 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 48.647760464999998, 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.037966516999999998, 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955225+0000", 2024-08-29T15:29:03.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955225+0000", 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955227+0000", 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0550000000000001e-06 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955234+0000", 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2139999999999999e-06 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955247+0000", 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3108000000000001e-05 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955254+0000", 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8739999999999997e-06 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955913+0000", 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00065940900000000002 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955938+0000", 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5282e-05 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.993087+0000", 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.037148331999999999 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.993192+0000", 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105243 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(6.3 286/218 [PushOp(6:cec7ee73:::benchmark_data_smithi027_169445_object116494:head, version: 188'7272, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:cec7ee73:::benchmark_data_smithi027_169445_object116494:head@188'7272, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:16.224833+0000", 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 47.378152706999998, 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039558263000000003, 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.660 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.224833+0000", 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.224833+0000", 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.224834+0000", 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5600000000000004e-07 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.224883+0000", 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9132000000000003e-05 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.224905+0000", 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1795999999999999e-05 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.224925+0000", 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0044999999999999e-05 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.238984+0000", 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014059156999999999 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.238999+0000", 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4555e-05 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:16.264392+0000", 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.025392722 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(6.3 286/218 [PushOp(6:c7b6f040:::benchmark_data_smithi027_169445_object117824:head, version: 188'7353, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:c7b6f040:::benchmark_data_smithi027_169445_object117824:head@188'7353, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.620719+0000", 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.982266514999999, 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046034538999999999, 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.620719+0000", 2024-08-29T15:29:03.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.620719+0000", 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.620720+0000", 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.92e-07 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.620792+0000", 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1903999999999995e-05 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.620824+0000", 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2073000000000003e-05 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.620842+0000", 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7637000000000001e-05 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.638832+0000", 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017989954999999998 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.638846+0000", 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4661000000000001e-05 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.666754+0000", 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027907616999999999 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.670 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243492 6.8 e286/218 6:1a349ad1:::benchmark_data_smithi027_169445_object118065:head v 286'15202, mlcod=181'5379)", 2024-08-29T15:29:03.672 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.942387+0000", 2024-08-29T15:29:03.672 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.660598720000003, 2024-08-29T15:29:03.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.051615868000000002, 2024-08-29T15:29:03.672 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.672 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.942387+0000", 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.942387+0000", 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.942389+0000", 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7349999999999999e-06 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.942397+0000", 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2900000000000002e-06 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.942415+0000", 2024-08-29T15:29:03.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7397999999999998e-05 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.942424+0000", 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9560000000000003e-06 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.946588+0000", 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041645049999999998 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.946671+0000", 2024-08-29T15:29:03.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2535999999999997e-05 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.993208+0000", 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046537145000000002 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.994003+0000", 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00079530300000000003 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243499 6.8 e286/218 6:1d2d7f4c:::benchmark_data_smithi027_169445_object118072:head v 286'15203, mlcod=181'5379)", 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.948931+0000", 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.65405543, 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.045257169, 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.948931+0000", 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.948931+0000", 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.948932+0000", 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2550000000000001e-06 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.948941+0000", 2024-08-29T15:29:03.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2809999999999997e-06 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.950207+0000", 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001265822 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.950217+0000", 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0242e-05 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.954936+0000", 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047191259999999997 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.955003+0000", 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7186000000000004e-05 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.994007+0000", 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.039003736999999997 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.994188+0000", 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018052000000000001 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243503 6.0 e286/218 6:0525f298:::benchmark_data_smithi027_169445_object118076:head v 286'15359, mlcod=181'5490)", 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.955070+0000", 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.647915615999999, 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.069110702999999996, 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.681 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.955070+0000", 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.955070+0000", 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.955072+0000", 2024-08-29T15:29:03.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.378e-06 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.955083+0000", 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1099e-05 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.955096+0000", 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3149999999999999e-05 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.955105+0000", 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3780000000000007e-06 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961622+0000", 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065164539999999996 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961679+0000", 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7407000000000001e-05 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.024006+0000", 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.062326473 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.024181+0000", 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017536400000000001 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243506 6.0 e286/218 6:06b66b9f:::benchmark_data_smithi027_169445_object118079:head v 286'15360, mlcod=181'5490)", 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.961442+0000", 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.641544256000003, 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.06281552, 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961442+0000", 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961442+0000", 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961443+0000", 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8299999999999995e-07 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961451+0000", 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4950000000000008e-06 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961468+0000", 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7254000000000001e-05 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.961477+0000", 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6030000000000005e-06 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.978851+0000", 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017374407000000001 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.978872+0000", 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1010999999999999e-05 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.024184+0000", 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.045311735999999998 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.024257+0000", 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3030999999999999e-05 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.690 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243508 6.8 e286/218 6:19a8cfe0:::benchmark_data_smithi027_169445_object118081:head v 286'15204, mlcod=181'5379)", 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.967665+0000", 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.635321255000001, 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.056665544999999998, 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.967665+0000", 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.967665+0000", 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.967666+0000", 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.133e-06 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.967675+0000", 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.969e-06 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.967690+0000", 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5336999999999999e-05 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.967701+0000", 2024-08-29T15:29:03.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0966e-05 2024-08-29T15:29:03.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.978763+0000", 2024-08-29T15:29:03.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011061547999999999 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.978791+0000", 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8846000000000001e-05 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.024260+0000", 2024-08-29T15:29:03.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.045468649 2024-08-29T15:29:03.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.024330+0000", 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0097000000000003e-05 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243513 6.0 e286/218 6:0910d1cd:::benchmark_data_smithi027_169445_object118086:head v 286'15361, mlcod=181'5490)", 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.980021+0000", 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.622964820999997, 2024-08-29T15:29:03.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.062196225000000001, 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.980021+0000", 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.980021+0000", 2024-08-29T15:29:03.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.980023+0000", 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.866e-06 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.980041+0000", 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8434e-05 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.980055+0000", 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3566e-05 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.980064+0000", 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9670000000000007e-06 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.993340+0000", 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013275604 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.993425+0000", 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5187999999999998e-05 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.042119+0000", 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.048694108 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.042217+0000", 2024-08-29T15:29:03.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8492000000000005e-05 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243516 6.0 e286/218 6:0d43d955:::benchmark_data_smithi027_169445_object118089:head v 286'15362, mlcod=181'5490)", 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:18.988906+0000", 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.614080389999998, 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053387291000000003, 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.702 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.988906+0000", 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.988906+0000", 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.988907+0000", 2024-08-29T15:29:03.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3599999999999999e-06 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.988913+0000", 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1129999999999997e-06 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.988932+0000", 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8919999999999998e-05 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:18.988942+0000", 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.781e-06 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.010276+0000", 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021334362999999999 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.010338+0000", 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1722000000000003e-05 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.042221+0000", 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.031883043999999999 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.042293+0000", 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1988000000000003e-05 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.44912.0:243538 6.0 e286/218 6:0deb5028:::benchmark_data_smithi027_169445_object118111:head v 286'15365, mlcod=181'5490)", 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:19.054977+0000", 2024-08-29T15:29:03.706 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.548008520000003, 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046740455, 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.054977+0000", 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.054977+0000", 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:29:03.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.054978+0000", 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6099999999999999e-07 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.054985+0000", 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0199999999999997e-06 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.055003+0000", 2024-08-29T15:29:03.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7274000000000001e-05 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.055012+0000", 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8380000000000008e-06 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.055091+0000", 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9785999999999998e-05 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.055118+0000", 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6400000000000001e-05 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.101621+0000", 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046503704999999999 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:19.101718+0000", 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6471000000000005e-05 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.710 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.711 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.711 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:29:03.711 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:29:03.711 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:29:03.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:03.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:03.861 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:03.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:03.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:03.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:03.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:04.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:04.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:04.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:04.238 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:04.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:03 smithi171 ceph-mon[90031]: pgmap v1036: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:04.301 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:03 smithi027 ceph-mon[93921]: pgmap v1036: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:04.302 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:03 smithi027 ceph-mon[98970]: pgmap v1036: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:04.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:04.363 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:04.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:04.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:04.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:04.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:04.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:04.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:04.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:04.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:04.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:04.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:04.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:04.966 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:04.992 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:04.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:05.094 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:05.119 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:05.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:05.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:05.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:05.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:05.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:05.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:05.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:05.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:29:05.475 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:05.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:05.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:05.602 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:05.626 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:05.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:05.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:05.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:05.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:05.846 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:05.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:05.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:05.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:05.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:06.003 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:06.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:06.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:06.130 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:06.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:06.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:06.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:06.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:05 smithi171 ceph-mon[90031]: pgmap v1037: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:06.310 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:05 smithi027 ceph-mon[93921]: pgmap v1037: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:06.311 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:05 smithi027 ceph-mon[98970]: pgmap v1037: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:06.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:06.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:06.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:06.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:06.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:06.628 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:06.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:06.756 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:06.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:06.858 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:06.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:06.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:06.985 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:07.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:07.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.136 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:07.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:07.237 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:07.263 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:07.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:07.365 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:07.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:07.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:07.489 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:07.515 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:07.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:07.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.640 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:07.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:07.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:07.748 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:29:07.748 INFO:teuthology.orchestra.run.smithi027.stdout:{"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":26,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1529,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":296,"num_osds":8,"num_up_osds":8,"osd_up_since":1724944712,"num_in_osds":8,"osd_in_since":1724944618,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":28,"data_bytes":4790245,"bytes_used":2703405056,"bytes_avail":765256880128,"bytes_total":767960285184},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi027.afjjno","status":"up:active","gid":14619}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.27:8443/","prometheus":"http://172.21.15.27:9283/"}},"servicemap":{"epoch":29,"modified":"2024-08-29T15:24:56.616982+0000","services":{}},"progress_events":{}} 2024-08-29T15:29:07.765 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:07.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:07.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:07.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:07.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:08.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:08.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:08.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:08.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:07 smithi171 ceph-mon[90031]: pgmap v1038: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2700748781' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:29:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.268 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:08.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:07 smithi027 ceph-mon[93921]: pgmap v1038: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2700748781' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:29:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:07 smithi027 ceph-mon[98970]: pgmap v1038: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2700748781' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-29T15:29:08.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:08.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:08.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:08.497 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:08.523 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:08.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:08.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:08.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:08.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:08.752 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:08.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:08.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:08.902 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:08.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:08.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:09.004 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:09.030 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:09.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:09.156 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:09.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:09.283 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:29:09] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-08-29T15:29:09.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:09.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:09.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:09.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:09.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:09.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:09.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:09.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:09.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:09.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:09.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:09.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:09.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:09.941 INFO:teuthology.orchestra.run.smithi027.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-08-29T15:29:09.941 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:29:09.974 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:29:09.974 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:29:09.995 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:29:09.995 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:29:10.013 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:29:10.013 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:29:10.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:10.028 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:29:10.028 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:29:10.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:10.044 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:29:10.045 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:29:10.063 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:29:10.063 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:29:10.080 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:29:10.080 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:29:10.097 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:29:10.097 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:29:10.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:10.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:10.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:10.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:09 smithi171 ceph-mon[90031]: pgmap v1039: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3717345244' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2965061581' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:29:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:29:10.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:10.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:10.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[98970]: pgmap v1039: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:10.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3717345244' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:10.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:10.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2965061581' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:29:10.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:29:10.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[93921]: pgmap v1039: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:10.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3717345244' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:10.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:10.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2965061581' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:29:10.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:29:10.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:10.416 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:10.488 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-08-29T15:29:10.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:29:10.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:10.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.555 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:10.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:10.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:10.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:10.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:10.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:10.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:10.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:10.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:10.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:11.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:11.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:11.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:11.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:11.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:10 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:29:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:10 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:29:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:10 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:29:11.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:10 smithi171 ceph-mon[90031]: osdmap e297: 8 total, 8 up, 8 in 2024-08-29T15:29:11.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/78212684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:11.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:11.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:11.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:29:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:29:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:29:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[93921]: osdmap e297: 8 total, 8 up, 8 in 2024-08-29T15:29:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/78212684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:11.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:29:11.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:29:11.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:29:11.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[98970]: osdmap e297: 8 total, 8 up, 8 in 2024-08-29T15:29:11.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/78212684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:11.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:11.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:11.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:11.559 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:11.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:11.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:11.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:11.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:11.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:11.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:11.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:11.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:11.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:12.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:12.043 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:12.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:12.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:12.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:12.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:11 smithi027 ceph-mon[98970]: pgmap v1041: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:12.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:11 smithi027 ceph-mon[98970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:29:12.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:11 smithi027 ceph-mon[93921]: pgmap v1041: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:12.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:11 smithi027 ceph-mon[93921]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:29:12.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:12.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:12.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:11 smithi171 ceph-mon[90031]: pgmap v1041: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:11 smithi171 ceph-mon[90031]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:29:12.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:12.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:12.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:12.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:12.420 DEBUG:teuthology.orchestra.run.smithi027:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-08-29T15:29:12.422 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-29T15:29:12.423 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-29T15:29:12.423 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:12.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:12.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:12.476 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:hints = 1 2024-08-29T15:29:12.476 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-29T15:29:12.476 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object prefix: benchmark_data_smithi027_271924 2024-08-29T15:29:12.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:29:12.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 0 0 0 0 0 0 - 0 2024-08-29T15:29:12.520 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:12.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:12.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:12.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:12.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:12.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:12.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:12.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:12.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:12.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:12.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:12.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:13.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:13.111 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:13.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:13.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:13.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:13.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:13.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:13.415 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:13.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 1 16 3455 3439 214.927 214.938 0.00385939 0.0046373 2024-08-29T15:29:13.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:13.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:13.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:13.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:13.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:13.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:13.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:13.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:29:13.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:13.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:13.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:13 smithi027 ceph-mon[98970]: pgmap v1042: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:14.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:13 smithi027 ceph-mon[93921]: pgmap v1042: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:14.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:14.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:14.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:14.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:14.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:13 smithi171 ceph-mon[90031]: pgmap v1042: 89 pgs: 89 active+clean; 4.6 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-29T15:29:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:14.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:14.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:14.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:14.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:14.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:14.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:14.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:14.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 2 16 6280 6264 195.736 176.562 0.00525783 0.00509906 2024-08-29T15:29:14.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:29:14.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:14.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:14.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:14.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:14.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:14.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:14.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:14.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:14.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:14.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:14.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:14.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:14.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:15.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:15.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:15.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:15.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:15.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:15.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:15.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:15.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:15.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:15.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:15.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:15.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:15.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 3 16 9357 9341 194.588 192.312 0.0033822 0.00513114 2024-08-29T15:29:15.485 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:15.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:15.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:15.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:15.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:15.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:15.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:15.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:15.772 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:15.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:15.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:15.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:15.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:15.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:16.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:16.077 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:16.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:16.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:16.212 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:15 smithi027 ceph-mon[98970]: pgmap v1043: 89 pgs: 89 active+clean; 107 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s 2024-08-29T15:29:16.212 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:15 smithi027 ceph-mon[93921]: pgmap v1043: 89 pgs: 89 active+clean; 107 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s 2024-08-29T15:29:16.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:16.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:15 smithi171 ceph-mon[90031]: pgmap v1043: 89 pgs: 89 active+clean; 107 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s wr, 164 op/s 2024-08-29T15:29:16.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:16.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:16.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:16.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:16.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 4 16 11881 11865 185.375 157.75 0.00271498 0.00538693 2024-08-29T15:29:16.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:16.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:16.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:16.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:16.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:16.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:16.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:16.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:16.771 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:16.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:16.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:16.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:16.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:16.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:17.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:17.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:17.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:17.047 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:29:17.048 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:29:17.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:17.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:17.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:17.245 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:17.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:17.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:17.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:17.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:17.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:17.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:17.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 5 16 14260 14244 178.035 148.688 0.00531939 0.00560844 2024-08-29T15:29:17.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:17.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:17.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:17.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:17.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:17.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:17.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:17.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:17.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:17.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:17.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:17.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:18.022 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:18.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:18.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:17 smithi027 ceph-mon[93921]: pgmap v1044: 89 pgs: 89 active+clean; 493 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2024-08-29T15:29:18.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:17 smithi027 ceph-mon[98970]: pgmap v1044: 89 pgs: 89 active+clean; 493 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2024-08-29T15:29:18.123 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:18.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:18.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:17 smithi171 ceph-mon[90031]: pgmap v1044: 89 pgs: 89 active+clean; 493 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2024-08-29T15:29:18.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:18.393 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:29:18.394 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:29:18.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:18.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:18.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 6 16 16427 16411 170.933 135.438 0.00320308 0.00584276 2024-08-29T15:29:18.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:18.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:18.655 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:18.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:18.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:18.831 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:18.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:18.933 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:18.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:18.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:18.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:19.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:19.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:19.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:19.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:19.247 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:19.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:19.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:29:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:29:19] "GET /metrics HTTP/1.1" 200 36447 "" "Prometheus/2.51.0" 2024-08-29T15:29:19.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:19.388 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:19.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:19.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:29:19.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 7 16 18998 18982 169.467 160.688 0.0046097 0.00589336 2024-08-29T15:29:19.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:19.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:19.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:19.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:19.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:19.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:19.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:19.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:19.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:19.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:19.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:19.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:19.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:20.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:20.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:19 smithi027 ceph-mon[98970]: from='client.55978 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:20.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:19 smithi027 ceph-mon[98970]: pgmap v1045: 89 pgs: 89 active+clean; 544 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-08-29T15:29:20.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:19 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:29:20.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:19 smithi027 ceph-mon[93921]: from='client.55978 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:20.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:19 smithi027 ceph-mon[93921]: pgmap v1045: 89 pgs: 89 active+clean; 544 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-08-29T15:29:20.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:19 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:29:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:20.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:20.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:20.082 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:20.120 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1044,"stamp":"2024-08-29T15:29:16.703637+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":516429797,"num_objects":7835,"num_object_clones":0,"num_object_copies":23505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7835,"num_whiteouts":0,"num_read":6365,"num_read_kb":13822,"num_write":8930,"num_write_kb":514689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":10166259,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":8989,"ondisk_log_size":8989,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3873656,"kb_used_data":1664692,"kb_used_omap":43,"kb_used_meta":2208852,"kb_avail":746087560,"statfs":{"total":767960285184,"available":763993661440,"internally_reserved":0,"allocated":1704644608,"data_stored":1690669454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2261864940},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":511639552,"num_objects":7807,"num_object_clones":0,"num_object_copies":23421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7807,"num_write_kb":499648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003306"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":480,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938590+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:29:09.938590+0000","last_peered":"2024-08-29T15:29:09.938590+0000","last_clean":"2024-08-29T15:29:09.938590+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:29:09.938590+0000","last_undegraded":"2024-08-29T15:29:09.938590+0000","last_fullsized":"2024-08-29T15:29:09.938590+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":85,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.888728+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:29:09.888728+0000","last_peered":"2024-08-29T15:29:09.888728+0000","last_clean":"2024-08-29T15:29:09.888728+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:29:09.888728+0000","last_undegraded":"2024-08-29T15:29:09.888728+0000","last_fullsized":"2024-08-29T15:29:09.888728+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":166,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922733+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:28:58.922733+0000","last_peered":"2024-08-29T15:28:58.922733+0000","last_clean":"2024-08-29T15:28:58.922733+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:28:58.922733+0000","last_undegraded":"2024-08-29T15:28:58.922733+0000","last_fullsized":"2024-08-29T15:28:58.922733+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":447,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956919+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:29:09.956919+0000","last_peered":"2024-08-29T15:29:09.956919+0000","last_clean":"2024-08-29T15:29:09.956919+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:29:09.956919+0000","last_undegraded":"2024-08-29T15:29:09.956919+0000","last_fullsized":"2024-08-29T15:29:09.956919+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":447,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957783+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:29:09.957783+0000","last_peered":"2024-08-29T15:29:09.957783+0000","last_clean":"2024-08-29T15:29:09.957783+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:29:09.957783+0000","last_undegraded":"2024-08-29T15:29:09.957783+0000","last_fullsized":"2024-08-29T15:29:09.957783+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.888642+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:29:09.888642+0000","last_peered":"2024-08-29T15:29:09.888642+0000","last_clean":"2024-08-29T15:29:09.888642+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:29:09.888642+0000","last_undegraded":"2024-08-29T15:29:09.888642+0000","last_fullsized":"2024-08-29T15:29:09.888642+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":258,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958755+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:29:09.958755+0000","last_peered":"2024-08-29T15:29:09.958755+0000","last_clean":"2024-08-29T15:29:09.958755+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:29:09.958755+0000","last_undegraded":"2024-08-29T15:29:09.958755+0000","last_fullsized":"2024-08-29T15:29:09.958755+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":168,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.153552+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:28:59.153552+0000","last_peered":"2024-08-29T15:28:59.153552+0000","last_clean":"2024-08-29T15:28:59.153552+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:28:59.153552+0000","last_undegraded":"2024-08-29T15:28:59.153552+0000","last_fullsized":"2024-08-29T15:28:59.153552+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"297'683","reported_seq":700,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:15.791503+0000","last_change":"2024-08-29T15:28:57.922512+0000","last_active":"2024-08-29T15:29:15.791503+0000","last_peered":"2024-08-29T15:29:15.791503+0000","last_clean":"2024-08-29T15:29:15.791503+0000","last_became_active":"2024-08-29T15:28:57.922093+0000","last_became_peered":"2024-08-29T15:28:57.922093+0000","last_unstale":"2024-08-29T15:29:15.791503+0000","last_undegraded":"2024-08-29T15:29:15.791503+0000","last_fullsized":"2024-08-29T15:29:15.791503+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":683,"log_dups_size":0,"ondisk_log_size":683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:26.218110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44761088,"num_objects":683,"num_object_clones":0,"num_object_copies":2049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":683,"num_write_kb":43712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":82,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154811+0000","last_change":"2024-08-29T15:28:35.524176+0000","last_active":"2024-08-29T15:28:59.154811+0000","last_peered":"2024-08-29T15:28:59.154811+0000","last_clean":"2024-08-29T15:28:59.154811+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:28:59.154811+0000","last_undegraded":"2024-08-29T15:28:59.154811+0000","last_fullsized":"2024-08-29T15:28:59.154811+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":185,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922308+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:28:58.922308+0000","last_peered":"2024-08-29T15:28:58.922308+0000","last_clean":"2024-08-29T15:28:58.922308+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:28:58.922308+0000","last_undegraded":"2024-08-29T15:28:58.922308+0000","last_fullsized":"2024-08-29T15:28:58.922308+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"297'778","reported_seq":795,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:16.567534+0000","last_change":"2024-08-29T15:28:57.922760+0000","last_active":"2024-08-29T15:29:16.567534+0000","last_peered":"2024-08-29T15:29:16.567534+0000","last_clean":"2024-08-29T15:29:16.567534+0000","last_became_active":"2024-08-29T15:28:57.922490+0000","last_became_peered":"2024-08-29T15:28:57.922490+0000","last_unstale":"2024-08-29T15:29:16.567534+0000","last_undegraded":"2024-08-29T15:29:16.567534+0000","last_fullsized":"2024-08-29T15:29:16.567534+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":778,"log_dups_size":0,"ondisk_log_size":778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:46:47.703614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50987008,"num_objects":778,"num_object_clones":0,"num_object_copies":2334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":778,"num_write_kb":49792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":82,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154380+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:28:59.154380+0000","last_peered":"2024-08-29T15:28:59.154380+0000","last_clean":"2024-08-29T15:28:59.154380+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:28:59.154380+0000","last_undegraded":"2024-08-29T15:28:59.154380+0000","last_fullsized":"2024-08-29T15:28:59.154380+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.153994+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:28:59.153994+0000","last_peered":"2024-08-29T15:28:59.153994+0000","last_clean":"2024-08-29T15:28:59.153994+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:28:59.153994+0000","last_undegraded":"2024-08-29T15:28:59.153994+0000","last_fullsized":"2024-08-29T15:28:59.153994+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"297'330","reported_seq":347,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.090664+0000","last_change":"2024-08-29T15:28:57.926748+0000","last_active":"2024-08-29T15:29:14.090664+0000","last_peered":"2024-08-29T15:29:14.090664+0000","last_clean":"2024-08-29T15:29:14.090664+0000","last_became_active":"2024-08-29T15:28:57.926391+0000","last_became_peered":"2024-08-29T15:28:57.926391+0000","last_unstale":"2024-08-29T15:29:14.090664+0000","last_undegraded":"2024-08-29T15:29:14.090664+0000","last_fullsized":"2024-08-29T15:29:14.090664+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":330,"log_dups_size":0,"ondisk_log_size":330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:30:16.060709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21626880,"num_objects":330,"num_object_clones":0,"num_object_copies":990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":330,"num_write_kb":21120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":263,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957203+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:29:09.957203+0000","last_peered":"2024-08-29T15:29:09.957203+0000","last_clean":"2024-08-29T15:29:09.957203+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:29:09.957203+0000","last_undegraded":"2024-08-29T15:29:09.957203+0000","last_fullsized":"2024-08-29T15:29:09.957203+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":123,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.153798+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:28:59.153798+0000","last_peered":"2024-08-29T15:28:59.153798+0000","last_clean":"2024-08-29T15:28:59.153798+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:28:59.153798+0000","last_undegraded":"2024-08-29T15:28:59.153798+0000","last_fullsized":"2024-08-29T15:28:59.153798+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"297'474","reported_seq":486,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.887791+0000","last_change":"2024-08-29T15:28:58.126108+0000","last_active":"2024-08-29T15:29:14.887791+0000","last_peered":"2024-08-29T15:29:14.887791+0000","last_clean":"2024-08-29T15:29:14.887791+0000","last_became_active":"2024-08-29T15:28:58.125784+0000","last_became_peered":"2024-08-29T15:28:58.125784+0000","last_unstale":"2024-08-29T15:29:14.887791+0000","last_undegraded":"2024-08-29T15:29:14.887791+0000","last_fullsized":"2024-08-29T15:29:14.887791+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":474,"log_dups_size":0,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:15:58.792157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31064064,"num_objects":474,"num_object_clones":0,"num_object_copies":1422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":474,"num_write_kb":30336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":185,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922727+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:28:58.922727+0000","last_peered":"2024-08-29T15:28:58.922727+0000","last_clean":"2024-08-29T15:28:58.922727+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:28:58.922727+0000","last_undegraded":"2024-08-29T15:28:58.922727+0000","last_fullsized":"2024-08-29T15:28:58.922727+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":461,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957421+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:29:09.957421+0000","last_peered":"2024-08-29T15:29:09.957421+0000","last_clean":"2024-08-29T15:29:09.957421+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:29:09.957421+0000","last_undegraded":"2024-08-29T15:29:09.957421+0000","last_fullsized":"2024-08-29T15:29:09.957421+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"297'325","reported_seq":342,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.071604+0000","last_change":"2024-08-29T15:28:58.127033+0000","last_active":"2024-08-29T15:29:14.071604+0000","last_peered":"2024-08-29T15:29:14.071604+0000","last_clean":"2024-08-29T15:29:14.071604+0000","last_became_active":"2024-08-29T15:28:58.126072+0000","last_became_peered":"2024-08-29T15:28:58.126072+0000","last_unstale":"2024-08-29T15:29:14.071604+0000","last_undegraded":"2024-08-29T15:29:14.071604+0000","last_fullsized":"2024-08-29T15:29:14.071604+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":325,"log_dups_size":0,"ondisk_log_size":325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:29:18.967329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21299200,"num_objects":325,"num_object_clones":0,"num_object_copies":975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":325,"num_write_kb":20800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":455,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938757+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:29:09.938757+0000","last_peered":"2024-08-29T15:29:09.938757+0000","last_clean":"2024-08-29T15:29:09.938757+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:29:09.938757+0000","last_undegraded":"2024-08-29T15:29:09.938757+0000","last_fullsized":"2024-08-29T15:29:09.938757+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.153928+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:28:59.153928+0000","last_peered":"2024-08-29T15:28:59.153928+0000","last_clean":"2024-08-29T15:28:59.153928+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:28:59.153928+0000","last_undegraded":"2024-08-29T15:28:59.153928+0000","last_fullsized":"2024-08-29T15:28:59.153928+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":17,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957777+0000","last_change":"2024-08-29T15:28:57.924976+0000","last_active":"2024-08-29T15:29:09.957777+0000","last_peered":"2024-08-29T15:29:09.957777+0000","last_clean":"2024-08-29T15:29:09.957777+0000","last_became_active":"2024-08-29T15:28:57.924668+0000","last_became_peered":"2024-08-29T15:28:57.924668+0000","last_unstale":"2024-08-29T15:29:09.957777+0000","last_undegraded":"2024-08-29T15:29:09.957777+0000","last_fullsized":"2024-08-29T15:29:09.957777+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:33:52.529565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":141,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154173+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:28:59.154173+0000","last_peered":"2024-08-29T15:28:59.154173+0000","last_clean":"2024-08-29T15:28:59.154173+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:28:59.154173+0000","last_undegraded":"2024-08-29T15:28:59.154173+0000","last_fullsized":"2024-08-29T15:28:59.154173+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154565+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:28:59.154565+0000","last_peered":"2024-08-29T15:28:59.154565+0000","last_clean":"2024-08-29T15:28:59.154565+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:28:59.154565+0000","last_undegraded":"2024-08-29T15:28:59.154565+0000","last_fullsized":"2024-08-29T15:28:59.154565+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"297'473","reported_seq":483,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.998908+0000","last_change":"2024-08-29T15:28:57.926441+0000","last_active":"2024-08-29T15:29:14.998908+0000","last_peered":"2024-08-29T15:29:14.998908+0000","last_clean":"2024-08-29T15:29:14.998908+0000","last_became_active":"2024-08-29T15:28:57.926173+0000","last_became_peered":"2024-08-29T15:28:57.926173+0000","last_unstale":"2024-08-29T15:29:14.998908+0000","last_undegraded":"2024-08-29T15:29:14.998908+0000","last_fullsized":"2024-08-29T15:29:14.998908+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":473,"log_dups_size":0,"ondisk_log_size":473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:43.091728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30998528,"num_objects":473,"num_object_clones":0,"num_object_copies":1419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":473,"num_write_kb":30272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154072+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:28:59.154072+0000","last_peered":"2024-08-29T15:28:59.154072+0000","last_clean":"2024-08-29T15:28:59.154072+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:28:59.154072+0000","last_undegraded":"2024-08-29T15:28:59.154072+0000","last_fullsized":"2024-08-29T15:28:59.154072+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":283,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154486+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:28:59.154486+0000","last_peered":"2024-08-29T15:28:59.154486+0000","last_clean":"2024-08-29T15:28:59.154486+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:28:59.154486+0000","last_undegraded":"2024-08-29T15:28:59.154486+0000","last_fullsized":"2024-08-29T15:28:59.154486+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"297'522","reported_seq":534,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.995486+0000","last_change":"2024-08-29T15:28:58.126443+0000","last_active":"2024-08-29T15:29:14.995486+0000","last_peered":"2024-08-29T15:29:14.995486+0000","last_clean":"2024-08-29T15:29:14.995486+0000","last_became_active":"2024-08-29T15:28:58.126054+0000","last_became_peered":"2024-08-29T15:28:58.126054+0000","last_unstale":"2024-08-29T15:29:14.995486+0000","last_undegraded":"2024-08-29T15:29:14.995486+0000","last_fullsized":"2024-08-29T15:29:14.995486+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":522,"log_dups_size":0,"ondisk_log_size":522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:42:38.613662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34209792,"num_objects":522,"num_object_clones":0,"num_object_copies":1566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":522,"num_write_kb":33408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":458,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938347+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:29:09.938347+0000","last_peered":"2024-08-29T15:29:09.938347+0000","last_clean":"2024-08-29T15:29:09.938347+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:29:09.938347+0000","last_undegraded":"2024-08-29T15:29:09.938347+0000","last_fullsized":"2024-08-29T15:29:09.938347+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":451,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958760+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:29:09.958760+0000","last_peered":"2024-08-29T15:29:09.958760+0000","last_clean":"2024-08-29T15:29:09.958760+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:29:09.958760+0000","last_undegraded":"2024-08-29T15:29:09.958760+0000","last_fullsized":"2024-08-29T15:29:09.958760+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"297'319","reported_seq":336,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.100792+0000","last_change":"2024-08-29T15:28:58.127095+0000","last_active":"2024-08-29T15:29:14.100792+0000","last_peered":"2024-08-29T15:29:14.100792+0000","last_clean":"2024-08-29T15:29:14.100792+0000","last_became_active":"2024-08-29T15:28:58.126393+0000","last_became_peered":"2024-08-29T15:28:58.126393+0000","last_unstale":"2024-08-29T15:29:14.100792+0000","last_undegraded":"2024-08-29T15:29:14.100792+0000","last_fullsized":"2024-08-29T15:29:14.100792+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:50:34.827428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20905984,"num_objects":319,"num_object_clones":0,"num_object_copies":957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":319,"num_write_kb":20416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":550,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958640+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:29:09.958640+0000","last_peered":"2024-08-29T15:29:09.958640+0000","last_clean":"2024-08-29T15:29:09.958640+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:29:09.958640+0000","last_undegraded":"2024-08-29T15:29:09.958640+0000","last_fullsized":"2024-08-29T15:29:09.958640+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":769,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922494+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:28:58.922494+0000","last_peered":"2024-08-29T15:28:58.922494+0000","last_clean":"2024-08-29T15:28:58.922494+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:28:58.922494+0000","last_undegraded":"2024-08-29T15:28:58.922494+0000","last_fullsized":"2024-08-29T15:28:58.922494+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":277,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922412+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:28:58.922412+0000","last_peered":"2024-08-29T15:28:58.922412+0000","last_clean":"2024-08-29T15:28:58.922412+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:28:58.922412+0000","last_undegraded":"2024-08-29T15:28:58.922412+0000","last_fullsized":"2024-08-29T15:28:58.922412+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":139,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154697+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:28:59.154697+0000","last_peered":"2024-08-29T15:28:59.154697+0000","last_clean":"2024-08-29T15:28:59.154697+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:28:59.154697+0000","last_undegraded":"2024-08-29T15:28:59.154697+0000","last_fullsized":"2024-08-29T15:28:59.154697+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"297'328","reported_seq":345,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.104151+0000","last_change":"2024-08-29T15:28:58.125997+0000","last_active":"2024-08-29T15:29:14.104151+0000","last_peered":"2024-08-29T15:29:14.104151+0000","last_clean":"2024-08-29T15:29:14.104151+0000","last_became_active":"2024-08-29T15:28:58.125680+0000","last_became_peered":"2024-08-29T15:28:58.125680+0000","last_unstale":"2024-08-29T15:29:14.104151+0000","last_undegraded":"2024-08-29T15:29:14.104151+0000","last_fullsized":"2024-08-29T15:29:14.104151+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":328,"log_dups_size":0,"ondisk_log_size":328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:30:49.334821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21495808,"num_objects":328,"num_object_clones":0,"num_object_copies":984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":328,"num_write_kb":20992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":924,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956814+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:29:09.956814+0000","last_peered":"2024-08-29T15:29:09.956814+0000","last_clean":"2024-08-29T15:29:09.956814+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:29:09.956814+0000","last_undegraded":"2024-08-29T15:29:09.956814+0000","last_fullsized":"2024-08-29T15:29:09.956814+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":462,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956668+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:29:09.956668+0000","last_peered":"2024-08-29T15:29:09.956668+0000","last_clean":"2024-08-29T15:29:09.956668+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:29:09.956668+0000","last_undegraded":"2024-08-29T15:29:09.956668+0000","last_fullsized":"2024-08-29T15:29:09.956668+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":470,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957981+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:29:09.957981+0000","last_peered":"2024-08-29T15:29:09.957981+0000","last_clean":"2024-08-29T15:29:09.957981+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:29:09.957981+0000","last_undegraded":"2024-08-29T15:29:09.957981+0000","last_fullsized":"2024-08-29T15:29:09.957981+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"297'339","reported_seq":356,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.100620+0000","last_change":"2024-08-29T15:28:57.925020+0000","last_active":"2024-08-29T15:29:14.100620+0000","last_peered":"2024-08-29T15:29:14.100620+0000","last_clean":"2024-08-29T15:29:14.100620+0000","last_became_active":"2024-08-29T15:28:57.924490+0000","last_became_peered":"2024-08-29T15:28:57.924490+0000","last_unstale":"2024-08-29T15:29:14.100620+0000","last_undegraded":"2024-08-29T15:29:14.100620+0000","last_fullsized":"2024-08-29T15:29:14.100620+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":339,"log_dups_size":0,"ondisk_log_size":339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:53:05.182238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22216704,"num_objects":339,"num_object_clones":0,"num_object_copies":1017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":339,"num_write_kb":21696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":732,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938650+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:29:09.938650+0000","last_peered":"2024-08-29T15:29:09.938650+0000","last_clean":"2024-08-29T15:29:09.938650+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:29:09.938650+0000","last_undegraded":"2024-08-29T15:29:09.938650+0000","last_fullsized":"2024-08-29T15:29:09.938650+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.153802+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:28:59.153802+0000","last_peered":"2024-08-29T15:28:59.153802+0000","last_clean":"2024-08-29T15:28:59.153802+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:28:59.153802+0000","last_undegraded":"2024-08-29T15:28:59.153802+0000","last_fullsized":"2024-08-29T15:28:59.153802+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":496,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958605+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:29:09.958605+0000","last_peered":"2024-08-29T15:29:09.958605+0000","last_clean":"2024-08-29T15:29:09.958605+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:29:09.958605+0000","last_undegraded":"2024-08-29T15:29:09.958605+0000","last_fullsized":"2024-08-29T15:29:09.958605+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"297'519","reported_seq":531,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.993259+0000","last_change":"2024-08-29T15:28:57.926317+0000","last_active":"2024-08-29T15:29:14.993259+0000","last_peered":"2024-08-29T15:29:14.993259+0000","last_clean":"2024-08-29T15:29:14.993259+0000","last_became_active":"2024-08-29T15:28:57.926116+0000","last_became_peered":"2024-08-29T15:28:57.926116+0000","last_unstale":"2024-08-29T15:29:14.993259+0000","last_undegraded":"2024-08-29T15:29:14.993259+0000","last_fullsized":"2024-08-29T15:29:14.993259+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":519,"log_dups_size":0,"ondisk_log_size":519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:35:53.613903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34013184,"num_objects":519,"num_object_clones":0,"num_object_copies":1557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":519,"num_write_kb":33216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":237,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922363+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:28:58.922363+0000","last_peered":"2024-08-29T15:28:58.922363+0000","last_clean":"2024-08-29T15:28:58.922363+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:28:58.922363+0000","last_undegraded":"2024-08-29T15:28:58.922363+0000","last_fullsized":"2024-08-29T15:28:58.922363+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154779+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:28:59.154779+0000","last_peered":"2024-08-29T15:28:59.154779+0000","last_clean":"2024-08-29T15:28:59.154779+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:28:59.154779+0000","last_undegraded":"2024-08-29T15:28:59.154779+0000","last_fullsized":"2024-08-29T15:28:59.154779+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":144,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922714+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:28:58.922714+0000","last_peered":"2024-08-29T15:28:58.922714+0000","last_clean":"2024-08-29T15:28:58.922714+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:28:58.922714+0000","last_undegraded":"2024-08-29T15:28:58.922714+0000","last_fullsized":"2024-08-29T15:28:58.922714+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.1","version":"297'503","reported_seq":513,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:14.865482+0000","last_change":"2024-08-29T15:28:57.924977+0000","last_active":"2024-08-29T15:29:14.865482+0000","last_peered":"2024-08-29T15:29:14.865482+0000","last_clean":"2024-08-29T15:29:14.865482+0000","last_became_active":"2024-08-29T15:28:57.924686+0000","last_became_peered":"2024-08-29T15:28:57.924686+0000","last_unstale":"2024-08-29T15:29:14.865482+0000","last_undegraded":"2024-08-29T15:29:14.865482+0000","last_fullsized":"2024-08-29T15:29:14.865482+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":503,"log_dups_size":0,"ondisk_log_size":503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:07:46.518392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32964608,"num_objects":503,"num_object_clones":0,"num_object_copies":1509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":503,"num_write_kb":32192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":169,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922630+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:28:58.922630+0000","last_peered":"2024-08-29T15:28:58.922630+0000","last_clean":"2024-08-29T15:28:58.922630+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:28:58.922630+0000","last_undegraded":"2024-08-29T15:28:58.922630+0000","last_fullsized":"2024-08-29T15:28:58.922630+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":889,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958038+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:29:09.958038+0000","last_peered":"2024-08-29T15:29:09.958038+0000","last_clean":"2024-08-29T15:29:09.958038+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:29:09.958038+0000","last_undegraded":"2024-08-29T15:29:09.958038+0000","last_fullsized":"2024-08-29T15:29:09.958038+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":466,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958117+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:29:09.958117+0000","last_peered":"2024-08-29T15:29:09.958117+0000","last_clean":"2024-08-29T15:29:09.958117+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:29:09.958117+0000","last_undegraded":"2024-08-29T15:29:09.958117+0000","last_fullsized":"2024-08-29T15:29:09.958117+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"297'698","reported_seq":710,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:16.374600+0000","last_change":"2024-08-29T15:28:57.922386+0000","last_active":"2024-08-29T15:29:16.374600+0000","last_peered":"2024-08-29T15:29:16.374600+0000","last_clean":"2024-08-29T15:29:16.374600+0000","last_became_active":"2024-08-29T15:28:57.922067+0000","last_became_peered":"2024-08-29T15:28:57.922067+0000","last_unstale":"2024-08-29T15:29:16.374600+0000","last_undegraded":"2024-08-29T15:29:16.374600+0000","last_fullsized":"2024-08-29T15:29:16.374600+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":698,"log_dups_size":0,"ondisk_log_size":698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:11:27.444379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45744128,"num_objects":698,"num_object_clones":0,"num_object_copies":2094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":698,"num_write_kb":44672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":471,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957944+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:29:09.957944+0000","last_peered":"2024-08-29T15:29:09.957944+0000","last_clean":"2024-08-29T15:29:09.957944+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:29:09.957944+0000","last_undegraded":"2024-08-29T15:29:09.957944+0000","last_fullsized":"2024-08-29T15:29:09.957944+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":749,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.076867+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:28:59.076867+0000","last_peered":"2024-08-29T15:28:59.076867+0000","last_clean":"2024-08-29T15:28:59.076867+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:28:59.076867+0000","last_undegraded":"2024-08-29T15:28:59.076867+0000","last_fullsized":"2024-08-29T15:28:59.076867+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":462,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957333+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:29:09.957333+0000","last_peered":"2024-08-29T15:29:09.957333+0000","last_clean":"2024-08-29T15:29:09.957333+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:29:09.957333+0000","last_undegraded":"2024-08-29T15:29:09.957333+0000","last_fullsized":"2024-08-29T15:29:09.957333+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"297'806","reported_seq":823,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:16.561674+0000","last_change":"2024-08-29T15:28:57.924025+0000","last_active":"2024-08-29T15:29:16.561674+0000","last_peered":"2024-08-29T15:29:16.561674+0000","last_clean":"2024-08-29T15:29:16.561674+0000","last_became_active":"2024-08-29T15:28:57.923719+0000","last_became_peered":"2024-08-29T15:28:57.923719+0000","last_unstale":"2024-08-29T15:29:16.561674+0000","last_undegraded":"2024-08-29T15:29:16.561674+0000","last_fullsized":"2024-08-29T15:29:16.561674+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":806,"log_dups_size":0,"ondisk_log_size":806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:46.870950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52822016,"num_objects":806,"num_object_clones":0,"num_object_copies":2418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":806,"num_write_kb":51584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":455,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.939045+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:29:09.939045+0000","last_peered":"2024-08-29T15:29:09.939045+0000","last_clean":"2024-08-29T15:29:09.939045+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:29:09.939045+0000","last_undegraded":"2024-08-29T15:29:09.939045+0000","last_fullsized":"2024-08-29T15:29:09.939045+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1107,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958571+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:29:09.958571+0000","last_peered":"2024-08-29T15:29:09.958571+0000","last_clean":"2024-08-29T15:29:09.958571+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:29:09.958571+0000","last_undegraded":"2024-08-29T15:29:09.958571+0000","last_fullsized":"2024-08-29T15:29:09.958571+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":453,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938318+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:29:09.938318+0000","last_peered":"2024-08-29T15:29:09.938318+0000","last_clean":"2024-08-29T15:29:09.938318+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:29:09.938318+0000","last_undegraded":"2024-08-29T15:29:09.938318+0000","last_fullsized":"2024-08-29T15:29:09.938318+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"297'710","reported_seq":726,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:16.562209+0000","last_change":"2024-08-29T15:28:57.923922+0000","last_active":"2024-08-29T15:29:16.562209+0000","last_peered":"2024-08-29T15:29:16.562209+0000","last_clean":"2024-08-29T15:29:16.562209+0000","last_became_active":"2024-08-29T15:28:57.923593+0000","last_became_peered":"2024-08-29T15:28:57.923593+0000","last_unstale":"2024-08-29T15:29:16.562209+0000","last_undegraded":"2024-08-29T15:29:16.562209+0000","last_fullsized":"2024-08-29T15:29:16.562209+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":710,"log_dups_size":0,"ondisk_log_size":710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:48:59.077875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46530560,"num_objects":710,"num_object_clones":0,"num_object_copies":2130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":710,"num_write_kb":45440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":452,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957667+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:29:09.957667+0000","last_peered":"2024-08-29T15:29:09.957667+0000","last_clean":"2024-08-29T15:29:09.957667+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:29:09.957667+0000","last_undegraded":"2024-08-29T15:29:09.957667+0000","last_fullsized":"2024-08-29T15:29:09.957667+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":150,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154162+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:28:59.154162+0000","last_peered":"2024-08-29T15:28:59.154162+0000","last_clean":"2024-08-29T15:28:59.154162+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:28:59.154162+0000","last_undegraded":"2024-08-29T15:28:59.154162+0000","last_fullsized":"2024-08-29T15:28:59.154162+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154148+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:28:59.154148+0000","last_peered":"2024-08-29T15:28:59.154148+0000","last_clean":"2024-08-29T15:28:59.154148+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:28:59.154148+0000","last_undegraded":"2024-08-29T15:28:59.154148+0000","last_fullsized":"2024-08-29T15:28:59.154148+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.888758+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:29:09.888758+0000","last_peered":"2024-08-29T15:29:09.888758+0000","last_clean":"2024-08-29T15:29:09.888758+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:29:09.888758+0000","last_undegraded":"2024-08-29T15:29:09.888758+0000","last_fullsized":"2024-08-29T15:29:09.888758+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":88,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.888789+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:29:09.888789+0000","last_peered":"2024-08-29T15:29:09.888789+0000","last_clean":"2024-08-29T15:29:09.888789+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:29:09.888789+0000","last_undegraded":"2024-08-29T15:29:09.888789+0000","last_fullsized":"2024-08-29T15:29:09.888789+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":460,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957789+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:29:09.957789+0000","last_peered":"2024-08-29T15:29:09.957789+0000","last_clean":"2024-08-29T15:29:09.957789+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:29:09.957789+0000","last_undegraded":"2024-08-29T15:29:09.957789+0000","last_fullsized":"2024-08-29T15:29:09.957789+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":218,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938900+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:29:09.938900+0000","last_peered":"2024-08-29T15:29:09.938900+0000","last_clean":"2024-08-29T15:29:09.938900+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:29:09.938900+0000","last_undegraded":"2024-08-29T15:29:09.938900+0000","last_fullsized":"2024-08-29T15:29:09.938900+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":166,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922638+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:28:58.922638+0000","last_peered":"2024-08-29T15:28:58.922638+0000","last_clean":"2024-08-29T15:28:58.922638+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:28:58.922638+0000","last_undegraded":"2024-08-29T15:28:58.922638+0000","last_fullsized":"2024-08-29T15:28:58.922638+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":418,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938367+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:29:09.938367+0000","last_peered":"2024-08-29T15:29:09.938367+0000","last_clean":"2024-08-29T15:29:09.938367+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:29:09.938367+0000","last_undegraded":"2024-08-29T15:29:09.938367+0000","last_fullsized":"2024-08-29T15:29:09.938367+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":473,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958135+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:29:09.958135+0000","last_peered":"2024-08-29T15:29:09.958135+0000","last_clean":"2024-08-29T15:29:09.958135+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:29:09.958135+0000","last_undegraded":"2024-08-29T15:29:09.958135+0000","last_fullsized":"2024-08-29T15:29:09.958135+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":470,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938245+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:29:09.938245+0000","last_peered":"2024-08-29T15:29:09.938245+0000","last_clean":"2024-08-29T15:29:09.938245+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:29:09.938245+0000","last_undegraded":"2024-08-29T15:29:09.938245+0000","last_fullsized":"2024-08-29T15:29:09.938245+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":138,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154743+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:28:59.154743+0000","last_peered":"2024-08-29T15:28:59.154743+0000","last_clean":"2024-08-29T15:28:59.154743+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:28:59.154743+0000","last_undegraded":"2024-08-29T15:28:59.154743+0000","last_fullsized":"2024-08-29T15:28:59.154743+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":328,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958597+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:29:09.958597+0000","last_peered":"2024-08-29T15:29:09.958597+0000","last_clean":"2024-08-29T15:29:09.958597+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:29:09.958597+0000","last_undegraded":"2024-08-29T15:29:09.958597+0000","last_fullsized":"2024-08-29T15:29:09.958597+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":448,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957894+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:29:09.957894+0000","last_peered":"2024-08-29T15:29:09.957894+0000","last_clean":"2024-08-29T15:29:09.957894+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:29:09.957894+0000","last_undegraded":"2024-08-29T15:29:09.957894+0000","last_fullsized":"2024-08-29T15:29:09.957894+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":451,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957497+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:29:09.957497+0000","last_peered":"2024-08-29T15:29:09.957497+0000","last_clean":"2024-08-29T15:29:09.957497+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:29:09.957497+0000","last_undegraded":"2024-08-29T15:29:09.957497+0000","last_fullsized":"2024-08-29T15:29:09.957497+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":455,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938747+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:29:09.938747+0000","last_peered":"2024-08-29T15:29:09.938747+0000","last_clean":"2024-08-29T15:29:09.938747+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:29:09.938747+0000","last_undegraded":"2024-08-29T15:29:09.938747+0000","last_fullsized":"2024-08-29T15:29:09.938747+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":77,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154335+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:28:59.154335+0000","last_peered":"2024-08-29T15:28:59.154335+0000","last_clean":"2024-08-29T15:28:59.154335+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:28:59.154335+0000","last_undegraded":"2024-08-29T15:28:59.154335+0000","last_fullsized":"2024-08-29T15:28:59.154335+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":476,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958264+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:29:09.958264+0000","last_peered":"2024-08-29T15:29:09.958264+0000","last_clean":"2024-08-29T15:29:09.958264+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:29:09.958264+0000","last_undegraded":"2024-08-29T15:29:09.958264+0000","last_fullsized":"2024-08-29T15:29:09.958264+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":439,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957223+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:29:09.957223+0000","last_peered":"2024-08-29T15:29:09.957223+0000","last_clean":"2024-08-29T15:29:09.957223+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:29:09.957223+0000","last_undegraded":"2024-08-29T15:29:09.957223+0000","last_fullsized":"2024-08-29T15:29:09.957223+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154671+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:28:59.154671+0000","last_peered":"2024-08-29T15:28:59.154671+0000","last_clean":"2024-08-29T15:28:59.154671+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:28:59.154671+0000","last_undegraded":"2024-08-29T15:28:59.154671+0000","last_fullsized":"2024-08-29T15:28:59.154671+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":196,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922608+0000","last_change":"2024-08-29T15:28:35.519733+0000","last_active":"2024-08-29T15:28:58.922608+0000","last_peered":"2024-08-29T15:28:58.922608+0000","last_clean":"2024-08-29T15:28:58.922608+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:28:58.922608+0000","last_undegraded":"2024-08-29T15:28:58.922608+0000","last_fullsized":"2024-08-29T15:28:58.922608+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":77,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.154416+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:28:59.154416+0000","last_peered":"2024-08-29T15:28:59.154416+0000","last_clean":"2024-08-29T15:28:59.154416+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:28:59.154416+0000","last_undegraded":"2024-08-29T15:28:59.154416+0000","last_fullsized":"2024-08-29T15:28:59.154416+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":196,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:58.922794+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:28:58.922794+0000","last_peered":"2024-08-29T15:28:58.922794+0000","last_clean":"2024-08-29T15:28:58.922794+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:28:58.922794+0000","last_undegraded":"2024-08-29T15:28:58.922794+0000","last_fullsized":"2024-08-29T15:28:58.922794+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":486,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.939076+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:29:09.939076+0000","last_peered":"2024-08-29T15:29:09.939076+0000","last_clean":"2024-08-29T15:29:09.939076+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:29:09.939076+0000","last_undegraded":"2024-08-29T15:29:09.939076+0000","last_fullsized":"2024-08-29T15:29:09.939076+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":488,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.939171+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:29:09.939171+0000","last_peered":"2024-08-29T15:29:09.939171+0000","last_clean":"2024-08-29T15:29:09.939171+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:29:09.939171+0000","last_undegraded":"2024-08-29T15:29:09.939171+0000","last_fullsized":"2024-08-29T15:29:09.939171+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-08-29T15:28:59.153699+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:28:59.153699+0000","last_peered":"2024-08-29T15:28:59.153699+0000","last_clean":"2024-08-29T15:28:59.153699+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:28:59.153699+0000","last_undegraded":"2024-08-29T15:28:59.153699+0000","last_fullsized":"2024-08-29T15:28:59.153699+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":496,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.958018+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:29:09.958018+0000","last_peered":"2024-08-29T15:29:09.958018+0000","last_clean":"2024-08-29T15:29:09.958018+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:29:09.958018+0000","last_undegraded":"2024-08-29T15:29:09.958018+0000","last_fullsized":"2024-08-29T15:29:09.958018+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":511639552,"num_objects":7807,"num_object_clones":0,"num_object_copies":23421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7807,"num_write_kb":499648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1552744448,"data_stored":1552744448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7807,"ondisk_log_size":7807,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":6528,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234415,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":471904,"kb_used_data":115928,"kb_used_omap":1,"kb_used_meta":355966,"kb_avail":93273248,"statfs":{"total":95995035648,"available":95511805952,"internally_reserved":0,"allocated":118710272,"data_stored":116949193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":364509643},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496046,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":376280,"kb_used_data":146640,"kb_used_omap":1,"kb_used_meta":229630,"kb_avail":93368872,"statfs":{"total":95995035648,"available":95609724928,"internally_reserved":0,"allocated":150159360,"data_stored":148406143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":235141575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426633,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":283592,"kb_used_data":188288,"kb_used_omap":8,"kb_used_meta":95287,"kb_avail":93461560,"statfs":{"total":95995035648,"available":95704637440,"internally_reserved":0,"allocated":192806912,"data_stored":191065233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":97574794},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019325,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502552,"kb_used_data":160528,"kb_used_omap":1,"kb_used_meta":342014,"kb_avail":93242600,"statfs":{"total":95995035648,"available":95480422400,"internally_reserved":0,"allocated":164380672,"data_stored":162626968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":350222790},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435510,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":779400,"kb_used_data":436800,"kb_used_omap":4,"kb_used_meta":342587,"kb_avail":92965752,"statfs":{"total":95995035648,"available":95196930048,"internally_reserved":0,"allocated":447283200,"data_stored":445541353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":350809353},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903361,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":207816,"kb_used_data":3900,"kb_used_omap":10,"kb_used_meta":203893,"kb_avail":93537336,"statfs":{"total":95995035648,"available":95782232064,"internally_reserved":0,"allocated":3993600,"data_stored":2255195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":208786960},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762563,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":828740,"kb_used_data":536060,"kb_used_omap":7,"kb_used_meta":292664,"kb_avail":92916412,"statfs":{"total":95995035648,"available":95146405888,"internally_reserved":0,"allocated":548925440,"data_stored":547186473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":299688290},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926089,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":423372,"kb_used_data":76548,"kb_used_omap":7,"kb_used_meta":346808,"kb_avail":93321780,"statfs":{"total":95995035648,"available":95561502720,"internally_reserved":0,"allocated":78385152,"data_stored":76638896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":355131535},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":174391296,"data_stored":174391296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":249888768,"data_stored":249888768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":447873024,"data_stored":447873024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":158859264,"data_stored":158859264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":191365120,"data_stored":191365120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":166658048,"data_stored":166658048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":163708928,"data_stored":163708928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:29:20.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:20.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:20.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:19 smithi171 ceph-mon[90031]: from='client.55978 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:19 smithi171 ceph-mon[90031]: pgmap v1045: 89 pgs: 89 active+clean; 544 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-08-29T15:29:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:19 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:29:20.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:20.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:20.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:20.427 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:20.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:20.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:20.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:20.477 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 8 16 21200 21184 165.485 137.625 0.00880526 0.00603656 2024-08-29T15:29:20.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:20.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:20.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:20.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:20.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:20.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:20.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:20.849 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:20.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:20.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:20.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:21.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:21.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:21.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:21.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:21.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:21.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:21.402 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:21.435 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:21.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 9 16 23201 23185 160.993 125.062 0.0053939 0.00620562 2024-08-29T15:29:21.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:21.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:21.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:21.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:21.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:21.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:21.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.855 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:21.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:21.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:21.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:21.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:22.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:22.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:22.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:22.224 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:22.261 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:22.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:21 smithi171 ceph-mon[90031]: pgmap v1046: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:29:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:21 smithi027 ceph-mon[93921]: pgmap v1046: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:29:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:21 smithi027 ceph-mon[98970]: pgmap v1046: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:29:22.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:22.363 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:22.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:22.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:22.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 10 15 25458 25443 159.003 141.125 0.00527766 0.00628216 2024-08-29T15:29:22.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:22.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:22.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:22.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:22.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:22.696 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:22.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:22.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:22.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:22.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:22.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:22.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:22.976 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:22.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:22.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:23.078 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:23.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:23.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:23.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:23.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:29:23.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:23.235 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:23.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:23.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:23.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:23.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:23.465 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:23.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 11 16 27402 27386 155.587 121.438 0.00429575 0.00642206 2024-08-29T15:29:23.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:23.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:23.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:23.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:23.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:23.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:23.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:23.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:23.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:23.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:23.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:23.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:23.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:23.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:23.996 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:29:24.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:24.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:24.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:24.166 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:24.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:24.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:24.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:23 smithi027 ceph-mon[98970]: pgmap v1047: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-29T15:29:24.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4254980574' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:29:24.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:29:24.246 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:23 smithi027 ceph-mon[93921]: pgmap v1047: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-29T15:29:24.246 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4254980574' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:29:24.246 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:24 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:29:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:23 smithi171 ceph-mon[90031]: pgmap v1047: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-29T15:29:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4254980574' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:29:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:29:24.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:24.337 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:24.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:24.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:24.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:24.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 12 16 29284 29268 152.423 117.625 0.0123802 0.00655461 2024-08-29T15:29:24.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:24.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:24.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:24.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:24.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:24.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:24.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:24.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:24.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:24.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:24.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:24.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:24.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:24.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:24.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:24.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:25.014 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:25.116 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:25.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:25.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:25.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:25 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:29:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:25 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:29:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:25 smithi171 ceph-mon[90031]: osdmap e298: 8 total, 8 up, 8 in 2024-08-29T15:29:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:25.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:25.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:29:25.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:29:25.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[98970]: osdmap e298: 8 total, 8 up, 8 in 2024-08-29T15:29:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:25.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:29:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:29:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[93921]: osdmap e298: 8 total, 8 up, 8 in 2024-08-29T15:29:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:25.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:25.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:25.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:25.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 13 16 31090 31074 149.38 112.875 0.00471037 0.00668933 2024-08-29T15:29:25.524 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:25.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:25.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:25.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:25.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:25.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:25.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:25.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:25.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:25.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:25.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:25.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:26.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:26.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:26.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:26.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:26.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:26.226 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:26.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:26.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:26 smithi171 ceph-mon[90031]: pgmap v1049: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-08-29T15:29:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:26 smithi027 ceph-mon[98970]: pgmap v1049: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-08-29T15:29:26.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:26 smithi027 ceph-mon[93921]: pgmap v1049: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-08-29T15:29:26.328 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:26.350 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:26.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:26.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:26.452 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:26.474 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:26.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:26.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 14 16 32246 32230 143.87 72.25 0.00180809 0.00685682 2024-08-29T15:29:26.577 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:26.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:26.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:26.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:26.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:26.771 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:26.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:26.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:26.873 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:26.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:26.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:27.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:27.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:27.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:27.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:27.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:27.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:27.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:27.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:27.413 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:27.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:27.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:27.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 15 16 33795 33779 140.733 96.8125 0.0214823 0.0070926 2024-08-29T15:29:27.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:27.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:27.670 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:27.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:27.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:27.748 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:29:27.749 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:29:27.789 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:29:27.789 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:29:27.799 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:27.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:27.826 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:29:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.826 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:29:27.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:27.869 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:29:27.870 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:29:27.904 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:29:27.904 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:29:27.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:27.939 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:29:27.939 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:29:27.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:27.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:27.975 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:29:27.975 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:29:28.007 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:29:28.007 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:29:28.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:28.095 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:28.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:28.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:28.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:28.225 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:28.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:28 smithi027 ceph-mon[98970]: pgmap v1050: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-29T15:29:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2850988004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:28 smithi027 ceph-mon[93921]: pgmap v1050: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-29T15:29:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2850988004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:28.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:28.355 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:28.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:28.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:28.457 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:28.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 16 16 35629 35613 139.1 114.625 0.00944964 0.00718244 2024-08-29T15:29:28.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:28.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:28.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:28 smithi171 ceph-mon[90031]: pgmap v1050: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-29T15:29:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2850988004' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:28.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:28.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:28.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:28.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:29:28.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:28.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:28.778 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:28.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:28.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:28.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:28.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:28.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:29.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:29.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:29.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:29.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:29.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:29.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:29:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:29:29] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-08-29T15:29:29.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:29.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:29.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:29.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:29.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 17 16 37377 37361 137.344 109.25 0.00318367 0.0072742 2024-08-29T15:29:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:29.555 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:29.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:29.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:29.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:29.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:29.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:29.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:29.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:29.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:29.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:29.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:29.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:30.077 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:30.096 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:30.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:30.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:30.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:30.136 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:29:30.136 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:29:30.136 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:29:30.136 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:29:30.136 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:29:30.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:30.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:30.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:30.303 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:30 smithi027 ceph-mon[98970]: pgmap v1051: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-08-29T15:29:30.303 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:30 smithi027 ceph-mon[93921]: pgmap v1051: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-08-29T15:29:30.365 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:30.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:30.394 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:30.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:30.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 18 16 39020 39004 135.418 102.688 0.0118178 0.00737931 2024-08-29T15:29:30.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:30 smithi171 ceph-mon[90031]: pgmap v1051: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-08-29T15:29:30.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:30.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:30.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:30.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:30.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:30.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:30.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:30.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:30.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:30.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:30.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:30.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:30.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:30.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:31.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:31.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:31.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:31.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:31.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:31.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:31.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:31.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:31.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:31.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:31.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:31.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:31.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:31.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:31.478 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 19 15 40786 40771 134.103 110.438 0.00513929 0.00745128 2024-08-29T15:29:31.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:29:31.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:31.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:31.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:31.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:31.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:31.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:31.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:31.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:31.871 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:31.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:31.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:32.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:32.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:32.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:32.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:32.264 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:32.291 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:32 smithi027 ceph-mon[98970]: pgmap v1052: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-08-29T15:29:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:32 smithi027 ceph-mon[93921]: pgmap v1052: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-08-29T15:29:32.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:32.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:32.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:29:32.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:29:32.478720+0000 min lat: 0.00124846 max lat: 0.409089 avg lat: 0.00753795 2024-08-29T15:29:32.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:29:32.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 20 16 42437 42421 132.553 103.125 0.0131899 0.00753795 2024-08-29T15:29:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:32 smithi171 ceph-mon[90031]: pgmap v1052: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-08-29T15:29:32.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:32.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:32.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:32.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:32.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:32.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:32.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:32.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:32.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:32.977 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:32.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:32.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:33.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:33.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:33.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:33.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:33.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:33.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:33.356 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:33.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:33.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:33.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:33.479 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 21 16 43800 43784 130.297 85.1875 0.00809599 0.00766931 2024-08-29T15:29:33.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:33.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:33.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:33.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:33.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:33.641 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:33.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:33.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:33.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:33.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:33.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:33.889 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:33.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:33.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:34.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:34.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:34.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:34.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:34.145 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:34.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:34.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:34.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:34.312 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:34 smithi027 ceph-mon[93921]: pgmap v1053: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-08-29T15:29:34.312 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:34 smithi027 ceph-mon[98970]: pgmap v1053: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-08-29T15:29:34.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:34.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:34.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:34.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:34.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 22 16 45260 45244 128.517 91.25 0.00884696 0.00777445 2024-08-29T15:29:34.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:34.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:34.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:34 smithi171 ceph-mon[90031]: pgmap v1053: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2024-08-29T15:29:34.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:34.623 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:34.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:34.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:34.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:34.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:34.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:34.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:34.896 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:34.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:34.997 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:35.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:35.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:35.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:35.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:35.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:35.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:35.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:35.377 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:29:35.378 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:29:35.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:35.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:35.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 23 16 46723 46707 126.904 91.4375 0.0087827 0.00787504 2024-08-29T15:29:35.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:35.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:35.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:35.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:35.844 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:35.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:35.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:35.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:35.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:36.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:36.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:36.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:36.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:36.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:36.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:36.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:36 smithi027 ceph-mon[98970]: pgmap v1054: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:29:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:36 smithi027 ceph-mon[98970]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:36.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:36 smithi027 ceph-mon[93921]: pgmap v1054: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:29:36.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:36 smithi027 ceph-mon[93921]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:36.319 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:36 smithi171 ceph-mon[90031]: pgmap v1054: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:29:36.319 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:36 smithi171 ceph-mon[90031]: from='client.36294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:29:36.357 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:36.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:36.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:36.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:36.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 24 16 48253 48237 125.601 95.625 0.00720039 0.00795631 2024-08-29T15:29:36.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:36.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:36.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:36.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:36.660 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1054,"stamp":"2024-08-29T15:29:34.710350+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2704349157,"num_objects":41220,"num_object_clones":0,"num_object_copies":123660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41220,"num_whiteouts":0,"num_read":6365,"num_read_kb":13822,"num_write":42315,"num_write_kb":2651329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":10166259,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42374,"ondisk_log_size":42374,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10417460,"kb_used_data":8028212,"kb_used_omap":43,"kb_used_meta":2389140,"kb_avail":739543756,"statfs":{"total":767960285184,"available":757292806144,"internally_reserved":0,"allocated":8220889088,"data_stored":8206850630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2446479852},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,1,1,6,2,8],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1346371584,"num_objects":20544,"num_object_clones":0,"num_object_copies":61632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20544,"num_write_kb":1314816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.719185"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":482,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999144+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:29:23.999144+0000","last_peered":"2024-08-29T15:29:23.999144+0000","last_clean":"2024-08-29T15:29:23.999144+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:29:23.999144+0000","last_undegraded":"2024-08-29T15:29:23.999144+0000","last_fullsized":"2024-08-29T15:29:23.999144+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":86,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956724+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:29:09.956724+0000","last_peered":"2024-08-29T15:29:09.956724+0000","last_clean":"2024-08-29T15:29:09.956724+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:29:09.956724+0000","last_undegraded":"2024-08-29T15:29:09.956724+0000","last_fullsized":"2024-08-29T15:29:09.956724+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936750+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:29:09.936750+0000","last_peered":"2024-08-29T15:29:09.936750+0000","last_clean":"2024-08-29T15:29:09.936750+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:29:09.936750+0000","last_undegraded":"2024-08-29T15:29:09.936750+0000","last_fullsized":"2024-08-29T15:29:09.936750+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":449,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.997785+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:29:23.997785+0000","last_peered":"2024-08-29T15:29:23.997785+0000","last_clean":"2024-08-29T15:29:23.997785+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:29:23.997785+0000","last_undegraded":"2024-08-29T15:29:23.997785+0000","last_fullsized":"2024-08-29T15:29:23.997785+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":449,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.006223+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:29:24.006223+0000","last_peered":"2024-08-29T15:29:24.006223+0000","last_clean":"2024-08-29T15:29:24.006223+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:29:24.006223+0000","last_undegraded":"2024-08-29T15:29:24.006223+0000","last_fullsized":"2024-08-29T15:29:24.006223+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956956+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:29:09.956956+0000","last_peered":"2024-08-29T15:29:09.956956+0000","last_clean":"2024-08-29T15:29:09.956956+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:29:09.956956+0000","last_undegraded":"2024-08-29T15:29:09.956956+0000","last_fullsized":"2024-08-29T15:29:09.956956+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":260,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993735+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:29:23.993735+0000","last_peered":"2024-08-29T15:29:23.993735+0000","last_clean":"2024-08-29T15:29:23.993735+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:29:23.993735+0000","last_undegraded":"2024-08-29T15:29:23.993735+0000","last_fullsized":"2024-08-29T15:29:23.993735+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":169,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938556+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:29:09.938556+0000","last_peered":"2024-08-29T15:29:09.938556+0000","last_clean":"2024-08-29T15:29:09.938556+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:29:09.938556+0000","last_undegraded":"2024-08-29T15:29:09.938556+0000","last_fullsized":"2024-08-29T15:29:09.938556+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"298'2563","reported_seq":2582,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:30.802846+0000","last_change":"2024-08-29T15:28:57.922512+0000","last_active":"2024-08-29T15:29:30.802846+0000","last_peered":"2024-08-29T15:29:30.802846+0000","last_clean":"2024-08-29T15:29:30.802846+0000","last_became_active":"2024-08-29T15:28:57.922093+0000","last_became_peered":"2024-08-29T15:28:57.922093+0000","last_unstale":"2024-08-29T15:29:30.802846+0000","last_undegraded":"2024-08-29T15:29:30.802846+0000","last_fullsized":"2024-08-29T15:29:30.802846+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2563,"log_dups_size":0,"ondisk_log_size":2563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:26.218110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167968768,"num_objects":2563,"num_object_clones":0,"num_object_copies":7689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2563,"num_write_kb":164032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":83,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956869+0000","last_change":"2024-08-29T15:28:35.524176+0000","last_active":"2024-08-29T15:29:09.956869+0000","last_peered":"2024-08-29T15:29:09.956869+0000","last_clean":"2024-08-29T15:29:09.956869+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:29:09.956869+0000","last_undegraded":"2024-08-29T15:29:09.956869+0000","last_fullsized":"2024-08-29T15:29:09.956869+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":186,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936304+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:29:09.936304+0000","last_peered":"2024-08-29T15:29:09.936304+0000","last_clean":"2024-08-29T15:29:09.936304+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:29:09.936304+0000","last_undegraded":"2024-08-29T15:29:09.936304+0000","last_fullsized":"2024-08-29T15:29:09.936304+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"298'2640","reported_seq":2658,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:31.551612+0000","last_change":"2024-08-29T15:28:57.922760+0000","last_active":"2024-08-29T15:29:31.551612+0000","last_peered":"2024-08-29T15:29:31.551612+0000","last_clean":"2024-08-29T15:29:31.551612+0000","last_became_active":"2024-08-29T15:28:57.922490+0000","last_became_peered":"2024-08-29T15:28:57.922490+0000","last_unstale":"2024-08-29T15:29:31.551612+0000","last_undegraded":"2024-08-29T15:29:31.551612+0000","last_fullsized":"2024-08-29T15:29:31.551612+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2640,"log_dups_size":0,"ondisk_log_size":2640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:46:47.703614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173015040,"num_objects":2640,"num_object_clones":0,"num_object_copies":7920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2640,"num_write_kb":168960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":83,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956822+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:29:09.956822+0000","last_peered":"2024-08-29T15:29:09.956822+0000","last_clean":"2024-08-29T15:29:09.956822+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:29:09.956822+0000","last_undegraded":"2024-08-29T15:29:09.956822+0000","last_fullsized":"2024-08-29T15:29:09.956822+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956379+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:29:09.956379+0000","last_peered":"2024-08-29T15:29:09.956379+0000","last_clean":"2024-08-29T15:29:09.956379+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:29:09.956379+0000","last_undegraded":"2024-08-29T15:29:09.956379+0000","last_fullsized":"2024-08-29T15:29:09.956379+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"298'2706","reported_seq":2725,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:34.104092+0000","last_change":"2024-08-29T15:28:57.926748+0000","last_active":"2024-08-29T15:29:34.104092+0000","last_peered":"2024-08-29T15:29:34.104092+0000","last_clean":"2024-08-29T15:29:34.104092+0000","last_became_active":"2024-08-29T15:28:57.926391+0000","last_became_peered":"2024-08-29T15:28:57.926391+0000","last_unstale":"2024-08-29T15:29:34.104092+0000","last_undegraded":"2024-08-29T15:29:34.104092+0000","last_fullsized":"2024-08-29T15:29:34.104092+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2706,"log_dups_size":0,"ondisk_log_size":2706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:30:16.060709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177340416,"num_objects":2706,"num_object_clones":0,"num_object_copies":8118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2706,"num_write_kb":173184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":265,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.006197+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:29:24.006197+0000","last_peered":"2024-08-29T15:29:24.006197+0000","last_clean":"2024-08-29T15:29:24.006197+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:29:24.006197+0000","last_undegraded":"2024-08-29T15:29:24.006197+0000","last_fullsized":"2024-08-29T15:29:24.006197+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":124,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938665+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:29:09.938665+0000","last_peered":"2024-08-29T15:29:09.938665+0000","last_clean":"2024-08-29T15:29:09.938665+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:29:09.938665+0000","last_undegraded":"2024-08-29T15:29:09.938665+0000","last_fullsized":"2024-08-29T15:29:09.938665+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"298'2422","reported_seq":2435,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:29.884371+0000","last_change":"2024-08-29T15:28:58.126108+0000","last_active":"2024-08-29T15:29:29.884371+0000","last_peered":"2024-08-29T15:29:29.884371+0000","last_clean":"2024-08-29T15:29:29.884371+0000","last_became_active":"2024-08-29T15:28:58.125784+0000","last_became_peered":"2024-08-29T15:28:58.125784+0000","last_unstale":"2024-08-29T15:29:29.884371+0000","last_undegraded":"2024-08-29T15:29:29.884371+0000","last_fullsized":"2024-08-29T15:29:29.884371+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2422,"log_dups_size":0,"ondisk_log_size":2422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:15:58.792157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158728192,"num_objects":2422,"num_object_clones":0,"num_object_copies":7266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2422,"num_write_kb":155008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936755+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:29:09.936755+0000","last_peered":"2024-08-29T15:29:09.936755+0000","last_clean":"2024-08-29T15:29:09.936755+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:29:09.936755+0000","last_undegraded":"2024-08-29T15:29:09.936755+0000","last_fullsized":"2024-08-29T15:29:09.936755+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":463,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.995556+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:29:23.995556+0000","last_peered":"2024-08-29T15:29:23.995556+0000","last_clean":"2024-08-29T15:29:23.995556+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:29:23.995556+0000","last_undegraded":"2024-08-29T15:29:23.995556+0000","last_fullsized":"2024-08-29T15:29:23.995556+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"298'2735","reported_seq":2754,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:34.113780+0000","last_change":"2024-08-29T15:28:58.127033+0000","last_active":"2024-08-29T15:29:34.113780+0000","last_peered":"2024-08-29T15:29:34.113780+0000","last_clean":"2024-08-29T15:29:34.113780+0000","last_became_active":"2024-08-29T15:28:58.126072+0000","last_became_peered":"2024-08-29T15:28:58.126072+0000","last_unstale":"2024-08-29T15:29:34.113780+0000","last_undegraded":"2024-08-29T15:29:34.113780+0000","last_fullsized":"2024-08-29T15:29:34.113780+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2735,"log_dups_size":0,"ondisk_log_size":2735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:29:18.967329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179240960,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2735,"num_write_kb":175040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":457,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.998932+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:29:23.998932+0000","last_peered":"2024-08-29T15:29:23.998932+0000","last_clean":"2024-08-29T15:29:23.998932+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:29:23.998932+0000","last_undegraded":"2024-08-29T15:29:23.998932+0000","last_fullsized":"2024-08-29T15:29:23.998932+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956391+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:29:09.956391+0000","last_peered":"2024-08-29T15:29:09.956391+0000","last_clean":"2024-08-29T15:29:09.956391+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:29:09.956391+0000","last_undegraded":"2024-08-29T15:29:09.956391+0000","last_fullsized":"2024-08-29T15:29:09.956391+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"298'2584","reported_seq":2603,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:32.210887+0000","last_change":"2024-08-29T15:28:57.924976+0000","last_active":"2024-08-29T15:29:32.210887+0000","last_peered":"2024-08-29T15:29:32.210887+0000","last_clean":"2024-08-29T15:29:32.210887+0000","last_became_active":"2024-08-29T15:28:57.924668+0000","last_became_peered":"2024-08-29T15:28:57.924668+0000","last_unstale":"2024-08-29T15:29:32.210887+0000","last_undegraded":"2024-08-29T15:29:32.210887+0000","last_fullsized":"2024-08-29T15:29:32.210887+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2584,"log_dups_size":0,"ondisk_log_size":2584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:33:52.529565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169345024,"num_objects":2584,"num_object_clones":0,"num_object_copies":7752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2584,"num_write_kb":165376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":142,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938547+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:29:09.938547+0000","last_peered":"2024-08-29T15:29:09.938547+0000","last_clean":"2024-08-29T15:29:09.938547+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:29:09.938547+0000","last_undegraded":"2024-08-29T15:29:09.938547+0000","last_fullsized":"2024-08-29T15:29:09.938547+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956557+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:29:09.956557+0000","last_peered":"2024-08-29T15:29:09.956557+0000","last_clean":"2024-08-29T15:29:09.956557+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:29:09.956557+0000","last_undegraded":"2024-08-29T15:29:09.956557+0000","last_fullsized":"2024-08-29T15:29:09.956557+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"298'2404","reported_seq":2417,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:29.981878+0000","last_change":"2024-08-29T15:28:57.926441+0000","last_active":"2024-08-29T15:29:29.981878+0000","last_peered":"2024-08-29T15:29:29.981878+0000","last_clean":"2024-08-29T15:29:29.981878+0000","last_became_active":"2024-08-29T15:28:57.926173+0000","last_became_peered":"2024-08-29T15:28:57.926173+0000","last_unstale":"2024-08-29T15:29:29.981878+0000","last_undegraded":"2024-08-29T15:29:29.981878+0000","last_fullsized":"2024-08-29T15:29:29.981878+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2404,"log_dups_size":0,"ondisk_log_size":2404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:43.091728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157548544,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2404,"num_write_kb":153856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956386+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:29:09.956386+0000","last_peered":"2024-08-29T15:29:09.956386+0000","last_clean":"2024-08-29T15:29:09.956386+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:29:09.956386+0000","last_undegraded":"2024-08-29T15:29:09.956386+0000","last_fullsized":"2024-08-29T15:29:09.956386+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":285,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999527+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:29:23.999527+0000","last_peered":"2024-08-29T15:29:23.999527+0000","last_clean":"2024-08-29T15:29:23.999527+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:29:23.999527+0000","last_undegraded":"2024-08-29T15:29:23.999527+0000","last_fullsized":"2024-08-29T15:29:23.999527+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"298'2352","reported_seq":2365,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:29.995099+0000","last_change":"2024-08-29T15:28:58.126443+0000","last_active":"2024-08-29T15:29:29.995099+0000","last_peered":"2024-08-29T15:29:29.995099+0000","last_clean":"2024-08-29T15:29:29.995099+0000","last_became_active":"2024-08-29T15:28:58.126054+0000","last_became_peered":"2024-08-29T15:28:58.126054+0000","last_unstale":"2024-08-29T15:29:29.995099+0000","last_undegraded":"2024-08-29T15:29:29.995099+0000","last_fullsized":"2024-08-29T15:29:29.995099+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2352,"log_dups_size":0,"ondisk_log_size":2352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:42:38.613662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154140672,"num_objects":2352,"num_object_clones":0,"num_object_copies":7056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2352,"num_write_kb":150528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":460,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.000066+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:29:24.000066+0000","last_peered":"2024-08-29T15:29:24.000066+0000","last_clean":"2024-08-29T15:29:24.000066+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:29:24.000066+0000","last_undegraded":"2024-08-29T15:29:24.000066+0000","last_fullsized":"2024-08-29T15:29:24.000066+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":453,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.995873+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:29:23.995873+0000","last_peered":"2024-08-29T15:29:23.995873+0000","last_clean":"2024-08-29T15:29:23.995873+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:29:23.995873+0000","last_undegraded":"2024-08-29T15:29:23.995873+0000","last_fullsized":"2024-08-29T15:29:23.995873+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"298'2755","reported_seq":2773,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:34.121580+0000","last_change":"2024-08-29T15:28:58.127095+0000","last_active":"2024-08-29T15:29:34.121580+0000","last_peered":"2024-08-29T15:29:34.121580+0000","last_clean":"2024-08-29T15:29:34.121580+0000","last_became_active":"2024-08-29T15:28:58.126393+0000","last_became_peered":"2024-08-29T15:28:58.126393+0000","last_unstale":"2024-08-29T15:29:34.121580+0000","last_undegraded":"2024-08-29T15:29:34.121580+0000","last_fullsized":"2024-08-29T15:29:34.121580+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2755,"log_dups_size":0,"ondisk_log_size":2755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:50:34.827428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180551680,"num_objects":2755,"num_object_clones":0,"num_object_copies":8265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2755,"num_write_kb":176320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":552,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.995198+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:29:23.995198+0000","last_peered":"2024-08-29T15:29:23.995198+0000","last_clean":"2024-08-29T15:29:23.995198+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:29:23.995198+0000","last_undegraded":"2024-08-29T15:29:23.995198+0000","last_fullsized":"2024-08-29T15:29:23.995198+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":770,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936477+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:29:09.936477+0000","last_peered":"2024-08-29T15:29:09.936477+0000","last_clean":"2024-08-29T15:29:09.936477+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:29:09.936477+0000","last_undegraded":"2024-08-29T15:29:09.936477+0000","last_fullsized":"2024-08-29T15:29:09.936477+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":278,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936438+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:29:09.936438+0000","last_peered":"2024-08-29T15:29:09.936438+0000","last_clean":"2024-08-29T15:29:09.936438+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:29:09.936438+0000","last_undegraded":"2024-08-29T15:29:09.936438+0000","last_fullsized":"2024-08-29T15:29:09.936438+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":140,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938797+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:29:09.938797+0000","last_peered":"2024-08-29T15:29:09.938797+0000","last_clean":"2024-08-29T15:29:09.938797+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:29:09.938797+0000","last_undegraded":"2024-08-29T15:29:09.938797+0000","last_fullsized":"2024-08-29T15:29:09.938797+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"298'2773","reported_seq":2792,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:34.090394+0000","last_change":"2024-08-29T15:28:58.125997+0000","last_active":"2024-08-29T15:29:34.090394+0000","last_peered":"2024-08-29T15:29:34.090394+0000","last_clean":"2024-08-29T15:29:34.090394+0000","last_became_active":"2024-08-29T15:28:58.125680+0000","last_became_peered":"2024-08-29T15:28:58.125680+0000","last_unstale":"2024-08-29T15:29:34.090394+0000","last_undegraded":"2024-08-29T15:29:34.090394+0000","last_fullsized":"2024-08-29T15:29:34.090394+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2773,"log_dups_size":0,"ondisk_log_size":2773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:30:49.334821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181731328,"num_objects":2773,"num_object_clones":0,"num_object_copies":8319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2773,"num_write_kb":177472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":926,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993113+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:29:23.993113+0000","last_peered":"2024-08-29T15:29:23.993113+0000","last_clean":"2024-08-29T15:29:23.993113+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:29:23.993113+0000","last_undegraded":"2024-08-29T15:29:23.993113+0000","last_fullsized":"2024-08-29T15:29:23.993113+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":464,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.997104+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:29:23.997104+0000","last_peered":"2024-08-29T15:29:23.997104+0000","last_clean":"2024-08-29T15:29:23.997104+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:29:23.997104+0000","last_undegraded":"2024-08-29T15:29:23.997104+0000","last_fullsized":"2024-08-29T15:29:23.997104+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":472,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.006527+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:29:24.006527+0000","last_peered":"2024-08-29T15:29:24.006527+0000","last_clean":"2024-08-29T15:29:24.006527+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:29:24.006527+0000","last_undegraded":"2024-08-29T15:29:24.006527+0000","last_fullsized":"2024-08-29T15:29:24.006527+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"298'2808","reported_seq":2827,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:34.081059+0000","last_change":"2024-08-29T15:28:57.925020+0000","last_active":"2024-08-29T15:29:34.081059+0000","last_peered":"2024-08-29T15:29:34.081059+0000","last_clean":"2024-08-29T15:29:34.081059+0000","last_became_active":"2024-08-29T15:28:57.924490+0000","last_became_peered":"2024-08-29T15:28:57.924490+0000","last_unstale":"2024-08-29T15:29:34.081059+0000","last_undegraded":"2024-08-29T15:29:34.081059+0000","last_fullsized":"2024-08-29T15:29:34.081059+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2808,"log_dups_size":0,"ondisk_log_size":2808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:53:05.182238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184025088,"num_objects":2808,"num_object_clones":0,"num_object_copies":8424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2808,"num_write_kb":179712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":734,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.998945+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:29:23.998945+0000","last_peered":"2024-08-29T15:29:23.998945+0000","last_clean":"2024-08-29T15:29:23.998945+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:29:23.998945+0000","last_undegraded":"2024-08-29T15:29:23.998945+0000","last_fullsized":"2024-08-29T15:29:23.998945+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938299+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:29:09.938299+0000","last_peered":"2024-08-29T15:29:09.938299+0000","last_clean":"2024-08-29T15:29:09.938299+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:29:09.938299+0000","last_undegraded":"2024-08-29T15:29:09.938299+0000","last_fullsized":"2024-08-29T15:29:09.938299+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":499,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993633+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:29:23.993633+0000","last_peered":"2024-08-29T15:29:23.993633+0000","last_clean":"2024-08-29T15:29:23.993633+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:29:23.993633+0000","last_undegraded":"2024-08-29T15:29:23.993633+0000","last_fullsized":"2024-08-29T15:29:23.993633+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"298'2440","reported_seq":2453,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:29.996734+0000","last_change":"2024-08-29T15:28:57.926317+0000","last_active":"2024-08-29T15:29:29.996734+0000","last_peered":"2024-08-29T15:29:29.996734+0000","last_clean":"2024-08-29T15:29:29.996734+0000","last_became_active":"2024-08-29T15:28:57.926116+0000","last_became_peered":"2024-08-29T15:28:57.926116+0000","last_unstale":"2024-08-29T15:29:29.996734+0000","last_undegraded":"2024-08-29T15:29:29.996734+0000","last_fullsized":"2024-08-29T15:29:29.996734+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2440,"log_dups_size":0,"ondisk_log_size":2440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:35:53.613903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159907840,"num_objects":2440,"num_object_clones":0,"num_object_copies":7320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2440,"num_write_kb":156160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":238,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936392+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:29:09.936392+0000","last_peered":"2024-08-29T15:29:09.936392+0000","last_clean":"2024-08-29T15:29:09.936392+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:29:09.936392+0000","last_undegraded":"2024-08-29T15:29:09.936392+0000","last_fullsized":"2024-08-29T15:29:09.936392+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956600+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:29:09.956600+0000","last_peered":"2024-08-29T15:29:09.956600+0000","last_clean":"2024-08-29T15:29:09.956600+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:29:09.956600+0000","last_undegraded":"2024-08-29T15:29:09.956600+0000","last_fullsized":"2024-08-29T15:29:09.956600+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936453+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:29:09.936453+0000","last_peered":"2024-08-29T15:29:09.936453+0000","last_clean":"2024-08-29T15:29:09.936453+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:29:09.936453+0000","last_undegraded":"2024-08-29T15:29:09.936453+0000","last_fullsized":"2024-08-29T15:29:09.936453+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.1","version":"298'2406","reported_seq":2419,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:29.791305+0000","last_change":"2024-08-29T15:28:57.924977+0000","last_active":"2024-08-29T15:29:29.791305+0000","last_peered":"2024-08-29T15:29:29.791305+0000","last_clean":"2024-08-29T15:29:29.791305+0000","last_became_active":"2024-08-29T15:28:57.924686+0000","last_became_peered":"2024-08-29T15:28:57.924686+0000","last_unstale":"2024-08-29T15:29:29.791305+0000","last_undegraded":"2024-08-29T15:29:29.791305+0000","last_fullsized":"2024-08-29T15:29:29.791305+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2406,"log_dups_size":0,"ondisk_log_size":2406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:07:46.518392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2406,"num_write_kb":153984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":170,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936608+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:29:09.936608+0000","last_peered":"2024-08-29T15:29:09.936608+0000","last_clean":"2024-08-29T15:29:09.936608+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:29:09.936608+0000","last_undegraded":"2024-08-29T15:29:09.936608+0000","last_fullsized":"2024-08-29T15:29:09.936608+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":891,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993843+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:29:23.993843+0000","last_peered":"2024-08-29T15:29:23.993843+0000","last_clean":"2024-08-29T15:29:23.993843+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:29:23.993843+0000","last_undegraded":"2024-08-29T15:29:23.993843+0000","last_fullsized":"2024-08-29T15:29:23.993843+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":468,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993629+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:29:23.993629+0000","last_peered":"2024-08-29T15:29:23.993629+0000","last_clean":"2024-08-29T15:29:23.993629+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:29:23.993629+0000","last_undegraded":"2024-08-29T15:29:23.993629+0000","last_fullsized":"2024-08-29T15:29:23.993629+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"298'2533","reported_seq":2546,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:31.363348+0000","last_change":"2024-08-29T15:28:57.922386+0000","last_active":"2024-08-29T15:29:31.363348+0000","last_peered":"2024-08-29T15:29:31.363348+0000","last_clean":"2024-08-29T15:29:31.363348+0000","last_became_active":"2024-08-29T15:28:57.922067+0000","last_became_peered":"2024-08-29T15:28:57.922067+0000","last_unstale":"2024-08-29T15:29:31.363348+0000","last_undegraded":"2024-08-29T15:29:31.363348+0000","last_fullsized":"2024-08-29T15:29:31.363348+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2533,"log_dups_size":0,"ondisk_log_size":2533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:11:27.444379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166002688,"num_objects":2533,"num_object_clones":0,"num_object_copies":7599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2533,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":473,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993738+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:29:23.993738+0000","last_peered":"2024-08-29T15:29:23.993738+0000","last_clean":"2024-08-29T15:29:23.993738+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:29:23.993738+0000","last_undegraded":"2024-08-29T15:29:23.993738+0000","last_fullsized":"2024-08-29T15:29:23.993738+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":750,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956872+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:29:09.956872+0000","last_peered":"2024-08-29T15:29:09.956872+0000","last_clean":"2024-08-29T15:29:09.956872+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:29:09.956872+0000","last_undegraded":"2024-08-29T15:29:09.956872+0000","last_fullsized":"2024-08-29T15:29:09.956872+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":464,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.995487+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:29:23.995487+0000","last_peered":"2024-08-29T15:29:23.995487+0000","last_clean":"2024-08-29T15:29:23.995487+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:29:23.995487+0000","last_undegraded":"2024-08-29T15:29:23.995487+0000","last_fullsized":"2024-08-29T15:29:23.995487+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"298'2576","reported_seq":2595,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:31.557533+0000","last_change":"2024-08-29T15:28:57.924025+0000","last_active":"2024-08-29T15:29:31.557533+0000","last_peered":"2024-08-29T15:29:31.557533+0000","last_clean":"2024-08-29T15:29:31.557533+0000","last_became_active":"2024-08-29T15:28:57.923719+0000","last_became_peered":"2024-08-29T15:28:57.923719+0000","last_unstale":"2024-08-29T15:29:31.557533+0000","last_undegraded":"2024-08-29T15:29:31.557533+0000","last_fullsized":"2024-08-29T15:29:31.557533+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2576,"log_dups_size":0,"ondisk_log_size":2576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:46.870950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168820736,"num_objects":2576,"num_object_clones":0,"num_object_copies":7728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2576,"num_write_kb":164864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":457,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999632+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:29:23.999632+0000","last_peered":"2024-08-29T15:29:23.999632+0000","last_clean":"2024-08-29T15:29:23.999632+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:29:23.999632+0000","last_undegraded":"2024-08-29T15:29:23.999632+0000","last_fullsized":"2024-08-29T15:29:23.999632+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1110,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.995767+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:29:23.995767+0000","last_peered":"2024-08-29T15:29:23.995767+0000","last_clean":"2024-08-29T15:29:23.995767+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:29:23.995767+0000","last_undegraded":"2024-08-29T15:29:23.995767+0000","last_fullsized":"2024-08-29T15:29:23.995767+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":455,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999969+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:29:23.999969+0000","last_peered":"2024-08-29T15:29:23.999969+0000","last_clean":"2024-08-29T15:29:23.999969+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:29:23.999969+0000","last_undegraded":"2024-08-29T15:29:23.999969+0000","last_fullsized":"2024-08-29T15:29:23.999969+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"298'2495","reported_seq":2514,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:31.557258+0000","last_change":"2024-08-29T15:28:57.923922+0000","last_active":"2024-08-29T15:29:31.557258+0000","last_peered":"2024-08-29T15:29:31.557258+0000","last_clean":"2024-08-29T15:29:31.557258+0000","last_became_active":"2024-08-29T15:28:57.923593+0000","last_became_peered":"2024-08-29T15:28:57.923593+0000","last_unstale":"2024-08-29T15:29:31.557258+0000","last_undegraded":"2024-08-29T15:29:31.557258+0000","last_fullsized":"2024-08-29T15:29:31.557258+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":2495,"log_dups_size":0,"ondisk_log_size":2495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:48:59.077875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163512320,"num_objects":2495,"num_object_clones":0,"num_object_copies":7485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2495,"num_write_kb":159680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":454,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.006089+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:29:24.006089+0000","last_peered":"2024-08-29T15:29:24.006089+0000","last_clean":"2024-08-29T15:29:24.006089+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:29:24.006089+0000","last_undegraded":"2024-08-29T15:29:24.006089+0000","last_fullsized":"2024-08-29T15:29:24.006089+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":151,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.938829+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:29:09.938829+0000","last_peered":"2024-08-29T15:29:09.938829+0000","last_clean":"2024-08-29T15:29:09.938829+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:29:09.938829+0000","last_undegraded":"2024-08-29T15:29:09.938829+0000","last_fullsized":"2024-08-29T15:29:09.938829+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.957022+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:29:09.957022+0000","last_peered":"2024-08-29T15:29:09.957022+0000","last_clean":"2024-08-29T15:29:09.957022+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:29:09.957022+0000","last_undegraded":"2024-08-29T15:29:09.957022+0000","last_fullsized":"2024-08-29T15:29:09.957022+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":79,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956736+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:29:09.956736+0000","last_peered":"2024-08-29T15:29:09.956736+0000","last_clean":"2024-08-29T15:29:09.956736+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:29:09.956736+0000","last_undegraded":"2024-08-29T15:29:09.956736+0000","last_fullsized":"2024-08-29T15:29:09.956736+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":89,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956438+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:29:09.956438+0000","last_peered":"2024-08-29T15:29:09.956438+0000","last_clean":"2024-08-29T15:29:09.956438+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:29:09.956438+0000","last_undegraded":"2024-08-29T15:29:09.956438+0000","last_fullsized":"2024-08-29T15:29:09.956438+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":462,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993487+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:29:23.993487+0000","last_peered":"2024-08-29T15:29:23.993487+0000","last_clean":"2024-08-29T15:29:23.993487+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:29:23.993487+0000","last_undegraded":"2024-08-29T15:29:23.993487+0000","last_fullsized":"2024-08-29T15:29:23.993487+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":220,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999022+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:29:23.999022+0000","last_peered":"2024-08-29T15:29:23.999022+0000","last_clean":"2024-08-29T15:29:23.999022+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:29:23.999022+0000","last_undegraded":"2024-08-29T15:29:23.999022+0000","last_fullsized":"2024-08-29T15:29:23.999022+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":167,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936697+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:29:09.936697+0000","last_peered":"2024-08-29T15:29:09.936697+0000","last_clean":"2024-08-29T15:29:09.936697+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:29:09.936697+0000","last_undegraded":"2024-08-29T15:29:09.936697+0000","last_fullsized":"2024-08-29T15:29:09.936697+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":420,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.998605+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:29:23.998605+0000","last_peered":"2024-08-29T15:29:23.998605+0000","last_clean":"2024-08-29T15:29:23.998605+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:29:23.998605+0000","last_undegraded":"2024-08-29T15:29:23.998605+0000","last_fullsized":"2024-08-29T15:29:23.998605+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":475,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.006749+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:29:24.006749+0000","last_peered":"2024-08-29T15:29:24.006749+0000","last_clean":"2024-08-29T15:29:24.006749+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:29:24.006749+0000","last_undegraded":"2024-08-29T15:29:24.006749+0000","last_fullsized":"2024-08-29T15:29:24.006749+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":472,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999761+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:29:23.999761+0000","last_peered":"2024-08-29T15:29:23.999761+0000","last_clean":"2024-08-29T15:29:23.999761+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:29:23.999761+0000","last_undegraded":"2024-08-29T15:29:23.999761+0000","last_fullsized":"2024-08-29T15:29:23.999761+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":140,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:20.004427+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:29:20.004427+0000","last_peered":"2024-08-29T15:29:20.004427+0000","last_clean":"2024-08-29T15:29:20.004427+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:29:20.004427+0000","last_undegraded":"2024-08-29T15:29:20.004427+0000","last_fullsized":"2024-08-29T15:29:20.004427+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":330,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.995095+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:29:23.995095+0000","last_peered":"2024-08-29T15:29:23.995095+0000","last_clean":"2024-08-29T15:29:23.995095+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:29:23.995095+0000","last_undegraded":"2024-08-29T15:29:23.995095+0000","last_fullsized":"2024-08-29T15:29:23.995095+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":451,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.006307+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:29:24.006307+0000","last_peered":"2024-08-29T15:29:24.006307+0000","last_clean":"2024-08-29T15:29:24.006307+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:29:24.006307+0000","last_undegraded":"2024-08-29T15:29:24.006307+0000","last_fullsized":"2024-08-29T15:29:24.006307+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":453,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993316+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:29:23.993316+0000","last_peered":"2024-08-29T15:29:23.993316+0000","last_clean":"2024-08-29T15:29:23.993316+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:29:23.993316+0000","last_undegraded":"2024-08-29T15:29:23.993316+0000","last_fullsized":"2024-08-29T15:29:23.993316+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":457,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.998871+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:29:23.998871+0000","last_peered":"2024-08-29T15:29:23.998871+0000","last_clean":"2024-08-29T15:29:23.998871+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:29:23.998871+0000","last_undegraded":"2024-08-29T15:29:23.998871+0000","last_fullsized":"2024-08-29T15:29:23.998871+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":78,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956743+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:29:09.956743+0000","last_peered":"2024-08-29T15:29:09.956743+0000","last_clean":"2024-08-29T15:29:09.956743+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:29:09.956743+0000","last_undegraded":"2024-08-29T15:29:09.956743+0000","last_fullsized":"2024-08-29T15:29:09.956743+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":478,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.993883+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:29:23.993883+0000","last_peered":"2024-08-29T15:29:23.993883+0000","last_clean":"2024-08-29T15:29:23.993883+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:29:23.993883+0000","last_undegraded":"2024-08-29T15:29:23.993883+0000","last_fullsized":"2024-08-29T15:29:23.993883+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":441,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.995406+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:29:23.995406+0000","last_peered":"2024-08-29T15:29:23.995406+0000","last_clean":"2024-08-29T15:29:23.995406+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:29:23.995406+0000","last_undegraded":"2024-08-29T15:29:23.995406+0000","last_fullsized":"2024-08-29T15:29:23.995406+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956507+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:29:09.956507+0000","last_peered":"2024-08-29T15:29:09.956507+0000","last_clean":"2024-08-29T15:29:09.956507+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:29:09.956507+0000","last_undegraded":"2024-08-29T15:29:09.956507+0000","last_fullsized":"2024-08-29T15:29:09.956507+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":197,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936644+0000","last_change":"2024-08-29T15:28:35.519733+0000","last_active":"2024-08-29T15:29:09.936644+0000","last_peered":"2024-08-29T15:29:09.936644+0000","last_clean":"2024-08-29T15:29:09.936644+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:29:09.936644+0000","last_undegraded":"2024-08-29T15:29:09.936644+0000","last_fullsized":"2024-08-29T15:29:09.936644+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.956818+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:29:09.956818+0000","last_peered":"2024-08-29T15:29:09.956818+0000","last_clean":"2024-08-29T15:29:09.956818+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:29:09.956818+0000","last_undegraded":"2024-08-29T15:29:09.956818+0000","last_fullsized":"2024-08-29T15:29:09.956818+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":197,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-08-29T15:29:09.936749+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:29:09.936749+0000","last_peered":"2024-08-29T15:29:09.936749+0000","last_clean":"2024-08-29T15:29:09.936749+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:29:09.936749+0000","last_undegraded":"2024-08-29T15:29:09.936749+0000","last_fullsized":"2024-08-29T15:29:09.936749+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":488,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999070+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:29:23.999070+0000","last_peered":"2024-08-29T15:29:23.999070+0000","last_clean":"2024-08-29T15:29:23.999070+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:29:23.999070+0000","last_undegraded":"2024-08-29T15:29:23.999070+0000","last_fullsized":"2024-08-29T15:29:23.999070+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":490,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.999741+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:29:23.999741+0000","last_peered":"2024-08-29T15:29:23.999741+0000","last_clean":"2024-08-29T15:29:23.999741+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:29:23.999741+0000","last_undegraded":"2024-08-29T15:29:23.999741+0000","last_fullsized":"2024-08-29T15:29:23.999741+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:23.998101+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:29:23.998101+0000","last_peered":"2024-08-29T15:29:23.998101+0000","last_clean":"2024-08-29T15:29:23.998101+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:29:23.998101+0000","last_undegraded":"2024-08-29T15:29:23.998101+0000","last_fullsized":"2024-08-29T15:29:23.998101+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":498,"reported_epoch":298,"state":"active+clean","last_fresh":"2024-08-29T15:29:24.006687+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:29:24.006687+0000","last_peered":"2024-08-29T15:29:24.006687+0000","last_clean":"2024-08-29T15:29:24.006687+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:29:24.006687+0000","last_undegraded":"2024-08-29T15:29:24.006687+0000","last_fullsized":"2024-08-29T15:29:24.006687+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2699558912,"num_objects":41192,"num_object_clones":0,"num_object_copies":123576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":41192,"num_write_kb":2636288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7990476800,"data_stored":7990476800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41192,"ondisk_log_size":41192,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":6528,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234418,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1127984,"kb_used_data":731816,"kb_used_omap":1,"kb_used_meta":396158,"kb_avail":92617168,"statfs":{"total":95995035648,"available":94839980032,"internally_reserved":0,"allocated":749379584,"data_stored":747610592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":405666251},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011880,"kb_used_data":774048,"kb_used_omap":1,"kb_used_meta":237822,"kb_avail":92733272,"statfs":{"total":95995035648,"available":94958870528,"internally_reserved":0,"allocated":792625152,"data_stored":790864022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243530183},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426636,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":748824,"kb_used_data":657616,"kb_used_omap":8,"kb_used_meta":91191,"kb_avail":92996328,"statfs":{"total":95995035648,"available":95228239872,"internally_reserved":0,"allocated":673398784,"data_stored":671649192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":93380490},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019328,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152744,"kb_used_data":770528,"kb_used_omap":1,"kb_used_meta":382206,"kb_avail":92592408,"statfs":{"total":95995035648,"available":94814625792,"internally_reserved":0,"allocated":789020672,"data_stored":787259055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":391379398},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435513,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1851736,"kb_used_data":1477200,"kb_used_omap":4,"kb_used_meta":374523,"kb_avail":91893416,"statfs":{"total":95995035648,"available":94098857984,"internally_reserved":0,"allocated":1512652800,"data_stored":1510903040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":383511817},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903365,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199124,"kb_used_data":991052,"kb_used_omap":10,"kb_used_meta":208053,"kb_avail":92546028,"statfs":{"total":95995035648,"available":94767132672,"internally_reserved":0,"allocated":1014837248,"data_stored":1013090930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":213046800},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762566,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1555028,"kb_used_data":1238732,"kb_used_omap":7,"kb_used_meta":316280,"kb_avail":92190124,"statfs":{"total":95995035648,"available":94402686976,"internally_reserved":0,"allocated":1268461568,"data_stored":1266714688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":323871074},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926093,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1770140,"kb_used_data":1387220,"kb_used_omap":7,"kb_used_meta":382904,"kb_avail":91975012,"statfs":{"total":95995035648,"available":94182412288,"internally_reserved":0,"allocated":1420513280,"data_stored":1418759111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":392093839},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,8],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1452015616,"data_stored":1452015616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":971243520,"data_stored":971243520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1036451840,"data_stored":1036451840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1508900864,"data_stored":1508900864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":780926976,"data_stored":780926976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":673251328,"data_stored":673251328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":790822912,"data_stored":790822912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":776863744,"data_stored":776863744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:29:36.661 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:36.665 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-29T15:29:36.665 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:29:36.665 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-08-29T15:29:36.665 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:29:36.665 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:29:36.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-08-29T15:29:36.676 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:36.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:36.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:36.801 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:36.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:36.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:36.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:36.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:36.972 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:29:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:36.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:36.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:37.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:37.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:37.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:37.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:37.267 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:37.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:37.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:37.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:37.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:37.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 25 16 49704 49688 124.204 90.6875 0.0109195 0.00804547 2024-08-29T15:29:37.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:37.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:37.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:37.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:37.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:37.682 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:37.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:37.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:37.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:37.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:37.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:37.921 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:37.951 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:37.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:37.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:38.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:38.077 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:29:38.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:38.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:38.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:38.214 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:38.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:38.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:38.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:38 smithi027 ceph-mon[93921]: pgmap v1055: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:29:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2168887805' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:38 smithi027 ceph-mon[98970]: pgmap v1055: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:29:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2168887805' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:38.349 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:38.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:38.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:38.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:38.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 26 16 51215 51199 123.059 94.4375 0.0039265 0.0081197 2024-08-29T15:29:38.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:38.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:38.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:38.495 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:38 smithi171 ceph-mon[90031]: pgmap v1055: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:29:38.495 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2168887805' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:29:38.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:38.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:38.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:38.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:38.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:38.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:38.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:38.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:38.880 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:38.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:38.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:38.968 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:29:38.968 INFO:teuthology.orchestra.run.smithi171.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:29:38.968 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:29:38.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:39.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:39.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:39.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:39.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:39.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:39.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:39.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:29:39.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:29:39.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2168887805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:29:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[98970]: osdmap e299: 8 total, 8 up, 8 in 2024-08-29T15:29:39.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:29:39.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:29:39.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2168887805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:29:39.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-mon[93921]: osdmap e299: 8 total, 8 up, 8 in 2024-08-29T15:29:39.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:29:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:29:39] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-08-29T15:29:39.338 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:39 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:29:39.338 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:39 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:29:39.339 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:39 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2168887805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:29:39.339 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:39 smithi171 ceph-mon[90031]: osdmap e299: 8 total, 8 up, 8 in 2024-08-29T15:29:39.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:39.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:39.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 27 16 52762 52746 122.082 96.6875 0.00599559 0.00818614 2024-08-29T15:29:39.524 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:39.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:39.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:39.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:39.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:39.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:39.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:39.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:39.974 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:39.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:40.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:40.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:40.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:40.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:40.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:40.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:40.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:40.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:40.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:40.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:40.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:40.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:40 smithi027 ceph-mon[93921]: pgmap v1057: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-29T15:29:40.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:40 smithi027 ceph-mon[98970]: pgmap v1057: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-29T15:29:40.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 28 16 54166 54150 120.855 87.75 0.0149105 0.00826913 2024-08-29T15:29:40.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:40 smithi171 ceph-mon[90031]: pgmap v1057: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-08-29T15:29:40.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:40.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:40.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:40.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:40.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:40.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:40.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:40.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:40.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:40.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:40.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:40.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:41.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:41.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:41.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:41.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:41.156 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:41.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:41.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:41.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:41.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:41.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:41.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:41.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:41.425 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:41.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:41.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:41.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 29 16 55709 55693 120.013 96.4375 0.00451241 0.00832743 2024-08-29T15:29:41.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:41.598 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:41.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:41.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:41.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:41.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:41.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:41.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:41.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:41.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:41.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:42.004 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:42.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:42.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:42.135 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:42.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:42.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:42.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:42.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:42.341 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:42 smithi027 ceph-mon[93921]: pgmap v1058: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-29T15:29:42.341 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:42 smithi027 ceph-mon[98970]: pgmap v1058: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-29T15:29:42.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:42.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:42.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:42.480 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 30 16 57248 57232 119.218 96.1875 0.0130362 0.00838244 2024-08-29T15:29:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:42 smithi171 ceph-mon[90031]: pgmap v1058: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-29T15:29:42.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:42.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:42.654 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:42.684 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:42.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:42.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:42.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:42.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:42.994 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:42.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:42.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:43.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:43.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:43.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:43.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:43.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:43.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:43.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:43.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:43.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:43.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:43.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:43.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 31 16 58668 58652 118.231 88.75 0.0142282 0.00845237 2024-08-29T15:29:43.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:43.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:43.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:43.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:43.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:43.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:43.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:43.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:43.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:43.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:43.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:43.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:43.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:44.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:44.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:44.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:44.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:44.162 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:44 smithi027 ceph-mon[98970]: pgmap v1059: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:29:44.163 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:44.163 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:44.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:44 smithi027 ceph-mon[93921]: pgmap v1059: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:29:44.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:44.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:44.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:44.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:44.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:44.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:44.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:44.371 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:44.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:44.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:44.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:29:44.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:44.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 32 16 60053 60037 117.241 86.5625 0.00983758 0.00852273 2024-08-29T15:29:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:44 smithi171 ceph-mon[90031]: pgmap v1059: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-29T15:29:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:44.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:44.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:44.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:44.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:44.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:44.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:44.676 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:29:44.676 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:29:44.676 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-08-29T15:29:44.676 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:29:44.676 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:29:44.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-08-29T15:29:44.742 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:29:44.743 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:29:44.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:44.784 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:29:44.785 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:29:44.822 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:29:44.823 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:29:44.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:44.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:44.865 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:29:44.865 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:29:44.901 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:29:44.901 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:29:44.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:44.940 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:29:44.941 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:29:44.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:44.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:29:44.973 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:29:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:44.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:45.012 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:29:45.012 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:29:45.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:45.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:45.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:45.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:45.237 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:45.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:45.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:45.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:45.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2089010201' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:45.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2089010201' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:45.366 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:45.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:45.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:45.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 33 16 61215 61199 115.889 72.625 0.00509094 0.00862365 2024-08-29T15:29:45.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2089010201' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:29:45.524 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:45.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:45.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:45.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:45.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:45.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:45.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:45.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:45.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:45.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:45.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:45.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:45.930 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:45.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:45.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:46.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:46.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:46.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:46.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:46.142 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:46.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:46.198 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:29:46.198 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:29:46.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:46.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:46.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:46.206 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:46.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:46.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:46.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:46.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:46.441 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:46 smithi027 ceph-mon[98970]: pgmap v1060: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-29T15:29:46.441 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:46 smithi027 ceph-mon[93921]: pgmap v1060: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-29T15:29:46.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:46.475 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:46.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:46.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 34 16 62550 62534 114.934 83.4375 0.0203268 0.00869508 2024-08-29T15:29:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:46 smithi171 ceph-mon[90031]: pgmap v1060: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-08-29T15:29:46.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:46.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:46.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:46.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:46.704 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:46.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:46.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:46.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:46.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:46.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:46.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:46.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:46.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:47.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:47.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:47.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:47.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:29:47.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:47.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:47.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:47.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:47.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:47.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:47.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:47.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:47.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:47.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:47.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:47.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 35 16 63835 63819 113.945 80.3125 0.00947342 0.00876383 2024-08-29T15:29:47.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:47.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:47.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:47.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:47.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:47.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:47.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:47.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:47.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:47.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:47.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:47.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:29:47.984 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:48.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:48.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:48.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:48.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:48.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:48.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:48.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:48 smithi027 ceph-mon[98970]: pgmap v1061: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-29T15:29:48.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:48 smithi027 ceph-mon[93921]: pgmap v1061: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-29T15:29:48.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:48.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:48.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:48.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 36 16 64964 64948 112.74 70.5625 0.00885761 0.0088654 2024-08-29T15:29:48.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:48 smithi171 ceph-mon[90031]: pgmap v1061: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-29T15:29:48.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:48.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:48.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:48.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:48.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:48.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:48.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:48.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:48.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:48.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:49.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:49.076 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:49.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:49.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:49.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:49.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:49.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:49.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:29:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:29:49] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-08-29T15:29:49.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:49.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:49.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:49.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:49.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:49.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:49.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 37 16 66030 66014 111.493 66.625 0.0145654 0.00895694 2024-08-29T15:29:49.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:49.611 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:49.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:49.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:49.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:49.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:49.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:49.844 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:49.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:49.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:49.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:49.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:50.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:50.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:50.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:50.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:50.242 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:50.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:50.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:50.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:50.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:50.457 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:50 smithi027 ceph-mon[93921]: pgmap v1062: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:29:50.457 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:50 smithi027 ceph-mon[98970]: pgmap v1062: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:29:50.482 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 38 16 67176 67160 110.444 71.625 0.0056504 0.00904887 2024-08-29T15:29:50.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:50 smithi171 ceph-mon[90031]: pgmap v1062: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-08-29T15:29:50.533 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:50.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.676 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:50.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:50.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:50.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:50.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:50.926 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:50.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:50.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:51.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:51.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:51.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:51.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:51.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:51.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:51.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:51.325 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:51.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:51.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:51.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:51.483 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 39 16 68485 68469 109.71 81.8125 0.0175285 0.00911002 2024-08-29T15:29:51.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:51.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:51.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:51.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:51.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:51.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:51.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:51.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:51.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:51.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:51.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:51.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:51.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:52.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:52.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:52.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:52.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:52.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:52.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:52.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:52.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:52.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:52.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:52.482 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:52.483 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:29:52.482761+0000 min lat: 0.00124846 max lat: 0.409089 avg lat: 0.00917911 2024-08-29T15:29:52.484 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:29:52.484 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 40 16 69710 69694 108.881 76.5625 0.0108542 0.00917911 2024-08-29T15:29:52.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:52.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:52 smithi171 ceph-mon[90031]: pgmap v1063: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-08-29T15:29:52.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:52 smithi027 ceph-mon[98970]: pgmap v1063: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-08-29T15:29:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:52 smithi027 ceph-mon[93921]: pgmap v1063: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-08-29T15:29:52.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:52.635 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:52.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:52.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:52.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:52.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:52.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:52.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:53.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:53.036 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:53.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:53.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:53.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:53.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:53.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:53.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:53.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:53.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:53.483 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 41 16 70554 70538 107.512 52.75 0.011311 0.00929644 2024-08-29T15:29:53.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:53.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:53.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:53.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:53.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:53.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:53.846 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:53.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:53.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:53.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:53.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:54.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:54.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:54.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:54.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:54.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:54.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:54.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:54.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:54.357 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:54.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:54.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:54.483 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 42 16 71613 71597 106.526 66.1875 0.0173209 0.00938208 2024-08-29T15:29:54.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:54.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:54 smithi171 ceph-mon[90031]: pgmap v1064: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-29T15:29:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:54 smithi027 ceph-mon[98970]: pgmap v1064: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-29T15:29:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:54 smithi027 ceph-mon[93921]: pgmap v1064: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-29T15:29:54.597 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:54.625 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:54.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:54.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:54.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:54.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:54.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:54.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:54.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:54.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:54.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:55.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:55.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:55.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:55.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:55.293 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:55.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:55.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:55.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:55.484 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 43 16 72889 72873 105.903 79.75 0.0123768 0.00943762 2024-08-29T15:29:55.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:55.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:55.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:55.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:55.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:55.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:55.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:55.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:55.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:55.962 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:55.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:55.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:56.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:56.119 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:56.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:56.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:56.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:56.268 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:56.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:56.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:56.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:56.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:56.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:56.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:56 smithi027 ceph-mon[98970]: pgmap v1065: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-29T15:29:56.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:56 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:56.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:56 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:56.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:56 smithi027 ceph-mon[93921]: pgmap v1065: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-29T15:29:56.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:56 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:56.466 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:56 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:56.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 44 16 74044 74028 105.132 72.1875 0.0224073 0.00950605 2024-08-29T15:29:56.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:56 smithi171 ceph-mon[90031]: pgmap v1065: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-29T15:29:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:56 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:56 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:56.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:56.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:56.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:56.653 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:29:56.653 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:29:56.653 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:29:56.672 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:29:56.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:56.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:56.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:56.800 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:56.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:56.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:56.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:56.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:56.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:57.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:57.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:57.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:29:57.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.239 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:57.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:29:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:29:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:29:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:57.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:29:57.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:29:57.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:57.340 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:57.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:57.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:57.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:57.486 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 45 16 74763 74747 103.794 44.9375 0.00356869 0.00958741 2024-08-29T15:29:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:29:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:29:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:29:57.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:57.611 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:57.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:57.735 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:57.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:57.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:57.863 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:57.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:57.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:57.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:57.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:57.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:58.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:58.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:58.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:58.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:58.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:58.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:58.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:58.397 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:58.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:58.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:58 smithi027 ceph-mon[98970]: pgmap v1066: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-29T15:29:58.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:58 smithi027 ceph-mon[93921]: pgmap v1066: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-29T15:29:58.486 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 46 16 75656 75640 102.751 55.8125 0.0113739 0.00972615 2024-08-29T15:29:58.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:58 smithi171 ceph-mon[90031]: pgmap v1066: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-29T15:29:58.527 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:58.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:58.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:58.629 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:29:58.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:58.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:58.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:29:58.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:29:58.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:58.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:58.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:29:58.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:29:58.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:58.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:58.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:29:59.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:29:59.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:29:59.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:29:59.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:29:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:29:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:29:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:29:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:29:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:29:59] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-08-29T15:29:59.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:59.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:59.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:59.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:59.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:59.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:59.486 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 47 16 76792 76776 102.076 71 0.0307754 0.00978974 2024-08-29T15:29:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:29:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:29:59.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:59.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:59.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:29:59.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:29:59.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:59.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:59.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:29:59.962 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:29:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:29:59.984 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:29:59.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:00.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:00.125 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:00.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:00.227 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:00.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:00.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:00.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:00.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:00.399 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:00.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:00.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:00.486 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 48 16 77452 77436 100.809 41.25 0.0092687 0.00991443 2024-08-29T15:30:00.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:00 smithi171 ceph-mon[90031]: pgmap v1067: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:00 smithi171 ceph-mon[90031]: overall HEALTH_OK 2024-08-29T15:30:00.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:00.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:00.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:00.561 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:00 smithi027 ceph-mon[98970]: pgmap v1067: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:00.561 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:00 smithi027 ceph-mon[98970]: overall HEALTH_OK 2024-08-29T15:30:00.561 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:00 smithi027 ceph-mon[93921]: pgmap v1067: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:00.561 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:00 smithi027 ceph-mon[93921]: overall HEALTH_OK 2024-08-29T15:30:00.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:00.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:00.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:00.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:00.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:00.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:00.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:00.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:00.848 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:00.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:00.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:00.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:01.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:01.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:01.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:01.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:01.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:01.223 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:01.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:01.325 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:01.357 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:01.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:01.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:01.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:01.486 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 49 16 78582 78566 100.192 70.625 0.0179259 0.00997457 2024-08-29T15:30:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:01.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:01.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:01.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:01.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:01.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:01.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:01.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:01.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:01.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:01.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:01.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:01.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:02.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:02.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:02.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:02.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:02.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:02.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:02.212 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:30:02.306 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:02 smithi027 ceph-mon[98970]: pgmap v1068: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-29T15:30:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/616913760' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:30:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:02 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:30:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:02 smithi027 ceph-mon[93921]: pgmap v1068: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-29T15:30:02.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/616913760' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:30:02.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:02 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:30:02.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:02.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:02.435 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:02.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:02.486 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 50 16 79743 79727 99.6391 72.5625 0.00765768 0.01003 2024-08-29T15:30:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:02 smithi171 ceph-mon[90031]: pgmap v1068: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-29T15:30:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/616913760' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:30:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:02 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:30:02.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:02.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:02.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:02.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:02.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:02.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:02.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:02.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:02.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:02.984 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:02.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:03.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:03.122 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:03.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:03.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:03.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:03.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:03.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:03.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:03.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:03.484 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:30:03.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:03 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:03.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:03 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:30:03.484 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:03 smithi027 ceph-mon[93921]: osdmap e300: 8 total, 8 up, 8 in 2024-08-29T15:30:03.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:03 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:03.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:03 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:30:03.485 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:03 smithi027 ceph-mon[98970]: osdmap e300: 8 total, 8 up, 8 in 2024-08-29T15:30:03.485 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:30:03.487 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 51 16 80842 80826 99.0321 68.6875 0.0194723 0.0100916 2024-08-29T15:30:03.508 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:03 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:03 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:30:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:03 smithi171 ceph-mon[90031]: osdmap e300: 8 total, 8 up, 8 in 2024-08-29T15:30:03.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:30:03.520 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:30:03.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:03.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:03.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:03.557 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:30:03.557 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:30:03.597 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:30:03.597 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:30:03.634 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:30:03.634 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:30:03.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:03.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:03.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:03.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:03.668 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:30:03.668 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:30:03.706 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:30:03.706 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:30:03.746 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:30:03.746 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:30:03.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:03.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:03.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:03.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:03.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:03.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:03.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:04.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:04.095 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:04.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:04.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:04.235 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:04.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:04.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:04.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:04.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:04.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:04.487 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 52 16 81958 81942 98.4681 69.75 0.0127707 0.0101499 2024-08-29T15:30:04.511 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:04.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:04.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:04 smithi171 ceph-mon[90031]: pgmap v1070: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:30:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1922786728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:04 smithi027 ceph-mon[93921]: pgmap v1070: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:30:04.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1922786728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:04 smithi027 ceph-mon[98970]: pgmap v1070: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:30:04.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1922786728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:04.613 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:04.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:04.641 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:04.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:04.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:04.768 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:04.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:04.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:04.912 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:04.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:05.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:05.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:05.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:05.167 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:05.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:05.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:30:05.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:05.296 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:05.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:05.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:05.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 53 16 83063 83047 97.9132 69.0625 0.0123079 0.0102079 2024-08-29T15:30:05.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:05.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:05.600 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:30:05.600 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:30:05.601 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-29T15:30:05.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:30:05.664 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:05.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:05.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:05.792 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:05.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:05.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:05.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:05.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:05.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:06.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:06.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:06.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:06.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:06.188 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:83822 7.2 e300/294 7:40b15b7a:::benchmark_data_smithi027_271924_object83821:head v 300'5129, mlcod=300'5127)", 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:06.181914+0000", 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00024716300000000001, 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027347500000000002, 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:06.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.181914+0000", 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.181914+0000", 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.181915+0000", 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.009e-06 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.181944+0000", 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8464999999999999e-05 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.181960+0000", 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5804e-05 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.181967+0000", 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.34e-06 2024-08-29T15:30:06.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.182089+0000", 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121929 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.182150+0000", 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0639e-05 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:06.192 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:30:06.193 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2024-08-29T15:30:06.193 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:30:06.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:06.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:06.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:06 smithi027 ceph-mon[93921]: pgmap v1071: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:30:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:06 smithi027 ceph-mon[98970]: pgmap v1071: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:30:06.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:06.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:06.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:06.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:06.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:06.487 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 54 16 84142 84126 97.3488 67.4375 0.00891781 0.0102671 2024-08-29T15:30:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:06 smithi171 ceph-mon[90031]: pgmap v1071: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:30:06.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:06.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:06.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:06.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:06.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:30:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:06.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:06.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:06.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:06.907 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:06.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:07.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:07.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:07.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:07.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:07.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:07.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:07.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:07.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:07.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:07.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:07.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:07.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:07.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:07.435 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:07.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:07.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:07.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 55 16 85024 85008 96.5811 55.125 0.011622 0.0103492 2024-08-29T15:30:07.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:07.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:30:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:07.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:07.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:07.611 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:07.680 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:07.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:07.702 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:07.751 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:07.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:07.852 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:07.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:07.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:08.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:08.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:08.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:08.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:08.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:08.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:08.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:08 smithi171 ceph-mon[90031]: pgmap v1072: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:08.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:08 smithi027 ceph-mon[98970]: pgmap v1072: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:08 smithi027 ceph-mon[93921]: pgmap v1072: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:08.338 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:08.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:08.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:08.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:08.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:08.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 56 16 86027 86011 95.9759 62.6875 0.00929613 0.0104138 2024-08-29T15:30:08.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:08.612 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:08.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:08.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:08.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:08.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:08.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:08.890 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:08.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:08.992 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:09.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:09.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:09.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:09.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:09.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:09.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:30:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:30:09] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.51.0" 2024-08-29T15:30:09.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:09.438 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:09.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:09.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 57 16 87101 87085 95.4696 67.125 0.0163248 0.0104647 2024-08-29T15:30:09.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:09.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:09.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:09.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:09.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:09.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:09.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:09.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:09.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:09.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:10.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:10.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:10.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:10.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:10.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:10.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:10.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:10.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:10.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:10.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:10.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 58 16 88202 88186 95.0099 68.8125 0.0185814 0.0105197 2024-08-29T15:30:10.493 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:10.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:10.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:10 smithi171 ceph-mon[90031]: pgmap v1073: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:10.543 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:10 smithi027 ceph-mon[98970]: pgmap v1073: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:10.543 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:10 smithi027 ceph-mon[93921]: pgmap v1073: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:30:10.595 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:10.623 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:10.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:10.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:10.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:10.756 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:10.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:10.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:10.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:10.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:10.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:10.997 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.046 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:11.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:11.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:11.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:11.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:11.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:11.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:11.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:11.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 59 16 89295 89279 94.5574 68.3125 0.00511899 0.0105701 2024-08-29T15:30:11.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:11.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:11.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:11.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:11.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:11.823 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:11.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:11.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:11.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:11.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:11.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:12.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:12.108 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:12.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:12.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:12.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:12.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:12.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:12.366 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:12.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:12.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:12.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:12.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:12 smithi027 ceph-mon[98970]: pgmap v1074: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-29T15:30:12.421 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:12 smithi027 ceph-mon[93921]: pgmap v1074: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-29T15:30:12.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:30:12.488097+0000 min lat: 0.00124846 max lat: 0.409089 avg lat: 0.0106273 2024-08-29T15:30:12.488 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:30:12.489 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 60 16 90324 90308 94.0532 64.3125 0.0152901 0.0106273 2024-08-29T15:30:12.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:12 smithi171 ceph-mon[90031]: pgmap v1074: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-29T15:30:12.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:12.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:12.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:12.658 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:12.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:12.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:12.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:12.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:12.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:12.977 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:12.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:13.078 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:13.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:13.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:13.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:13.244 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:13.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:13.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:13.261 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:30:13.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2295972816' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:13.290 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2295972816' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:13.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:13.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:13.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:13.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:13.490 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 61 16 91229 91213 93.4346 56.5625 0.0195045 0.0106961 2024-08-29T15:30:13.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2295972816' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:13.523 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:13.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:13.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:13.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:13.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:13.720 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:30:13.721 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:30:13.742 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:30:13.742 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:30:13.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:13.783 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:30:13.784 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:30:13.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:13.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:13.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:13.834 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:30:13.835 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:30:13.872 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:30:13.872 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:30:13.904 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:30:13.904 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:30:13.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:13.940 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:30:13.940 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:30:13.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:13.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:13.976 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:30:13.976 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:30:14.010 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:30:14.010 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:30:14.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:14.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:14.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:14.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:14.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:14.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:14.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:14.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:14.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:14.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:14.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:14.491 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 62 16 92052 92036 92.7572 51.4375 0.00278107 0.0107662 2024-08-29T15:30:14.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:14 smithi171 ceph-mon[90031]: pgmap v1075: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:30:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:14 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2295972816' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:30:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:14 smithi171 ceph-mon[90031]: osdmap e301: 8 total, 8 up, 8 in 2024-08-29T15:30:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2523098104' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:14.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:30:14.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:14.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:14.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[98970]: pgmap v1075: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:30:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2295972816' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[98970]: osdmap e301: 8 total, 8 up, 8 in 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2523098104' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[93921]: pgmap v1075: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2295972816' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[93921]: osdmap e301: 8 total, 8 up, 8 in 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2523098104' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:30:14.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:14.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:14.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:14.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:14.786 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:14.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:14.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:14.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:14.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:14.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:14.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:15.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:15.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:15.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:15.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:15.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:15.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:15.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:15.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:15.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:15.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:15.444 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:15.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:15.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:15.491 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 63 16 92962 92946 92.1876 56.875 0.012806 0.010842 2024-08-29T15:30:15.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:15 smithi027 ceph-mon[98970]: from='client.46250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:30:15.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:15 smithi027 ceph-mon[93921]: from='client.46250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:30:15.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:15.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:15.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:15.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:15.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:15.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:15.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:15 smithi171 ceph-mon[90031]: from='client.46250 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:30:15.861 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:15.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:15.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:15.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:16.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:16.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:16.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:16.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:16.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:16.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:16.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:16.491 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 64 16 93967 93951 91.7286 62.8125 0.00667006 0.0108964 2024-08-29T15:30:16.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:16.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:16.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:16 smithi027 ceph-mon[98970]: pgmap v1077: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:30:16.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:16 smithi027 ceph-mon[93921]: pgmap v1077: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:30:16.589 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:16.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:16.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:16.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:16 smithi171 ceph-mon[90031]: pgmap v1077: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:30:16.791 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:16.893 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:16.920 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:16.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:17.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:17.068 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:17.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:17.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:17.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:17.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:17.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:17.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:17.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:17.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:17.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:17.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:17.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:17.456 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:17.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:17.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:17.491 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 65 16 95048 95032 91.3568 67.5625 0.0183729 0.0109409 2024-08-29T15:30:17.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:17.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:17.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:17.636 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1076,"stamp":"2024-08-29T15:30:13.257084+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5775103973,"num_objects":88076,"num_object_clones":0,"num_object_copies":264228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88076,"num_whiteouts":0,"num_read":6365,"num_read_kb":13822,"num_write":89171,"num_write_kb":5650113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":10166259,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89230,"ondisk_log_size":89230,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19755828,"kb_used_data":17162228,"kb_used_omap":43,"kb_used_meta":2593492,"kb_avail":730205388,"statfs":{"total":767960285184,"available":747730317312,"internally_reserved":0,"allocated":17574121472,"data_stored":17559956406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":2655736300},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,5,7,4,11,9],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":698220544,"num_objects":10654,"num_object_clones":0,"num_object_copies":31962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10654,"num_write_kb":681856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.538569"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":487,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.223022+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:30:02.223022+0000","last_peered":"2024-08-29T15:30:02.223022+0000","last_clean":"2024-08-29T15:30:02.223022+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:30:02.223022+0000","last_undegraded":"2024-08-29T15:30:02.223022+0000","last_fullsized":"2024-08-29T15:30:02.223022+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":88,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084246+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:29:38.084246+0000","last_peered":"2024-08-29T15:29:38.084246+0000","last_clean":"2024-08-29T15:29:38.084246+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:29:38.084246+0000","last_undegraded":"2024-08-29T15:29:38.084246+0000","last_fullsized":"2024-08-29T15:29:38.084246+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":170,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:41.381026+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:29:41.381026+0000","last_peered":"2024-08-29T15:29:41.381026+0000","last_clean":"2024-08-29T15:29:41.381026+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:29:41.381026+0000","last_undegraded":"2024-08-29T15:29:41.381026+0000","last_fullsized":"2024-08-29T15:29:41.381026+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":453,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222649+0000","last_change":"2024-08-29T15:08:58.889616+0000","last_active":"2024-08-29T15:30:02.222649+0000","last_peered":"2024-08-29T15:30:02.222649+0000","last_clean":"2024-08-29T15:30:02.222649+0000","last_became_active":"2024-08-29T15:08:58.889326+0000","last_became_peered":"2024-08-29T15:08:58.889326+0000","last_unstale":"2024-08-29T15:30:02.222649+0000","last_undegraded":"2024-08-29T15:30:02.222649+0000","last_fullsized":"2024-08-29T15:30:02.222649+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:21.581963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":453,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222969+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:30:02.222969+0000","last_peered":"2024-08-29T15:30:02.222969+0000","last_clean":"2024-08-29T15:30:02.222969+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:30:02.222969+0000","last_undegraded":"2024-08-29T15:30:02.222969+0000","last_fullsized":"2024-08-29T15:30:02.222969+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.083861+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:29:38.083861+0000","last_peered":"2024-08-29T15:29:38.083861+0000","last_clean":"2024-08-29T15:29:38.083861+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:29:38.083861+0000","last_undegraded":"2024-08-29T15:29:38.083861+0000","last_fullsized":"2024-08-29T15:29:38.083861+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":264,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.210153+0000","last_change":"2024-08-29T15:18:33.991461+0000","last_active":"2024-08-29T15:30:02.210153+0000","last_peered":"2024-08-29T15:30:02.210153+0000","last_clean":"2024-08-29T15:30:02.210153+0000","last_became_active":"2024-08-29T15:18:33.991125+0000","last_became_peered":"2024-08-29T15:18:33.991125+0000","last_unstale":"2024-08-29T15:30:02.210153+0000","last_undegraded":"2024-08-29T15:30:02.210153+0000","last_fullsized":"2024-08-29T15:30:02.210153+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:07:14.641121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":171,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.080914+0000","last_change":"2024-08-29T15:18:36.379319+0000","last_active":"2024-08-29T15:29:38.080914+0000","last_peered":"2024-08-29T15:29:38.080914+0000","last_clean":"2024-08-29T15:29:38.080914+0000","last_became_active":"2024-08-29T15:18:34.987765+0000","last_became_peered":"2024-08-29T15:18:34.987765+0000","last_unstale":"2024-08-29T15:29:38.080914+0000","last_undegraded":"2024-08-29T15:29:38.080914+0000","last_fullsized":"2024-08-29T15:29:38.080914+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:35:07.198456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"300'5637","reported_seq":5660,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:10.833099+0000","last_change":"2024-08-29T15:28:57.922512+0000","last_active":"2024-08-29T15:30:10.833099+0000","last_peered":"2024-08-29T15:30:10.833099+0000","last_clean":"2024-08-29T15:30:10.833099+0000","last_became_active":"2024-08-29T15:28:57.922093+0000","last_became_peered":"2024-08-29T15:28:57.922093+0000","last_unstale":"2024-08-29T15:30:10.833099+0000","last_undegraded":"2024-08-29T15:30:10.833099+0000","last_fullsized":"2024-08-29T15:30:10.833099+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5637,"log_dups_size":0,"ondisk_log_size":5637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:26.218110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369426432,"num_objects":5637,"num_object_clones":0,"num_object_copies":16911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5637,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":85,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084400+0000","last_change":"2024-08-29T15:28:35.524176+0000","last_active":"2024-08-29T15:29:38.084400+0000","last_peered":"2024-08-29T15:29:38.084400+0000","last_clean":"2024-08-29T15:29:38.084400+0000","last_became_active":"2024-08-29T15:19:46.805404+0000","last_became_peered":"2024-08-29T15:19:46.805404+0000","last_unstale":"2024-08-29T15:29:38.084400+0000","last_undegraded":"2024-08-29T15:29:38.084400+0000","last_fullsized":"2024-08-29T15:29:38.084400+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:58:31.307229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":188,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.078257+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:29:38.078257+0000","last_peered":"2024-08-29T15:29:38.078257+0000","last_clean":"2024-08-29T15:29:38.078257+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:29:38.078257+0000","last_undegraded":"2024-08-29T15:29:38.078257+0000","last_fullsized":"2024-08-29T15:29:38.078257+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"300'5643","reported_seq":5665,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:11.664415+0000","last_change":"2024-08-29T15:28:57.922760+0000","last_active":"2024-08-29T15:30:11.664415+0000","last_peered":"2024-08-29T15:30:11.664415+0000","last_clean":"2024-08-29T15:30:11.664415+0000","last_became_active":"2024-08-29T15:28:57.922490+0000","last_became_peered":"2024-08-29T15:28:57.922490+0000","last_unstale":"2024-08-29T15:30:11.664415+0000","last_undegraded":"2024-08-29T15:30:11.664415+0000","last_fullsized":"2024-08-29T15:30:11.664415+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:46:47.703614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369819648,"num_objects":5643,"num_object_clones":0,"num_object_copies":16929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":361152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":85,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084390+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:29:38.084390+0000","last_peered":"2024-08-29T15:29:38.084390+0000","last_clean":"2024-08-29T15:29:38.084390+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:29:38.084390+0000","last_undegraded":"2024-08-29T15:29:38.084390+0000","last_fullsized":"2024-08-29T15:29:38.084390+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.083687+0000","last_change":"2024-08-29T15:19:46.807910+0000","last_active":"2024-08-29T15:29:38.083687+0000","last_peered":"2024-08-29T15:29:38.083687+0000","last_clean":"2024-08-29T15:29:38.083687+0000","last_became_active":"2024-08-29T15:19:46.807562+0000","last_became_peered":"2024-08-29T15:19:46.807562+0000","last_unstale":"2024-08-29T15:29:38.083687+0000","last_undegraded":"2024-08-29T15:29:38.083687+0000","last_fullsized":"2024-08-29T15:29:38.083687+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:49.357664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"300'5328","reported_seq":5351,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.208049+0000","last_change":"2024-08-29T15:28:57.926748+0000","last_active":"2024-08-29T15:30:09.208049+0000","last_peered":"2024-08-29T15:30:09.208049+0000","last_clean":"2024-08-29T15:30:09.208049+0000","last_became_active":"2024-08-29T15:28:57.926391+0000","last_became_peered":"2024-08-29T15:28:57.926391+0000","last_unstale":"2024-08-29T15:30:09.208049+0000","last_undegraded":"2024-08-29T15:30:09.208049+0000","last_fullsized":"2024-08-29T15:30:09.208049+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5328,"log_dups_size":0,"ondisk_log_size":5328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:30:16.060709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349175808,"num_objects":5328,"num_object_clones":0,"num_object_copies":15984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5328,"num_write_kb":340992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":270,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.225955+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:30:02.225955+0000","last_peered":"2024-08-29T15:30:02.225955+0000","last_clean":"2024-08-29T15:30:02.225955+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:30:02.225955+0000","last_undegraded":"2024-08-29T15:30:02.225955+0000","last_fullsized":"2024-08-29T15:30:02.225955+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":127,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.216911+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:30:02.216911+0000","last_peered":"2024-08-29T15:30:02.216911+0000","last_clean":"2024-08-29T15:30:02.216911+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:30:02.216911+0000","last_undegraded":"2024-08-29T15:30:02.216911+0000","last_fullsized":"2024-08-29T15:30:02.216911+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"300'5536","reported_seq":5550,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.891212+0000","last_change":"2024-08-29T15:28:58.126108+0000","last_active":"2024-08-29T15:30:09.891212+0000","last_peered":"2024-08-29T15:30:09.891212+0000","last_clean":"2024-08-29T15:30:09.891212+0000","last_became_active":"2024-08-29T15:28:58.125784+0000","last_became_peered":"2024-08-29T15:28:58.125784+0000","last_unstale":"2024-08-29T15:30:09.891212+0000","last_undegraded":"2024-08-29T15:30:09.891212+0000","last_fullsized":"2024-08-29T15:30:09.891212+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5536,"log_dups_size":0,"ondisk_log_size":5536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:15:58.792157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362807296,"num_objects":5536,"num_object_clones":0,"num_object_copies":16608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5536,"num_write_kb":354304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":188,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.077824+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:29:38.077824+0000","last_peered":"2024-08-29T15:29:38.077824+0000","last_clean":"2024-08-29T15:29:38.077824+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:29:38.077824+0000","last_undegraded":"2024-08-29T15:29:38.077824+0000","last_fullsized":"2024-08-29T15:29:38.077824+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":468,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222194+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:30:02.222194+0000","last_peered":"2024-08-29T15:30:02.222194+0000","last_clean":"2024-08-29T15:30:02.222194+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:30:02.222194+0000","last_undegraded":"2024-08-29T15:30:02.222194+0000","last_fullsized":"2024-08-29T15:30:02.222194+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"300'5336","reported_seq":5359,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.210904+0000","last_change":"2024-08-29T15:28:58.127033+0000","last_active":"2024-08-29T15:30:09.210904+0000","last_peered":"2024-08-29T15:30:09.210904+0000","last_clean":"2024-08-29T15:30:09.210904+0000","last_became_active":"2024-08-29T15:28:58.126072+0000","last_became_peered":"2024-08-29T15:28:58.126072+0000","last_unstale":"2024-08-29T15:30:09.210904+0000","last_undegraded":"2024-08-29T15:30:09.210904+0000","last_fullsized":"2024-08-29T15:30:09.210904+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5336,"log_dups_size":0,"ondisk_log_size":5336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:29:18.967329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349700096,"num_objects":5336,"num_object_clones":0,"num_object_copies":16008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5336,"num_write_kb":341504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":461,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.221446+0000","last_change":"2024-08-29T15:08:58.889463+0000","last_active":"2024-08-29T15:30:02.221446+0000","last_peered":"2024-08-29T15:30:02.221446+0000","last_clean":"2024-08-29T15:30:02.221446+0000","last_became_active":"2024-08-29T15:08:58.889146+0000","last_became_peered":"2024-08-29T15:08:58.889146+0000","last_unstale":"2024-08-29T15:30:02.221446+0000","last_undegraded":"2024-08-29T15:30:02.221446+0000","last_fullsized":"2024-08-29T15:30:02.221446+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:23.058589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084312+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:29:38.084312+0000","last_peered":"2024-08-29T15:29:38.084312+0000","last_clean":"2024-08-29T15:29:38.084312+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:29:38.084312+0000","last_undegraded":"2024-08-29T15:29:38.084312+0000","last_fullsized":"2024-08-29T15:29:38.084312+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"300'5517","reported_seq":5538,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:12.266909+0000","last_change":"2024-08-29T15:28:57.924976+0000","last_active":"2024-08-29T15:30:12.266909+0000","last_peered":"2024-08-29T15:30:12.266909+0000","last_clean":"2024-08-29T15:30:12.266909+0000","last_became_active":"2024-08-29T15:28:57.924668+0000","last_became_peered":"2024-08-29T15:28:57.924668+0000","last_unstale":"2024-08-29T15:30:12.266909+0000","last_undegraded":"2024-08-29T15:30:12.266909+0000","last_fullsized":"2024-08-29T15:30:12.266909+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5517,"log_dups_size":0,"ondisk_log_size":5517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:33:52.529565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361562112,"num_objects":5517,"num_object_clones":0,"num_object_copies":16551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5517,"num_write_kb":353088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":145,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.216788+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:30:02.216788+0000","last_peered":"2024-08-29T15:30:02.216788+0000","last_clean":"2024-08-29T15:30:02.216788+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:30:02.216788+0000","last_undegraded":"2024-08-29T15:30:02.216788+0000","last_fullsized":"2024-08-29T15:30:02.216788+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084068+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:29:38.084068+0000","last_peered":"2024-08-29T15:29:38.084068+0000","last_clean":"2024-08-29T15:29:38.084068+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:29:38.084068+0000","last_undegraded":"2024-08-29T15:29:38.084068+0000","last_fullsized":"2024-08-29T15:29:38.084068+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"300'5451","reported_seq":5466,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:10.001724+0000","last_change":"2024-08-29T15:28:57.926441+0000","last_active":"2024-08-29T15:30:10.001724+0000","last_peered":"2024-08-29T15:30:10.001724+0000","last_clean":"2024-08-29T15:30:10.001724+0000","last_became_active":"2024-08-29T15:28:57.926173+0000","last_became_peered":"2024-08-29T15:28:57.926173+0000","last_unstale":"2024-08-29T15:30:10.001724+0000","last_undegraded":"2024-08-29T15:30:10.001724+0000","last_fullsized":"2024-08-29T15:30:10.001724+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:43.091728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357236736,"num_objects":5451,"num_object_clones":0,"num_object_copies":16353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5451,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084553+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:29:38.084553+0000","last_peered":"2024-08-29T15:29:38.084553+0000","last_clean":"2024-08-29T15:29:38.084553+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:29:38.084553+0000","last_undegraded":"2024-08-29T15:29:38.084553+0000","last_fullsized":"2024-08-29T15:29:38.084553+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":287,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084486+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:29:38.084486+0000","last_peered":"2024-08-29T15:29:38.084486+0000","last_clean":"2024-08-29T15:29:38.084486+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:29:38.084486+0000","last_undegraded":"2024-08-29T15:29:38.084486+0000","last_fullsized":"2024-08-29T15:29:38.084486+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"300'5513","reported_seq":5528,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:10.003430+0000","last_change":"2024-08-29T15:28:58.126443+0000","last_active":"2024-08-29T15:30:10.003430+0000","last_peered":"2024-08-29T15:30:10.003430+0000","last_clean":"2024-08-29T15:30:10.003430+0000","last_became_active":"2024-08-29T15:28:58.126054+0000","last_became_peered":"2024-08-29T15:28:58.126054+0000","last_unstale":"2024-08-29T15:30:10.003430+0000","last_undegraded":"2024-08-29T15:30:10.003430+0000","last_fullsized":"2024-08-29T15:30:10.003430+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:42:38.613662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361299968,"num_objects":5513,"num_object_clones":0,"num_object_copies":16539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":352832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":464,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.228903+0000","last_change":"2024-08-29T15:08:58.881901+0000","last_active":"2024-08-29T15:30:02.228903+0000","last_peered":"2024-08-29T15:30:02.228903+0000","last_clean":"2024-08-29T15:30:02.228903+0000","last_became_active":"2024-08-29T15:08:58.881461+0000","last_became_peered":"2024-08-29T15:08:58.881461+0000","last_unstale":"2024-08-29T15:30:02.228903+0000","last_undegraded":"2024-08-29T15:30:02.228903+0000","last_fullsized":"2024-08-29T15:30:02.228903+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:09:14.405139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":457,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.210277+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:30:02.210277+0000","last_peered":"2024-08-29T15:30:02.210277+0000","last_clean":"2024-08-29T15:30:02.210277+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:30:02.210277+0000","last_undegraded":"2024-08-29T15:30:02.210277+0000","last_fullsized":"2024-08-29T15:30:02.210277+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"300'5425","reported_seq":5448,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.222705+0000","last_change":"2024-08-29T15:28:58.127095+0000","last_active":"2024-08-29T15:30:09.222705+0000","last_peered":"2024-08-29T15:30:09.222705+0000","last_clean":"2024-08-29T15:30:09.222705+0000","last_became_active":"2024-08-29T15:28:58.126393+0000","last_became_peered":"2024-08-29T15:28:58.126393+0000","last_unstale":"2024-08-29T15:30:09.222705+0000","last_undegraded":"2024-08-29T15:30:09.222705+0000","last_fullsized":"2024-08-29T15:30:09.222705+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5425,"log_dups_size":0,"ondisk_log_size":5425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:50:34.827428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355532800,"num_objects":5425,"num_object_clones":0,"num_object_copies":16275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5425,"num_write_kb":347200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":556,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.221269+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:30:02.221269+0000","last_peered":"2024-08-29T15:30:02.221269+0000","last_clean":"2024-08-29T15:30:02.221269+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:30:02.221269+0000","last_undegraded":"2024-08-29T15:30:02.221269+0000","last_fullsized":"2024-08-29T15:30:02.221269+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:56:24.334516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":773,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.204340+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:30:02.204340+0000","last_peered":"2024-08-29T15:30:02.204340+0000","last_clean":"2024-08-29T15:30:02.204340+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:30:02.204340+0000","last_undegraded":"2024-08-29T15:30:02.204340+0000","last_fullsized":"2024-08-29T15:30:02.204340+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":281,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.204198+0000","last_change":"2024-08-29T15:18:34.989789+0000","last_active":"2024-08-29T15:30:02.204198+0000","last_peered":"2024-08-29T15:30:02.204198+0000","last_clean":"2024-08-29T15:30:02.204198+0000","last_became_active":"2024-08-29T15:18:34.989446+0000","last_became_peered":"2024-08-29T15:18:34.989446+0000","last_unstale":"2024-08-29T15:30:02.204198+0000","last_undegraded":"2024-08-29T15:30:02.204198+0000","last_fullsized":"2024-08-29T15:30:02.204198+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:23:26.089665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":143,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:40.013483+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:29:40.013483+0000","last_peered":"2024-08-29T15:29:40.013483+0000","last_clean":"2024-08-29T15:29:40.013483+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:29:40.013483+0000","last_undegraded":"2024-08-29T15:29:40.013483+0000","last_fullsized":"2024-08-29T15:29:40.013483+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"300'5463","reported_seq":5486,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.152106+0000","last_change":"2024-08-29T15:28:58.125997+0000","last_active":"2024-08-29T15:30:09.152106+0000","last_peered":"2024-08-29T15:30:09.152106+0000","last_clean":"2024-08-29T15:30:09.152106+0000","last_became_active":"2024-08-29T15:28:58.125680+0000","last_became_peered":"2024-08-29T15:28:58.125680+0000","last_unstale":"2024-08-29T15:30:09.152106+0000","last_undegraded":"2024-08-29T15:30:09.152106+0000","last_fullsized":"2024-08-29T15:30:09.152106+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5463,"log_dups_size":0,"ondisk_log_size":5463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:30:49.334821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358023168,"num_objects":5463,"num_object_clones":0,"num_object_copies":16389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5463,"num_write_kb":349632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":931,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.219542+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:30:02.219542+0000","last_peered":"2024-08-29T15:30:02.219542+0000","last_clean":"2024-08-29T15:30:02.219542+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:30:02.219542+0000","last_undegraded":"2024-08-29T15:30:02.219542+0000","last_fullsized":"2024-08-29T15:30:02.219542+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":468,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.219838+0000","last_change":"2024-08-29T15:19:46.801435+0000","last_active":"2024-08-29T15:30:02.219838+0000","last_peered":"2024-08-29T15:30:02.219838+0000","last_clean":"2024-08-29T15:30:02.219838+0000","last_became_active":"2024-08-29T15:19:46.801002+0000","last_became_peered":"2024-08-29T15:19:46.801002+0000","last_unstale":"2024-08-29T15:30:02.219838+0000","last_undegraded":"2024-08-29T15:30:02.219838+0000","last_fullsized":"2024-08-29T15:30:02.219838+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:44.629555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":476,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.223136+0000","last_change":"2024-08-29T15:18:34.187799+0000","last_active":"2024-08-29T15:30:02.223136+0000","last_peered":"2024-08-29T15:30:02.223136+0000","last_clean":"2024-08-29T15:30:02.223136+0000","last_became_active":"2024-08-29T15:18:34.187522+0000","last_became_peered":"2024-08-29T15:18:34.187522+0000","last_unstale":"2024-08-29T15:30:02.223136+0000","last_undegraded":"2024-08-29T15:30:02.223136+0000","last_fullsized":"2024-08-29T15:30:02.223136+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:16:39.817265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"300'5380","reported_seq":5403,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.212625+0000","last_change":"2024-08-29T15:28:57.925020+0000","last_active":"2024-08-29T15:30:09.212625+0000","last_peered":"2024-08-29T15:30:09.212625+0000","last_clean":"2024-08-29T15:30:09.212625+0000","last_became_active":"2024-08-29T15:28:57.924490+0000","last_became_peered":"2024-08-29T15:28:57.924490+0000","last_unstale":"2024-08-29T15:30:09.212625+0000","last_undegraded":"2024-08-29T15:30:09.212625+0000","last_fullsized":"2024-08-29T15:30:09.212625+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5380,"log_dups_size":0,"ondisk_log_size":5380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:53:05.182238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352583680,"num_objects":5380,"num_object_clones":0,"num_object_copies":16140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5380,"num_write_kb":344320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":738,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.226610+0000","last_change":"2024-08-29T15:18:33.993343+0000","last_active":"2024-08-29T15:30:02.226610+0000","last_peered":"2024-08-29T15:30:02.226610+0000","last_clean":"2024-08-29T15:30:02.226610+0000","last_became_active":"2024-08-29T15:18:33.992754+0000","last_became_peered":"2024-08-29T15:18:33.992754+0000","last_unstale":"2024-08-29T15:30:02.226610+0000","last_undegraded":"2024-08-29T15:30:02.226610+0000","last_fullsized":"2024-08-29T15:30:02.226610+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:02:00.394506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.080733+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:29:38.080733+0000","last_peered":"2024-08-29T15:29:38.080733+0000","last_clean":"2024-08-29T15:29:38.080733+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:29:38.080733+0000","last_undegraded":"2024-08-29T15:29:38.080733+0000","last_fullsized":"2024-08-29T15:29:38.080733+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":503,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.209988+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:30:02.209988+0000","last_peered":"2024-08-29T15:30:02.209988+0000","last_clean":"2024-08-29T15:30:02.209988+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:30:02.209988+0000","last_undegraded":"2024-08-29T15:30:02.209988+0000","last_fullsized":"2024-08-29T15:30:02.209988+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"300'5568","reported_seq":5583,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.958822+0000","last_change":"2024-08-29T15:28:57.926317+0000","last_active":"2024-08-29T15:30:09.958822+0000","last_peered":"2024-08-29T15:30:09.958822+0000","last_clean":"2024-08-29T15:30:09.958822+0000","last_became_active":"2024-08-29T15:28:57.926116+0000","last_became_peered":"2024-08-29T15:28:57.926116+0000","last_unstale":"2024-08-29T15:30:09.958822+0000","last_undegraded":"2024-08-29T15:30:09.958822+0000","last_fullsized":"2024-08-29T15:30:09.958822+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:35:53.613903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364904448,"num_objects":5568,"num_object_clones":0,"num_object_copies":16704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5568,"num_write_kb":356352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":241,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.204188+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:30:02.204188+0000","last_peered":"2024-08-29T15:30:02.204188+0000","last_clean":"2024-08-29T15:30:02.204188+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:30:02.204188+0000","last_undegraded":"2024-08-29T15:30:02.204188+0000","last_fullsized":"2024-08-29T15:30:02.204188+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.083952+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:29:38.083952+0000","last_peered":"2024-08-29T15:29:38.083952+0000","last_clean":"2024-08-29T15:29:38.083952+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:29:38.083952+0000","last_undegraded":"2024-08-29T15:29:38.083952+0000","last_fullsized":"2024-08-29T15:29:38.083952+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":148,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.204437+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:30:02.204437+0000","last_peered":"2024-08-29T15:30:02.204437+0000","last_clean":"2024-08-29T15:30:02.204437+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:30:02.204437+0000","last_undegraded":"2024-08-29T15:30:02.204437+0000","last_fullsized":"2024-08-29T15:30:02.204437+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.1","version":"300'5490","reported_seq":5505,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:09.899148+0000","last_change":"2024-08-29T15:28:57.924977+0000","last_active":"2024-08-29T15:30:09.899148+0000","last_peered":"2024-08-29T15:30:09.899148+0000","last_clean":"2024-08-29T15:30:09.899148+0000","last_became_active":"2024-08-29T15:28:57.924686+0000","last_became_peered":"2024-08-29T15:28:57.924686+0000","last_unstale":"2024-08-29T15:30:09.899148+0000","last_undegraded":"2024-08-29T15:30:09.899148+0000","last_fullsized":"2024-08-29T15:30:09.899148+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:07:46.518392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359792640,"num_objects":5490,"num_object_clones":0,"num_object_copies":16470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5490,"num_write_kb":351360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":173,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.204243+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:30:02.204243+0000","last_peered":"2024-08-29T15:30:02.204243+0000","last_clean":"2024-08-29T15:30:02.204243+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:30:02.204243+0000","last_undegraded":"2024-08-29T15:30:02.204243+0000","last_fullsized":"2024-08-29T15:30:02.204243+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":895,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.214740+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:30:02.214740+0000","last_peered":"2024-08-29T15:30:02.214740+0000","last_clean":"2024-08-29T15:30:02.214740+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:30:02.214740+0000","last_undegraded":"2024-08-29T15:30:02.214740+0000","last_fullsized":"2024-08-29T15:30:02.214740+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":472,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.210677+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:30:02.210677+0000","last_peered":"2024-08-29T15:30:02.210677+0000","last_clean":"2024-08-29T15:30:02.210677+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:30:02.210677+0000","last_undegraded":"2024-08-29T15:30:02.210677+0000","last_fullsized":"2024-08-29T15:30:02.210677+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"300'5548","reported_seq":5563,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:11.372612+0000","last_change":"2024-08-29T15:28:57.922386+0000","last_active":"2024-08-29T15:30:11.372612+0000","last_peered":"2024-08-29T15:30:11.372612+0000","last_clean":"2024-08-29T15:30:11.372612+0000","last_became_active":"2024-08-29T15:28:57.922067+0000","last_became_peered":"2024-08-29T15:28:57.922067+0000","last_unstale":"2024-08-29T15:30:11.372612+0000","last_undegraded":"2024-08-29T15:30:11.372612+0000","last_fullsized":"2024-08-29T15:30:11.372612+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:11:27.444379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363593728,"num_objects":5548,"num_object_clones":0,"num_object_copies":16644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5548,"num_write_kb":355072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":477,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.214397+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:30:02.214397+0000","last_peered":"2024-08-29T15:30:02.214397+0000","last_clean":"2024-08-29T15:30:02.214397+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:30:02.214397+0000","last_undegraded":"2024-08-29T15:30:02.214397+0000","last_fullsized":"2024-08-29T15:30:02.214397+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":753,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:55.049796+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:29:55.049796+0000","last_peered":"2024-08-29T15:29:55.049796+0000","last_clean":"2024-08-29T15:29:55.049796+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:29:55.049796+0000","last_undegraded":"2024-08-29T15:29:55.049796+0000","last_fullsized":"2024-08-29T15:29:55.049796+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":468,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222099+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:30:02.222099+0000","last_peered":"2024-08-29T15:30:02.222099+0000","last_clean":"2024-08-29T15:30:02.222099+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:30:02.222099+0000","last_undegraded":"2024-08-29T15:30:02.222099+0000","last_fullsized":"2024-08-29T15:30:02.222099+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"300'5719","reported_seq":5742,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:11.615258+0000","last_change":"2024-08-29T15:28:57.924025+0000","last_active":"2024-08-29T15:30:11.615258+0000","last_peered":"2024-08-29T15:30:11.615258+0000","last_clean":"2024-08-29T15:30:11.615258+0000","last_became_active":"2024-08-29T15:28:57.923719+0000","last_became_peered":"2024-08-29T15:28:57.923719+0000","last_unstale":"2024-08-29T15:30:11.615258+0000","last_undegraded":"2024-08-29T15:30:11.615258+0000","last_fullsized":"2024-08-29T15:30:11.615258+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5719,"log_dups_size":0,"ondisk_log_size":5719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:46.870950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374800384,"num_objects":5719,"num_object_clones":0,"num_object_copies":17157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5719,"num_write_kb":366016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":461,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.223512+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:30:02.223512+0000","last_peered":"2024-08-29T15:30:02.223512+0000","last_clean":"2024-08-29T15:30:02.223512+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:30:02.223512+0000","last_undegraded":"2024-08-29T15:30:02.223512+0000","last_fullsized":"2024-08-29T15:30:02.223512+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1114,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.210672+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:30:02.210672+0000","last_peered":"2024-08-29T15:30:02.210672+0000","last_clean":"2024-08-29T15:30:02.210672+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:30:02.210672+0000","last_undegraded":"2024-08-29T15:30:02.210672+0000","last_fullsized":"2024-08-29T15:30:02.210672+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":459,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.228811+0000","last_change":"2024-08-29T15:08:58.888490+0000","last_active":"2024-08-29T15:30:02.228811+0000","last_peered":"2024-08-29T15:30:02.228811+0000","last_clean":"2024-08-29T15:30:02.228811+0000","last_became_active":"2024-08-29T15:08:58.886329+0000","last_became_peered":"2024-08-29T15:08:58.886329+0000","last_unstale":"2024-08-29T15:30:02.228811+0000","last_undegraded":"2024-08-29T15:30:02.228811+0000","last_fullsized":"2024-08-29T15:30:02.228811+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:13:14.888354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"300'5494","reported_seq":5517,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:11.641323+0000","last_change":"2024-08-29T15:28:57.923922+0000","last_active":"2024-08-29T15:30:11.641323+0000","last_peered":"2024-08-29T15:30:11.641323+0000","last_clean":"2024-08-29T15:30:11.641323+0000","last_became_active":"2024-08-29T15:28:57.923593+0000","last_became_peered":"2024-08-29T15:28:57.923593+0000","last_unstale":"2024-08-29T15:30:11.641323+0000","last_undegraded":"2024-08-29T15:30:11.641323+0000","last_fullsized":"2024-08-29T15:30:11.641323+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":5494,"log_dups_size":0,"ondisk_log_size":5494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:48:59.077875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360054784,"num_objects":5494,"num_object_clones":0,"num_object_copies":16482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5494,"num_write_kb":351616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":458,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222866+0000","last_change":"2024-08-29T15:07:52.140268+0000","last_active":"2024-08-29T15:30:02.222866+0000","last_peered":"2024-08-29T15:30:02.222866+0000","last_clean":"2024-08-29T15:30:02.222866+0000","last_became_active":"2024-08-29T15:07:52.139905+0000","last_became_peered":"2024-08-29T15:07:52.139905+0000","last_unstale":"2024-08-29T15:30:02.222866+0000","last_undegraded":"2024-08-29T15:30:02.222866+0000","last_fullsized":"2024-08-29T15:30:02.222866+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:26.921237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":154,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:30:00.028381+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:30:00.028381+0000","last_peered":"2024-08-29T15:30:00.028381+0000","last_clean":"2024-08-29T15:30:00.028381+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:30:00.028381+0000","last_undegraded":"2024-08-29T15:30:00.028381+0000","last_fullsized":"2024-08-29T15:30:00.028381+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.083776+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:29:38.083776+0000","last_peered":"2024-08-29T15:29:38.083776+0000","last_clean":"2024-08-29T15:29:38.083776+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:29:38.083776+0000","last_undegraded":"2024-08-29T15:29:38.083776+0000","last_fullsized":"2024-08-29T15:29:38.083776+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084078+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:29:38.084078+0000","last_peered":"2024-08-29T15:29:38.084078+0000","last_clean":"2024-08-29T15:29:38.084078+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:29:38.084078+0000","last_undegraded":"2024-08-29T15:29:38.084078+0000","last_fullsized":"2024-08-29T15:29:38.084078+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":91,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.084394+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:29:38.084394+0000","last_peered":"2024-08-29T15:29:38.084394+0000","last_clean":"2024-08-29T15:29:38.084394+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:29:38.084394+0000","last_undegraded":"2024-08-29T15:29:38.084394+0000","last_fullsized":"2024-08-29T15:29:38.084394+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":466,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.210538+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:30:02.210538+0000","last_peered":"2024-08-29T15:30:02.210538+0000","last_clean":"2024-08-29T15:30:02.210538+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:30:02.210538+0000","last_undegraded":"2024-08-29T15:30:02.210538+0000","last_fullsized":"2024-08-29T15:30:02.210538+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":224,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.220679+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:30:02.220679+0000","last_peered":"2024-08-29T15:30:02.220679+0000","last_clean":"2024-08-29T15:30:02.220679+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:30:02.220679+0000","last_undegraded":"2024-08-29T15:30:02.220679+0000","last_fullsized":"2024-08-29T15:30:02.220679+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":170,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:41.381052+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:29:41.381052+0000","last_peered":"2024-08-29T15:29:41.381052+0000","last_clean":"2024-08-29T15:29:41.381052+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:29:41.381052+0000","last_undegraded":"2024-08-29T15:29:41.381052+0000","last_fullsized":"2024-08-29T15:29:41.381052+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":425,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.221273+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:30:02.221273+0000","last_peered":"2024-08-29T15:30:02.221273+0000","last_clean":"2024-08-29T15:30:02.221273+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:30:02.221273+0000","last_undegraded":"2024-08-29T15:30:02.221273+0000","last_fullsized":"2024-08-29T15:30:02.221273+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":479,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222656+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:30:02.222656+0000","last_peered":"2024-08-29T15:30:02.222656+0000","last_clean":"2024-08-29T15:30:02.222656+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:30:02.222656+0000","last_undegraded":"2024-08-29T15:30:02.222656+0000","last_fullsized":"2024-08-29T15:30:02.222656+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":477,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.228720+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:30:02.228720+0000","last_peered":"2024-08-29T15:30:02.228720+0000","last_clean":"2024-08-29T15:30:02.228720+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:30:02.228720+0000","last_undegraded":"2024-08-29T15:30:02.228720+0000","last_fullsized":"2024-08-29T15:30:02.228720+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":142,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.082383+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:29:38.082383+0000","last_peered":"2024-08-29T15:29:38.082383+0000","last_clean":"2024-08-29T15:29:38.082383+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:29:38.082383+0000","last_undegraded":"2024-08-29T15:29:38.082383+0000","last_fullsized":"2024-08-29T15:29:38.082383+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":334,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.221170+0000","last_change":"2024-08-29T15:18:35.014315+0000","last_active":"2024-08-29T15:30:02.221170+0000","last_peered":"2024-08-29T15:30:02.221170+0000","last_clean":"2024-08-29T15:30:02.221170+0000","last_became_active":"2024-08-29T15:18:35.013309+0000","last_became_peered":"2024-08-29T15:18:35.013309+0000","last_unstale":"2024-08-29T15:30:02.221170+0000","last_undegraded":"2024-08-29T15:30:02.221170+0000","last_fullsized":"2024-08-29T15:30:02.221170+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:31:20.108177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":455,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.223052+0000","last_change":"2024-08-29T15:08:58.889319+0000","last_active":"2024-08-29T15:30:02.223052+0000","last_peered":"2024-08-29T15:30:02.223052+0000","last_clean":"2024-08-29T15:30:02.223052+0000","last_became_active":"2024-08-29T15:08:58.889006+0000","last_became_peered":"2024-08-29T15:08:58.889006+0000","last_unstale":"2024-08-29T15:30:02.223052+0000","last_undegraded":"2024-08-29T15:30:02.223052+0000","last_fullsized":"2024-08-29T15:30:02.223052+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:56.672976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":458,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.219003+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:30:02.219003+0000","last_peered":"2024-08-29T15:30:02.219003+0000","last_clean":"2024-08-29T15:30:02.219003+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:30:02.219003+0000","last_undegraded":"2024-08-29T15:30:02.219003+0000","last_fullsized":"2024-08-29T15:30:02.219003+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":461,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.220519+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:30:02.220519+0000","last_peered":"2024-08-29T15:30:02.220519+0000","last_clean":"2024-08-29T15:30:02.220519+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:30:02.220519+0000","last_undegraded":"2024-08-29T15:30:02.220519+0000","last_fullsized":"2024-08-29T15:30:02.220519+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:49.896470+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:29:49.896470+0000","last_peered":"2024-08-29T15:29:49.896470+0000","last_clean":"2024-08-29T15:29:49.896470+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:29:49.896470+0000","last_undegraded":"2024-08-29T15:29:49.896470+0000","last_fullsized":"2024-08-29T15:29:49.896470+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":482,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.215605+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:30:02.215605+0000","last_peered":"2024-08-29T15:30:02.215605+0000","last_clean":"2024-08-29T15:30:02.215605+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:30:02.215605+0000","last_undegraded":"2024-08-29T15:30:02.215605+0000","last_fullsized":"2024-08-29T15:30:02.215605+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":446,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222028+0000","last_change":"2024-08-29T15:07:20.818708+0000","last_active":"2024-08-29T15:30:02.222028+0000","last_peered":"2024-08-29T15:30:02.222028+0000","last_clean":"2024-08-29T15:30:02.222028+0000","last_became_active":"2024-08-29T15:07:20.818222+0000","last_became_peered":"2024-08-29T15:07:20.818222+0000","last_unstale":"2024-08-29T15:30:02.222028+0000","last_undegraded":"2024-08-29T15:30:02.222028+0000","last_fullsized":"2024-08-29T15:30:02.222028+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:27.520147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:49.896518+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:29:49.896518+0000","last_peered":"2024-08-29T15:29:49.896518+0000","last_clean":"2024-08-29T15:29:49.896518+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:29:49.896518+0000","last_undegraded":"2024-08-29T15:29:49.896518+0000","last_fullsized":"2024-08-29T15:29:49.896518+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":200,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.204497+0000","last_change":"2024-08-29T15:28:35.519733+0000","last_active":"2024-08-29T15:30:02.204497+0000","last_peered":"2024-08-29T15:30:02.204497+0000","last_clean":"2024-08-29T15:30:02.204497+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:30:02.204497+0000","last_undegraded":"2024-08-29T15:30:02.204497+0000","last_fullsized":"2024-08-29T15:30:02.204497+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:49.896501+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:29:49.896501+0000","last_peered":"2024-08-29T15:29:49.896501+0000","last_clean":"2024-08-29T15:29:49.896501+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:29:49.896501+0000","last_undegraded":"2024-08-29T15:29:49.896501+0000","last_fullsized":"2024-08-29T15:29:49.896501+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":200,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.204371+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:30:02.204371+0000","last_peered":"2024-08-29T15:30:02.204371+0000","last_clean":"2024-08-29T15:30:02.204371+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:30:02.204371+0000","last_undegraded":"2024-08-29T15:30:02.204371+0000","last_fullsized":"2024-08-29T15:30:02.204371+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":492,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.226775+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:30:02.226775+0000","last_peered":"2024-08-29T15:30:02.226775+0000","last_clean":"2024-08-29T15:30:02.226775+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:30:02.226775+0000","last_undegraded":"2024-08-29T15:30:02.226775+0000","last_fullsized":"2024-08-29T15:30:02.226775+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":494,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.223241+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:30:02.223241+0000","last_peered":"2024-08-29T15:30:02.223241+0000","last_clean":"2024-08-29T15:30:02.223241+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:30:02.223241+0000","last_undegraded":"2024-08-29T15:30:02.223241+0000","last_fullsized":"2024-08-29T15:30:02.223241+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-08-29T15:29:38.082761+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:29:38.082761+0000","last_peered":"2024-08-29T15:29:38.082761+0000","last_clean":"2024-08-29T15:29:38.082761+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:29:38.082761+0000","last_undegraded":"2024-08-29T15:29:38.082761+0000","last_fullsized":"2024-08-29T15:29:38.082761+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":502,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-08-29T15:30:02.222519+0000","last_change":"2024-08-29T15:18:33.996328+0000","last_active":"2024-08-29T15:30:02.222519+0000","last_peered":"2024-08-29T15:30:02.222519+0000","last_clean":"2024-08-29T15:30:02.222519+0000","last_became_active":"2024-08-29T15:18:33.995357+0000","last_became_peered":"2024-08-29T15:18:33.995357+0000","last_unstale":"2024-08-29T15:30:02.222519+0000","last_undegraded":"2024-08-29T15:30:02.222519+0000","last_fullsized":"2024-08-29T15:30:02.222519+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:15.545461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5770313728,"num_objects":88048,"num_object_clones":0,"num_object_copies":264144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88048,"num_write_kb":5635072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17385979904,"data_stored":17385979904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88048,"ondisk_log_size":88048,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":6528,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234426,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179216,"kb_used_data":1765000,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":91565936,"statfs":{"total":95995035648,"available":93763518464,"internally_reserved":0,"allocated":1807360000,"data_stored":1805575182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":424147403},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496057,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2041992,"kb_used_data":1772160,"kb_used_omap":1,"kb_used_meta":269822,"kb_avail":91703160,"statfs":{"total":95995035648,"available":93904035840,"internally_reserved":0,"allocated":1814691840,"data_stored":1812914884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":276298183},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426644,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1551544,"kb_used_data":1428336,"kb_used_omap":8,"kb_used_meta":123191,"kb_avail":92193608,"statfs":{"total":95995035648,"available":94406254592,"internally_reserved":0,"allocated":1462616064,"data_stored":1460850646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":126148490},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019336,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2139784,"kb_used_data":1747712,"kb_used_omap":1,"kb_used_meta":392062,"kb_avail":91605368,"statfs":{"total":95995035648,"available":93803896832,"internally_reserved":0,"allocated":1789657088,"data_stored":1787879645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":401471942},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435521,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3624824,"kb_used_data":3210032,"kb_used_omap":4,"kb_used_meta":414779,"kb_avail":90120328,"statfs":{"total":95995035648,"available":92283215872,"internally_reserved":0,"allocated":3287072768,"data_stored":3285307182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":424733961},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903373,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400884,"kb_used_data":2156716,"kb_used_omap":10,"kb_used_meta":244149,"kb_avail":91344268,"statfs":{"total":95995035648,"available":93536530432,"internally_reserved":0,"allocated":2208477184,"data_stored":2206715040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":250009104},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762574,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2675316,"kb_used_data":2340972,"kb_used_omap":7,"kb_used_meta":334328,"kb_avail":91069836,"statfs":{"total":95995035648,"available":93255512064,"internally_reserved":0,"allocated":2397155328,"data_stored":2395392622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":342352226},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926100,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3142268,"kb_used_data":2741300,"kb_used_omap":7,"kb_used_meta":400952,"kb_avail":90602884,"statfs":{"total":95995035648,"available":92777353216,"internally_reserved":0,"allocated":2807091200,"data_stored":2805321205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":410574991},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,3,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2828206080,"data_stored":2828206080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2162360320,"data_stored":2162360320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2218590208,"data_stored":2218590208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3308060672,"data_stored":3308060672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1796341760,"data_stored":1796341760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1461583872,"data_stored":1461583872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1810432000,"data_stored":1810432000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1800404992,"data_stored":1800404992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:30:17.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-29T15:30:17.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:17.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:17.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:17.861 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:17.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:17.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:17.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:18.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:18.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:18.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:18.147 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:18.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:18.278 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:18 smithi027 ceph-mon[93921]: pgmap v1078: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:30:18.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:18.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:18.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.426 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:18.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:18.491 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 66 16 96022 96006 90.8949 60.875 0.0131781 0.0109966 2024-08-29T15:30:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:18 smithi171 ceph-mon[90031]: pgmap v1078: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:30:18.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:18 smithi027 ceph-mon[98970]: pgmap v1078: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:30:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:18.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:18.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:30:18.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:18.698 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:18.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:18.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:18.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:18.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:18.840 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:18.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:18.970 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:18.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:19.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:19.117 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:19.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:19.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:19.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:19.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:19.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:30:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:30:19] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.51.0" 2024-08-29T15:30:19.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:19.375 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:19.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:19.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:19.477 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:19.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 67 16 97023 97007 90.472 62.5625 0.0140995 0.0110476 2024-08-29T15:30:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:19.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:19.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:19.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:19.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:19.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:19.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:19.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:19.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:19.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:19.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:19.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:20.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:20.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:30:20.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:20.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:20.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:20.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:20.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:20.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:20 smithi027 ceph-mon[93921]: pgmap v1079: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:30:20.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:20 smithi027 ceph-mon[98970]: pgmap v1079: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:30:20.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:20.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.474 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:20.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:20.491 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 68 16 98007 97991 90.0459 61.5 0.00885125 0.0111001 2024-08-29T15:30:20.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:20.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:20.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:20.700 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:20.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:20 smithi171 ceph-mon[90031]: pgmap v1079: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:30:20.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:20.871 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:20.973 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:20.998 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:20.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:21.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:21.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:21.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:21.230 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:21.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:21.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:21.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:21.371 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:21.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:21.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:21.409 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:21.437 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:30:21.437 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:30:21.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 69 16 98943 98927 89.5887 58.5 0.00927635 0.0111556 2024-08-29T15:30:21.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:21.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:21.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:21.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:21.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:21.682 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:21.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:21.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:21.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:21.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:21.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:21.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:21.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:21.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:22.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:22.081 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:22.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:22.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:22.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:22.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:22.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:22.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:22.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:22.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:22.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:22.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:22.492 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 70 16 99905 99889 89.1677 60.125 0.0196936 0.0112099 2024-08-29T15:30:22.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:22.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:22.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:22 smithi027 ceph-mon[93921]: pgmap v1080: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:30:22.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:22 smithi027 ceph-mon[98970]: pgmap v1080: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:30:22.597 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:22.623 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:22.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:22.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:22.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:22.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:22.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:22.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:22 smithi171 ceph-mon[90031]: pgmap v1080: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:30:22.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:22.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:30:22.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:22.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:22.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:23.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:23.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:23.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:23.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:23.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:23.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:23.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:23.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:23.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:23.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:23.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:23.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:23.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:23.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:23.491 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 71 16 100833 100817 88.7287 58 0.0102351 0.0112652 2024-08-29T15:30:23.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:23.626 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:23.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:23.635 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:23.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:23.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:23.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:23.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:23.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:23.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:23.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:23.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:24.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:24.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:24.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:24.181 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:24.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:24.292 INFO:teuthology.orchestra.run.smithi027.stderr:set pool 1 pgp_num to 1 2024-08-29T15:30:24.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:24.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:24.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:24.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:24.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:24.492 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 72 16 101730 101714 88.275 56.0625 0.014967 0.0113228 2024-08-29T15:30:24.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:24.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:24 smithi027 ceph-mon[98970]: pgmap v1081: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-08-29T15:30:24.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3352160427' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-29T15:30:24.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:24 smithi027 ceph-mon[93921]: pgmap v1081: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-08-29T15:30:24.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3352160427' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-29T15:30:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:24.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:24.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:24.712 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:24 smithi171 ceph-mon[90031]: pgmap v1081: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-08-29T15:30:24.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3352160427' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-29T15:30:24.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:24.853 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:24.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:24.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:24.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:24.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:25.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:25.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:25.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:25.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:25.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:25.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:25.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:25.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:25.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:25.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:25.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:25.492 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 73 16 102724 102708 87.9167 62.125 0.0233901 0.0113687 2024-08-29T15:30:25.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:25.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:25.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:25.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3352160427' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-29T15:30:25.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:25 smithi027 ceph-mon[93921]: osdmap e302: 8 total, 8 up, 8 in 2024-08-29T15:30:25.614 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3352160427' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-29T15:30:25.614 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:25 smithi027 ceph-mon[98970]: osdmap e302: 8 total, 8 up, 8 in 2024-08-29T15:30:25.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:25.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:25.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3352160427' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-29T15:30:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:25 smithi171 ceph-mon[90031]: osdmap e302: 8 total, 8 up, 8 in 2024-08-29T15:30:25.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:25.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:25.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:25.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:25.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:25.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:26.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:26.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:26.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:26.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:26.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:26.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:26.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:26.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:26.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:26.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:26.383 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:26.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:26.492 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 74 16 103681 103665 87.5364 59.8125 0.0149439 0.0114174 2024-08-29T15:30:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:26.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:26.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:26.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:26.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:26.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:26.677 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:26 smithi027 ceph-mon[93921]: pgmap v1083: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-08-29T15:30:26.677 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:26 smithi027 ceph-mon[98970]: pgmap v1083: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-08-29T15:30:26.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:26 smithi171 ceph-mon[90031]: pgmap v1083: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2024-08-29T15:30:26.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:26.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:26.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:26.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:26.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:26.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:27.041 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:27.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:27.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:27.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:27.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:27.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:27.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:27.346 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:30:27.346 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:30:27.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:27.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:27.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:27.388 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:30:27.388 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:30:27.430 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:30:27.430 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:30:27.462 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:27.478 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:30:27.479 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:30:27.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:27.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:27.492 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 75 16 104578 104562 87.1167 56.0625 0.00775973 0.0114734 2024-08-29T15:30:27.511 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:30:27.511 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:30:27.528 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:30:27.528 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:30:27.544 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:30:27.544 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:30:27.563 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:30:27.563 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:30:27.581 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2828241383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:27.581 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2828241383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:27.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:27.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:27.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:27.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:27.721 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:27.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:27.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:27.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2828241383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:27.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:27.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:27.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:27.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.014 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:28.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:28.115 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:28.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:28.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:28.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:28.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:28.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:28.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:28.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 76 16 105355 105339 86.6094 48.5625 0.00660744 0.0115406 2024-08-29T15:30:28.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:28.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:28.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:28.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:28.667 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:28 smithi027 ceph-mon[98970]: pgmap v1084: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-08-29T15:30:28.667 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:28 smithi027 ceph-mon[93921]: pgmap v1084: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-08-29T15:30:28.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:28.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:28 smithi171 ceph-mon[90031]: pgmap v1084: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-08-29T15:30:28.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:28.827 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:28.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:28.952 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:28.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:29.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:29.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:29.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:29.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:29.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:29.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:30:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:30:29] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-08-29T15:30:29.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:29.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:29.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:29.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:29.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 77 16 106187 106171 86.1599 52 0.0145474 0.0116014 2024-08-29T15:30:29.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:29.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:29.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:29.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:29.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:29.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:29.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:29.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:29.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:29.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:29.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:29.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:29.896 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:29.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:29.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:29.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:30.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:30.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:30.169 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:30.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:30.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:30.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:30.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:30.426 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:30.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:30.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 78 16 106946 106930 85.6636 47.4375 0.0172248 0.0116679 2024-08-29T15:30:30.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:30.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:30.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:30.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:30 smithi171 ceph-mon[90031]: pgmap v1085: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-08-29T15:30:30.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:30.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:30 smithi027 ceph-mon[98970]: pgmap v1085: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-08-29T15:30:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:30 smithi027 ceph-mon[93921]: pgmap v1085: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2024-08-29T15:30:30.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:30.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:30.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:30.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:30.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:31.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:31.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:31.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:31.192 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:31.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:31.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:31.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:31.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:31.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:31.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:31.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:31.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:31.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 79 16 107578 107562 85.0792 39.5 0.00778127 0.0117436 2024-08-29T15:30:31.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:31.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:31.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:31.595 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:31.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:31.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:31.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:31.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:31.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:31.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:31.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:31.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:32.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:32.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:32.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:32.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:32.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:32.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:32.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:32.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:32.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:32.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:32.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:32.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:32.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:32.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:30:32.492997+0000 min lat: 0.00124846 max lat: 0.409089 avg lat: 0.0117995 2024-08-29T15:30:32.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:30:32.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 80 16 108463 108447 84.7071 55.3125 0.0343437 0.0117995 2024-08-29T15:30:32.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:32.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:32 smithi027 ceph-mon[93921]: pgmap v1086: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-29T15:30:32.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:32 smithi027 ceph-mon[98970]: pgmap v1086: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-29T15:30:32.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:32.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:32.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:32.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:32.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:32.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:32 smithi171 ceph-mon[90031]: pgmap v1086: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-29T15:30:32.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:32.880 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:32.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:32.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:32.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:33.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:33.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:33.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:33.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:33.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:33.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:33.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:33.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:33.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:33.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:33.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:33.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 81 16 109217 109201 84.2432 47.125 0.00654692 0.0118626 2024-08-29T15:30:33.549 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:33.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:33.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:33.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:33.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:33.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:33.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:33.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:33.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:33.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:33.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:34.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:34.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:34.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:34.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:34.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:34.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:34.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:34.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:34.287 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:30:34.287 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:30:34.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:30:34.287 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:30:34.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.287 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:241256 6.1 6:88fa083a:::benchmark_data_smithi027_169445_object115829:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.954534+0000", 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 139.32719117400001, 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.043345168000000003, 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 241256 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954534+0000", 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954534+0000", 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954536+0000", 2024-08-29T15:30:34.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9589999999999998e-06 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954544+0000", 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6920000000000002e-06 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954564+0000", 2024-08-29T15:30:34.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9848999999999999e-05 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954573+0000", 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1959999999999997e-06 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954641+0000", 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8052000000000005e-05 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.954860+0000", 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021875799999999999 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.955021+0000", 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161263 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.973289+0000", 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018267921999999999 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.973504+0000", 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000215782 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.979893+0000", 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063884989999999997 2024-08-29T15:30:34.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.996324+0000", 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016430953000000002 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.996389+0000", 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4713000000000002e-05 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.997879+0000", 2024-08-29T15:30:34.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014905299999999999 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:241259 6.1 6:8ff3c1d3:::benchmark_data_smithi027_169445_object115832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.956404+0000", 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 139.32532077600001, 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.043690310000000003, 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.296 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 241259 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956404+0000", 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956404+0000", 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956406+0000", 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.406e-06 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956411+0000", 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7589999999999997e-06 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956424+0000", 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3879e-05 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956432+0000", 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9710000000000004e-06 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.958573+0000", 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021406020000000001 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.958731+0000", 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015795300000000001 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.958903+0000", 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017155299999999999 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.975333+0000", 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016430700999999999 2024-08-29T15:30:34.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.975389+0000", 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6107000000000003e-05 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.979899+0000", 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045101029999999997 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.997948+0000", 2024-08-29T15:30:34.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018048297000000001 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.998042+0000", 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4704999999999995e-05 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:15.000095+0000", 2024-08-29T15:30:34.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020522740000000002 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:241260 6.1 6:8b29504b:::benchmark_data_smithi027_169445_object115833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.956595+0000", 2024-08-29T15:30:34.303 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 139.32513023000001, 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.045375179000000002, 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 241260 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956595+0000", 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956595+0000", 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956596+0000", 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8999999999999995e-07 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956601+0000", 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0069999999999997e-06 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956611+0000", 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0618000000000001e-05 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.956619+0000", 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4730000000000003e-06 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.962850+0000", 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006230752 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.963038+0000", 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018873699999999999 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.963207+0000", 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016905199999999999 2024-08-29T15:30:34.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.975442+0000", 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012234355000000001 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.975492+0000", 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9762000000000002e-05 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.979905+0000", 2024-08-29T15:30:34.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044135839999999999 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:15.000647+0000", 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020741512 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:15.000732+0000", 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4884999999999998e-05 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:15.001970+0000", 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001238552 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.44912.0:241264 6.1 6:88ef1080:::benchmark_data_smithi027_169445_object115837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:28:14.957329+0000", 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 139.32439584700001, 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046079504, 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.310 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.44912", 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 241264 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.957329+0000", 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.957329+0000", 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.957330+0000", 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6700000000000003e-07 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.957333+0000", 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1999999999999999e-06 2024-08-29T15:30:34.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.957343+0000", 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0047999999999999e-05 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.957351+0000", 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0749999999999998e-06 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.967047+0000", 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096959060000000007 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.967458+0000", 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041038100000000003 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6,7", 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.968766+0000", 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013086180000000001 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.975559+0000", 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067925219999999996 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.975608+0000", 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9419000000000002e-05 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:14.979911+0000", 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043030660000000004 2024-08-29T15:30:34.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:15.002034+0000", 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022122810999999999 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:15.002110+0000", 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6429999999999998e-05 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:28:15.003409+0000", 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001298361 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:48470 7.b 7:dc64407d:::benchmark_data_smithi027_271924_object48469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e298)", 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:29:36.609909+0000", 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 57.671815979000002, 2024-08-29T15:30:34.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.048603259000000003, 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 48470 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.318 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.609909+0000", 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.609909+0000", 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.609911+0000", 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5680000000000001e-06 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.609936+0000", 2024-08-29T15:30:34.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.514e-05 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.609950+0000", 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3854e-05 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.609959+0000", 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8459999999999997e-06 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.610020+0000", 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1740000000000002e-05 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.610197+0000", 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017624700000000001 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.610521+0000", 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00032409799999999999 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.612642+0000", 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021213859999999998 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.612789+0000", 2024-08-29T15:30:34.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000146952 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.658352+0000", 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.045563379000000001 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.658461+0000", 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109028 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:29:36.658512+0000", 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1020999999999998e-05 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:84178 7.b 7:d50fb677:::benchmark_data_smithi027_271924_object84177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:06.518934+0000", 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 27.762791330999999, 2024-08-29T15:30:34.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.094624502999999999, 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 84178 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.518934+0000", 2024-08-29T15:30:34.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.518934+0000", 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.518936+0000", 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.703e-06 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.518959+0000", 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3305000000000001e-05 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.518977+0000", 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8071999999999999e-05 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.518985+0000", 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2879999999999993e-06 2024-08-29T15:30:34.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519053+0000", 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8202999999999997e-05 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519189+0000", 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013584599999999999 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519326+0000", 2024-08-29T15:30:34.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013721600000000001 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.525345+0000", 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060184000000000001 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.525377+0000", 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2292000000000002e-05 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613462+0000", 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.088085187999999995 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613526+0000", 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3597999999999997e-05 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613558+0000", 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2391999999999998e-05 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.331 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:84181 7.b 7:dce3e7d2:::benchmark_data_smithi027_271924_object84180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:06.519549+0000", 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 27.762176681, 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.094165445, 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 84181 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.332 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519549+0000", 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519549+0000", 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519550+0000", 2024-08-29T15:30:34.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.077e-06 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519571+0000", 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.122e-05 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519583+0000", 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1939999999999999e-05 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.519590+0000", 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.187e-06 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.522297+0000", 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027069699999999999 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.522446+0000", 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014864699999999999 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.522589+0000", 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014359600000000001 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.525370+0000", 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027806160000000001 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.525485+0000", 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011507599999999999 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613612+0000", 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.088127010000000006 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613668+0000", 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5739000000000003e-05 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.337 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613714+0000", 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6366999999999998e-05 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:84191 7.b 7:d7b32b4f:::benchmark_data_smithi027_271924_object84190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:06.534615+0000", 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 27.747110393, 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.079424860999999999, 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.338 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 84191 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534615+0000", 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534615+0000", 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534616+0000", 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.271e-06 2024-08-29T15:30:34.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534637+0000", 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0934e-05 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534647+0000", 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0254999999999999e-05 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534653+0000", 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8050000000000003e-06 2024-08-29T15:30:34.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534714+0000", 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0438e-05 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.534879+0000", 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000165174 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.535041+0000", 2024-08-29T15:30:34.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016279200000000001 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.537189+0000", 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021470790000000001 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.537468+0000", 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027922500000000002 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613779+0000", 2024-08-29T15:30:34.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.076311595999999995 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.613993+0000", 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021350899999999999 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:06.614040+0000", 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6783000000000003e-05 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:105265 7.b 7:d5489c2d:::benchmark_data_smithi027_271924_object105264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:28.346908+0000", 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 5.9348172029999997, 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.076100212, 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 105265 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.346908+0000", 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.346908+0000", 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.346909+0000", 2024-08-29T15:30:34.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4559999999999999e-06 2024-08-29T15:30:34.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.346930+0000", 2024-08-29T15:30:34.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0438999999999999e-05 2024-08-29T15:30:34.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.346939+0000", 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0210000000000005e-06 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.346945+0000", 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7599999999999999e-06 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.347505+0000", 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00056047900000000001 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.347658+0000", 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015328799999999999 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.347801+0000", 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000142385 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.351949+0000", 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004147881 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.351968+0000", 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8921999999999999e-05 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.422104+0000", 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.070136588999999999 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.422860+0000", 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00075605599999999996 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.423008+0000", 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014793599999999999 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:105266 7.d e302/294 7:b0cb28f4:::benchmark_data_smithi027_271924_object105265:head v 302'6536, mlcod=302'6531)", 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:28.353412+0000", 2024-08-29T15:30:34.351 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 5.9283132009999999, 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.072021327999999996, 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353412+0000", 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353412+0000", 2024-08-29T15:30:34.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353413+0000", 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8400000000000004e-07 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353587+0000", 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017408300000000001 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353595+0000", 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1380000000000007e-06 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353600+0000", 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2839999999999997e-06 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353641+0000", 2024-08-29T15:30:34.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1165999999999999e-05 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353660+0000", 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8192000000000001e-05 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.425318+0000", 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.071658688999999998 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.425433+0000", 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115092 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:105271 7.0 e302/294 7:0ff7bc74:::benchmark_data_smithi027_271924_object105270:head v 302'6527, mlcod=302'6524)", 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:28.353504+0000", 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 5.9282210529999997, 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.068987989999999999, 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.356 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353504+0000", 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353504+0000", 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353505+0000", 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5899999999999995e-07 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353606+0000", 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100912 2024-08-29T15:30:34.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353654+0000", 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7848e-05 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353659+0000", 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.006e-06 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353702+0000", 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3603000000000002e-05 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.353721+0000", 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9057000000000001e-05 2024-08-29T15:30:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.422295+0000", 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.068574080999999995 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.422492+0000", 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019672400000000001 2024-08-29T15:30:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.36273.0:105265 7.b e302/294 ondisk, result = 0)", 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:28.356047+0000", 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 5.9256785519999999, 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.066969493000000005, 2024-08-29T15:30:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356047+0000", 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356047+0000", 2024-08-29T15:30:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356047+0000", 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7000000000000004e-07 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356051+0000", 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8399999999999997e-06 2024-08-29T15:30:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.417629+0000", 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.061578331 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.417637+0000", 2024-08-29T15:30:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3699999999999997e-06 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.422062+0000", 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044252019999999996 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.422084+0000", 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2011999999999999e-05 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.423016+0000", 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00093206799999999998 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:105275 7.2 e302/294 7:45c59dd8:::benchmark_data_smithi027_271924_object105274:head v 302'6489, mlcod=302'6487)", 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:28.356786+0000", 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 5.9249391899999999, 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.066275054, 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356786+0000", 2024-08-29T15:30:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356786+0000", 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356787+0000", 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9299999999999997e-07 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.356805+0000", 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8549000000000001e-05 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.417569+0000", 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.060763389000000001 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.417576+0000", 2024-08-29T15:30:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1829999999999997e-06 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.418933+0000", 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013567220000000001 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.418982+0000", 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9112e-05 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.423024+0000", 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040421509999999999 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:28.423061+0000", 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7255000000000002e-05 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:106261 7.9 e302/294 7:9bd2556c:::benchmark_data_smithi027_271924_object106260:head v 302'6510, mlcod=302'6508)", 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.574680+0000", 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.7070451100000001, 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16662368699999999, 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574680+0000", 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574680+0000", 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574681+0000", 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.02e-06 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574706+0000", 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4428999999999999e-05 2024-08-29T15:30:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574719+0000", 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3730999999999999e-05 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574727+0000", 2024-08-29T15:30:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5689999999999999e-06 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574802+0000", 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5628000000000002e-05 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574827+0000", 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4507000000000001e-05 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741093+0000", 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.166265993 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741304+0000", 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021081 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:106260 7.0 e302/294 7:0f42b86f:::benchmark_data_smithi027_271924_object106259:head v 302'6580, mlcod=302'6575)", 2024-08-29T15:30:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.575142+0000", 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.7065832519999997, 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.163563086, 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575142+0000", 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575142+0000", 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575143+0000", 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5300000000000003e-07 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575174+0000", 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1155000000000002e-05 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575187+0000", 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3251000000000001e-05 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575195+0000", 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6539999999999999e-06 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575256+0000", 2024-08-29T15:30:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0912999999999998e-05 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575274+0000", 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8063999999999998e-05 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.582407+0000", 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071335180000000002 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.738705+0000", 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.156297678 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106263 7.b 7:d36e8c4b:::benchmark_data_smithi027_271924_object106262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.577672+0000", 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.704052838, 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.166003496, 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106263 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577672+0000", 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577672+0000", 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577674+0000", 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7069999999999999e-06 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577698+0000", 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4281e-05 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577712+0000", 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.383e-05 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577721+0000", 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4120000000000001e-06 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577785+0000", 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4077000000000003e-05 2024-08-29T15:30:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577900+0000", 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011547199999999999 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,6", 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577989+0000", 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9209999999999995e-05 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.742132+0000", 2024-08-29T15:30:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16414261899999999 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.742179+0000", 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7074999999999999e-05 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743410+0000", 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012312709999999999 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743561+0000", 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015049400000000001 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743676+0000", 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115048 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:106262 7.9 e302/294 7:939d7ac7:::benchmark_data_smithi027_271924_object106261:head v 302'6511, mlcod=302'6508)", 2024-08-29T15:30:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.578586+0000", 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.7031391899999999, 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.162894276, 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578586+0000", 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578586+0000", 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578587+0000", 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8599999999999996e-07 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578609+0000", 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1710999999999999e-05 2024-08-29T15:30:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578621+0000", 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2508e-05 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578627+0000", 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9390000000000002e-06 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578765+0000", 2024-08-29T15:30:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137787 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578783+0000", 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8422000000000001e-05 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741310+0000", 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.162526433 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741480+0000", 2024-08-29T15:30:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017049 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:106265 7.2 e302/294 7:4db730ac:::benchmark_data_smithi027_271924_object106264:head v 302'6553, mlcod=302'6548)", 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.580504+0000", 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.7012209540000001, 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16673094599999999, 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580504+0000", 2024-08-29T15:30:34.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580504+0000", 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580505+0000", 2024-08-29T15:30:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0699999999999996e-07 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580529+0000", 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4185999999999998e-05 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580541+0000", 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1814000000000001e-05 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580549+0000", 2024-08-29T15:30:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3980000000000002e-06 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580605+0000", 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6576999999999997e-05 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580625+0000", 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.033e-05 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.746974+0000", 2024-08-29T15:30:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.166348212 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.747235+0000", 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026152199999999998 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:106267 7.d e302/294 7:b9baa89f:::benchmark_data_smithi027_271924_object106266:head v 302'6601, mlcod=302'6599)", 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.581226+0000", 2024-08-29T15:30:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.7004987030000001, 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.165466425, 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581226+0000", 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581226+0000", 2024-08-29T15:30:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581227+0000", 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4099999999999998e-07 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581273+0000", 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5646999999999999e-05 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.738713+0000", 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15743998200000001 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.738722+0000", 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6959999999999994e-06 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.738835+0000", 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113115 2024-08-29T15:30:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.738858+0000", 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3811999999999998e-05 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.746599+0000", 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007740757 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.746693+0000", 2024-08-29T15:30:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3675e-05 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.36273.0:106263 7.b e302/294 ondisk, result = 0)", 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.581583+0000", 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.7001424949999997, 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16210090899999999, 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:30:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581583+0000", 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581583+0000", 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581583+0000", 2024-08-29T15:30:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0100000000000005e-07 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581587+0000", 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.985e-06 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.739244+0000", 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.157656828 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:30:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.739251+0000", 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1010000000000003e-06 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743397+0000", 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041453219999999999 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743404+0000", 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9419999999999995e-06 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:30:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743684+0000", 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027912999999999999 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:30:34.409 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:30:34.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:34.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 82 16 109991 109975 83.8057 48.375 0.00499821 0.0119245 2024-08-29T15:30:34.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:34.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:34.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:30:34.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:34.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:34 smithi027 ceph-mon[98970]: pgmap v1087: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-08-29T15:30:34.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3599577270' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:30:34.695 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:34 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:30:34.695 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:34 smithi027 ceph-mon[93921]: pgmap v1087: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-08-29T15:30:34.695 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3599577270' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:30:34.695 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:34 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:30:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.703 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:34.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:34 smithi171 ceph-mon[90031]: pgmap v1087: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-08-29T15:30:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3599577270' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:30:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:34 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:30:34.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:34.827 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:34.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:34.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:34.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:35.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:35.076 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:35.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:35.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:35.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:35.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:35.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:35.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:35.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:35.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:35.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:35.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:35.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 83 16 110507 110491 83.1846 32.25 0.00843455 0.0120162 2024-08-29T15:30:35.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:35.641 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:35.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:35.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:35 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:35 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:30:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:35 smithi171 ceph-mon[90031]: osdmap e303: 8 total, 8 up, 8 in 2024-08-29T15:30:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:35 smithi171 ceph-mon[90031]: pgmap v1089: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2024-08-29T15:30:35.784 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:35.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:30:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[98970]: osdmap e303: 8 total, 8 up, 8 in 2024-08-29T15:30:35.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[98970]: pgmap v1089: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2024-08-29T15:30:35.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:30:35.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:30:35.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[93921]: osdmap e303: 8 total, 8 up, 8 in 2024-08-29T15:30:35.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:35 smithi027 ceph-mon[93921]: pgmap v1089: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2024-08-29T15:30:35.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:35.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:35.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:36.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:36.039 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:36.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:36.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:36.180 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:36.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:36.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:36.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:36.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:36.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:36.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:36.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:36.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:36.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 84 16 111443 111427 82.8907 58.5 0.0196406 0.0120588 2024-08-29T15:30:36.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:36.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:30:36.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:36.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:36.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:36.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:36.735 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:36.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:36.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:36.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:36.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:36.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:36.942 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:30:36.942 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:30:36.942 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-29T15:30:36.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd test-reweight-by-utilization 110 1.0 2024-08-29T15:30:36.994 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:37.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:37.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:37.086 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:37.155 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:37.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:37.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:37.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:37.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:37.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:37.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:37.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:37.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:37.452 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:37.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 85 16 112219 112203 82.486 48.5 0.0181064 0.0121167 2024-08-29T15:30:37.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:37.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:37.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:37.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:37.637 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:37.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:37.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:37.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:37.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:37.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:37.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:37 smithi027 ceph-mon[98970]: pgmap v1090: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s 2024-08-29T15:30:37.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:37 smithi027 ceph-mon[93921]: pgmap v1090: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s 2024-08-29T15:30:37.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:37.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:37.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:38.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:38.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:38.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:38.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:38.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:38.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:38.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:38.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:37 smithi171 ceph-mon[90031]: pgmap v1090: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 730 op/s 2024-08-29T15:30:38.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:38.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:38.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:38.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:38.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:38.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:38.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 86 16 113080 113064 82.1526 53.8125 0.00505752 0.0121646 2024-08-29T15:30:38.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:38.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:38.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:38.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:38.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:38.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:38.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:38.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:38.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:38.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:38.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:38.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:39.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:30:39.101 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:39.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:39.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:39.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:39.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:39.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:30:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:30:39] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.51.0" 2024-08-29T15:30:39.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:39.435 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:39.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 87 16 113973 113957 81.8498 55.8125 0.012422 0.0122117 2024-08-29T15:30:39.537 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:39.562 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:39.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:39.664 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:39.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:39.727 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:39.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:39.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.848 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:39.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:39.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:39.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:39.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:39 smithi027 ceph-mon[98970]: pgmap v1091: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2024-08-29T15:30:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:39 smithi027 ceph-mon[93921]: pgmap v1091: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2024-08-29T15:30:40.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:40.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:40.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:40.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:40.229 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:40.233 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:40.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:40.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:39 smithi171 ceph-mon[90031]: pgmap v1091: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2024-08-29T15:30:40.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:40.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:40.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:40.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:40.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 88 16 114840 114824 81.5355 54.1875 0.0127279 0.0122595 2024-08-29T15:30:40.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:40.533 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:40.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:40.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:40.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:40.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:40.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:40.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:40.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:40.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:40.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:40.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:40.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:41.061 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:41.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:41.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:41.092 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:30:41.092 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:30:41.139 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:30:41.139 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:30:41.173 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:30:41.173 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:30:41.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:41.209 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:30:41.210 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:30:41.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:41.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:41.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:41.248 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:30:41.248 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:30:41.285 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:30:41.286 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:30:41.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:41.328 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:30:41.328 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:30:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:41.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:41.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:41.360 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:30:41.360 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:30:41.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:41.490 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:41.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:41.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:41.493 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 89 16 115637 115621 81.179 49.8125 0.0150986 0.0123131 2024-08-29T15:30:41.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:41.636 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:41.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:41.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:41.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:41.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:41.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:41.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:41.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:41.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:42.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:41 smithi027 ceph-mon[98970]: pgmap v1092: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s 2024-08-29T15:30:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/999247705' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:41 smithi027 ceph-mon[93921]: pgmap v1092: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s 2024-08-29T15:30:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/999247705' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:42.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:42.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:42.204 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:41 smithi171 ceph-mon[90031]: pgmap v1092: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 775 op/s 2024-08-29T15:30:42.204 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/999247705' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:42.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:42.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:42.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:42.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:42.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total time run: 90.0061 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total writes made: 116338 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Write size: 65536 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object size: 65536 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Bandwidth (MB/sec): 80.7848 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Bandwidth: 34.9124 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max bandwidth (MB/sec): 214.938 2024-08-29T15:30:42.494 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min bandwidth (MB/sec): 32.25 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average IOPS: 1292 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev IOPS: 558.599 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max IOPS: 3439 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min IOPS: 516 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average Latency(s): 0.0123741 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Latency(s): 0.0134766 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max latency(s): 0.409089 2024-08-29T15:30:42.495 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min latency(s): 0.00124846 2024-08-29T15:30:42.500 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Cleaning up (deleting benchmark objects) 2024-08-29T15:30:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:30:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=cleanup t=2024-08-29T15:30:42.202821909Z level=info msg="Completed cleanup jobs" duration=4.937744ms 2024-08-29T15:30:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:30:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugins.update.checker t=2024-08-29T15:30:42.353144953Z level=info msg="Update check succeeded" duration=64.099606ms 2024-08-29T15:30:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:30:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana.update.checker t=2024-08-29T15:30:42.355059753Z level=info msg="Update check succeeded" duration=71.026968ms 2024-08-29T15:30:42.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:42.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:42.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:42.673 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:42.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:42.807 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:42.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:42.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:42.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:42.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:43.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:43.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:43.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:43.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:43.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:43.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:43.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:43.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:43.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:43.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:43.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:43.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:43.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:43.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:43.520 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:43.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:43.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:43.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:43.659 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:43.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:43.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:43.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:43.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:43.906 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:43.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:43.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:43.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:44.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:44.057 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:44.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:44.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:43 smithi027 ceph-mon[98970]: pgmap v1093: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2024-08-29T15:30:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:43 smithi027 ceph-mon[93921]: pgmap v1093: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2024-08-29T15:30:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:44.159 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:44.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:44.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:44.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:43 smithi171 ceph-mon[90031]: pgmap v1093: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2024-08-29T15:30:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:30:44.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:44.365 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:44.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:44.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:44.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:44.511 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:44.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:44.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:44.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:44.638 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:44.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:44.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:44.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:44.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:44.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:44.882 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:44.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:44.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:45.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:45.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:45.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:45.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:45.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:45.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:45.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:45.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:45.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:45.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:45.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:45.511 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:45.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:45.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:45.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:45.637 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:45.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:45.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:45.739 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:45.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:45.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:45.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:45.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:45.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:45.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:46.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:46.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:46.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:45 smithi027 ceph-mon[93921]: pgmap v1094: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 774 op/s 2024-08-29T15:30:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:45 smithi027 ceph-mon[98970]: pgmap v1094: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 774 op/s 2024-08-29T15:30:46.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:30:46.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:46.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:46.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:46.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:45 smithi171 ceph-mon[90031]: pgmap v1094: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 774 op/s 2024-08-29T15:30:46.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:46.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:46.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:46.421 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:30:46.421 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:30:46.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:46.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:46.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:46.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:46.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:46.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:46.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:46.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.845 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:46.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:46.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:46.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:46.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:46.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:47.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:47.115 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:47.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:47.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:47.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:47.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:47.266 INFO:teuthology.orchestra.run.smithi027.stdout:moved 22 / 267 (8.2397%) 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:avg 33.375 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 3.99805 -> 6.89089 (expected baseline 5.40399) 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.5 with 25 -> 28 pgs (0.749064 -> 0.838951 * mean) 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.4 with 38 -> 40 pgs (1.13858 -> 1.1985 * mean) 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 1 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0334 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0367 2024-08-29T15:30:47.267 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 1.0000 -> 0.6728 2024-08-29T15:30:47.268 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 1.0000 -> 0.7666 2024-08-29T15:30:47.268 INFO:teuthology.orchestra.run.smithi027.stderr:no change 2024-08-29T15:30:47.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:30:47.265+0000 7f7dea97a640 -1 mgr.server reply reply (2) No such file or directory no change 2024-08-29T15:30:47.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:47.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:47.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:47.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:47.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:47.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:47.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:47.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:47.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:47.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:47.773 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:47.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:47.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:47.826 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:30:47.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:47.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:47.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:48.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[93921]: pgmap v1095: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 821 op/s 2024-08-29T15:30:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3874668004' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:48.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[93921]: from='client.36360 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:30:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[98970]: pgmap v1095: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 821 op/s 2024-08-29T15:30:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3874668004' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:48.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:47 smithi027 ceph-mon[98970]: from='client.36360 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:30:48.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:48.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:48.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:48.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:47 smithi171 ceph-mon[90031]: pgmap v1095: 89 pgs: 89 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 821 op/s 2024-08-29T15:30:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3874668004' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:47 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:30:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:47 smithi171 ceph-mon[90031]: from='client.36360 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:30:48.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:48.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:48.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:48.429 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:48.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:48.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:48.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:48.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:48.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:48.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:48.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:48.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:48.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:30:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:48.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:48.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:48.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:48.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:48.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:49.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:30:49.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:30:49.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:30:49.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[93921]: osdmap e304: 8 total, 8 up, 8 in 2024-08-29T15:30:49.075 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:30:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:30:49] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.51.0" 2024-08-29T15:30:49.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:30:49.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:30:49.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:30:49.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:48 smithi027 ceph-mon[98970]: osdmap e304: 8 total, 8 up, 8 in 2024-08-29T15:30:49.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:49.115 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:49.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:49.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:49.217 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:49.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:49.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:48 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:30:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:48 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:30:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:48 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:30:49.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:48 smithi171 ceph-mon[90031]: osdmap e304: 8 total, 8 up, 8 in 2024-08-29T15:30:49.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:49.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:49.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:49.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:49.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:49.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:49.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:49.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:49.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:49.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:49.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:49.822 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:49.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:49.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:49.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:49.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:49.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:50.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:50.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:49 smithi027 ceph-mon[93921]: pgmap v1097: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 840 op/s 2024-08-29T15:30:50.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:49 smithi027 ceph-mon[98970]: pgmap v1097: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 840 op/s 2024-08-29T15:30:50.097 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:50.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:50.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:50.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:50.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:50.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:50.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:49 smithi171 ceph-mon[90031]: pgmap v1097: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 36 MiB/s wr, 840 op/s 2024-08-29T15:30:50.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:50.391 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:50.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:50.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:50.430 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-29T15:30:50.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-29T15:30:50.491 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:50.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:50.523 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:50.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:50.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:50.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:50.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:50.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:50.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:50.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:50.882 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:50.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:50.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:51.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:51.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:51.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:51.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:51.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:51.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:51.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:51.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:51.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:51.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:51.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:51.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:51.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:51.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:51.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:51.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:51.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:51.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:51.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:51.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:51.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:51.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:30:51.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:51.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:51.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:51.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:52.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:52.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:51 smithi027 ceph-mon[93921]: pgmap v1098: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 861 op/s 2024-08-29T15:30:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:51 smithi027 ceph-mon[98970]: pgmap v1098: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 861 op/s 2024-08-29T15:30:52.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:52.133 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:52.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:52.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:52.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:51 smithi171 ceph-mon[90031]: pgmap v1098: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 861 op/s 2024-08-29T15:30:52.367 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:52.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:52.493 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:52.518 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:52.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:52.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:52.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:52.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:52.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:52.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:52.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:52.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:53.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:53.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:53.126 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:53.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:53.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:53.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:53.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:53.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:53.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:53.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:53.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:53.429 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:53.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:53.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:53.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:53.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:53.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:53.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:53.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:53.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:53.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:53.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:53.905 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:53 smithi027 ceph-mon[93921]: pgmap v1099: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 863 op/s 2024-08-29T15:30:53.906 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:53 smithi027 ceph-mon[98970]: pgmap v1099: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 863 op/s 2024-08-29T15:30:53.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:53.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:53.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:54.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:54.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:54.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:54.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:54.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:54.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:53 smithi171 ceph-mon[90031]: pgmap v1099: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 10 MiB/s wr, 863 op/s 2024-08-29T15:30:54.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:54.359 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:54.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:54.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:54.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:54.493 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:54.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:54.595 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:54.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:54.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:54.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:54.662 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:30:54.662 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:30:54.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:54.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:54.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:54.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:54.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:54.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:54.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:55.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.053 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:55.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:55.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:55.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:55.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:55.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:55.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:55.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:55.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:55.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:55.434 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:30:55.434 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:30:55.475 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:30:55.476 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:30:55.515 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:30:55.515 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:30:55.530 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:55.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:55.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:55.559 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:30:55.559 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:30:55.597 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:30:55.597 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:30:55.630 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:30:55.630 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:30:55.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:55.664 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:30:55.664 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:30:55.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:55.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:55.700 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:30:55.700 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:30:55.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:55.824 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:55.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:55.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:55.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:55.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:56.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:30:56.051 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:55 smithi027 ceph-mon[93921]: pgmap v1100: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 860 op/s 2024-08-29T15:30:56.051 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3938232935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:56.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:55 smithi027 ceph-mon[98970]: pgmap v1100: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 860 op/s 2024-08-29T15:30:56.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3938232935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:56.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:56.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:56.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:56.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:56.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:56.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:55 smithi171 ceph-mon[90031]: pgmap v1100: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 860 op/s 2024-08-29T15:30:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3938232935' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:30:56.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:56.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:56.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:56.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:56.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:56.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:56.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:56.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:56.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:56.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:56.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:56.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:56.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:56.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:56.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:56.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:56.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:56.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:57.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:57.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:57.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:57.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:57.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:57.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:57.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:57.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:30:57.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:57.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:57.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:30:57.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:57.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:57.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:57.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:57.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:57.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:57.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:57.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:57.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:57.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:57.809 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:57.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:57.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:57.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:57.934 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:30:58.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:57 smithi027 ceph-mon[93921]: pgmap v1101: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 544 KiB/s wr, 870 op/s 2024-08-29T15:30:58.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:30:58.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:57 smithi027 ceph-mon[98970]: pgmap v1101: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 544 KiB/s wr, 870 op/s 2024-08-29T15:30:58.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:30:58.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:30:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:58.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:58.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:30:58.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:30:58.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:58.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:58.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:30:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:57 smithi171 ceph-mon[90031]: pgmap v1101: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 544 KiB/s wr, 870 op/s 2024-08-29T15:30:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:30:58.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:58.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:58.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:58.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:58.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:58.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:58.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:58.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:58.630 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:58.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:58.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:58.732 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:30:58.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:30:58.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:58.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:58.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:30:58.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:30:58.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:58.933 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:58.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:30:59.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:59.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:59.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:30:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:30:59] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.51.0" 2024-08-29T15:30:59.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:59.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:59.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:59.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:59.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:59.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:30:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:59.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:30:59.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:59.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:59.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:30:59.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:30:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:30:59.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.851 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:59.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:30:59.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:30:59.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:30:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:30:59.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:30:59 smithi171 ceph-mon[90031]: pgmap v1102: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-29T15:31:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:30:59 smithi027 ceph-mon[98970]: pgmap v1102: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-29T15:31:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:30:59 smithi027 ceph-mon[93921]: pgmap v1102: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-29T15:31:00.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:00.126 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:00.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:00.228 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:00.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:00.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:00.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:00.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:00.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:00.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:00.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:00.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:00.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:00.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:00.658 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:00.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:00.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:moved 2 / 267 (0.749064%) 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:avg 33.375 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 3.99805 -> 4.21122 (expected baseline 5.40399) 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.5 with 25 -> 25 pgs (0.749064 -> 0.749064 * mean) 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.4 with 38 -> 38 pgs (1.13858 -> 1.13858 * mean) 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 0.05 2024-08-29T15:31:00.728 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:31:00.729 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0309 2024-08-29T15:31:00.729 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0340 2024-08-29T15:31:00.729 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 1.0000 -> 0.9500 2024-08-29T15:31:00.729 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 1.0000 -> 0.9500 2024-08-29T15:31:00.760 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:00.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:00.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:00.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:00.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:00.921 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:00.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:00 smithi171 ceph-mon[90031]: from='client.56053 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"} : dispatch 2024-08-29T15:31:01.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:01.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:01.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:00 smithi027 ceph-mon[98970]: from='client.56053 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"} : dispatch 2024-08-29T15:31:01.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:00 smithi027 ceph-mon[93921]: from='client.56053 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"} : dispatch 2024-08-29T15:31:01.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:01.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:01.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:01.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:01.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:01.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:01.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:01.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:01.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:01.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:01.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:01.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:01.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:01.808 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:01.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:01.931 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:01.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:01.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:01 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]': finished 2024-08-29T15:31:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:01 smithi171 ceph-mon[90031]: osdmap e305: 8 total, 8 up, 8 in 2024-08-29T15:31:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:01 smithi171 ceph-mon[90031]: pgmap v1104: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-29T15:31:02.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:02.054 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:02.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:01 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]': finished 2024-08-29T15:31:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:01 smithi027 ceph-mon[98970]: osdmap e305: 8 total, 8 up, 8 in 2024-08-29T15:31:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:01 smithi027 ceph-mon[98970]: pgmap v1104: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-29T15:31:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:01 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260'}"}]': finished 2024-08-29T15:31:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:01 smithi027 ceph-mon[93921]: osdmap e305: 8 total, 8 up, 8 in 2024-08-29T15:31:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:01 smithi027 ceph-mon[93921]: pgmap v1104: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-29T15:31:02.156 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:02.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:02.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:02.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:02.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:02.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:02.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:02.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:02.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:02.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:02.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:02.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:02.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:02.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:02.934 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:02.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:02 smithi171 ceph-mon[90031]: osdmap e306: 8 total, 8 up, 8 in 2024-08-29T15:31:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:02 smithi171 ceph-mon[90031]: osdmap e307: 8 total, 8 up, 8 in 2024-08-29T15:31:03.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:03.059 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:02 smithi027 ceph-mon[93921]: osdmap e306: 8 total, 8 up, 8 in 2024-08-29T15:31:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:02 smithi027 ceph-mon[93921]: osdmap e307: 8 total, 8 up, 8 in 2024-08-29T15:31:03.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:02 smithi027 ceph-mon[98970]: osdmap e306: 8 total, 8 up, 8 in 2024-08-29T15:31:03.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:02 smithi027 ceph-mon[98970]: osdmap e307: 8 total, 8 up, 8 in 2024-08-29T15:31:03.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:03.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:03.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:03.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:03.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:03.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:03.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:03.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:03.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:03.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:03.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:03.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:03.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:03.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:03.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:03.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:03.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[93921]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 304'8221 2024-08-29T15:31:04.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[93921]: pgmap v1107: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-29T15:31:04.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:04.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[98970]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 304'8221 2024-08-29T15:31:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[98970]: pgmap v1107: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-29T15:31:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:04.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:04.094 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:04.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:04.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:04.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:04.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:04.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:04.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:03 smithi171 ceph-mon[90031]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 304'8221 2024-08-29T15:31:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:03 smithi171 ceph-mon[90031]: pgmap v1107: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2024-08-29T15:31:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:04.347 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:04.374 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:04.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:04.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:04.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:04.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:04.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:04.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:04.586 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-08-29T15:31:04.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 0.05 2024-08-29T15:31:04.590 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:04.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:04.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:04.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:04.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:04.771 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:04.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:04.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:04.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:04.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:05.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:05.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:05.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:05.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:05.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:05.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:05.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:05.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:05.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:05.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:05.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:05.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:05.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:05.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:05.637 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:05.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:05.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:05.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:05.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:05.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:05.888 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:05.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:05.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:05.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:05.920 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:31:06.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:06.060 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:06.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:06.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:06.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:05 smithi027 ceph-mon[93921]: pgmap v1108: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-08-29T15:31:06.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3427652583' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:31:06.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:31:06.179 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:05 smithi027 ceph-mon[98970]: pgmap v1108: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-08-29T15:31:06.179 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3427652583' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:31:06.179 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:31:06.206 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:06.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:05 smithi171 ceph-mon[90031]: pgmap v1108: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-08-29T15:31:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3427652583' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:31:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:31:06.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:06.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:06.355 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:06.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:06.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:06.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:06.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:06.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:06.586 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:06.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:06.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:06.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:06.713 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:06.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:06.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:06.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:06.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:06.880 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:06.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:06.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:06.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:07.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:07.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:07.148 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:07.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:06 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:07.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:06 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:31:07.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:06 smithi027 ceph-mon[98970]: osdmap e308: 8 total, 8 up, 8 in 2024-08-29T15:31:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:07.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:06 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:07.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:06 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:31:07.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:06 smithi027 ceph-mon[93921]: osdmap e308: 8 total, 8 up, 8 in 2024-08-29T15:31:07.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:06 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:06 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:31:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:06 smithi171 ceph-mon[90031]: osdmap e308: 8 total, 8 up, 8 in 2024-08-29T15:31:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.312 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:07.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:07.413 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:07.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:07.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:07.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:07.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:07.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:07.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:07.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:07.756 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:07.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:07.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:07.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:07.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:07.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:08.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:07 smithi027 ceph-mon[98970]: pgmap v1110: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 5941/299310 objects misplaced (1.985%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:31:08.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:07 smithi027 ceph-mon[93921]: pgmap v1110: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 5941/299310 objects misplaced (1.985%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:31:08.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:08.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:08.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:08.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:08.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:07 smithi171 ceph-mon[90031]: pgmap v1110: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 5941/299310 objects misplaced (1.985%); 1.5 MiB/s, 23 objects/s recovering 2024-08-29T15:31:08.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:08.282 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:08.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:08.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:08.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:08.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:08.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:08.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:08.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:08.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:08.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:08.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:08.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:08.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:08.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:08.819 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:31:08.819 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:31:08.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:31:08.819 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:31:08.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.819 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106229 7.2 7:4f0c68a4:::benchmark_data_smithi027_271924_object106228:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.542883+0000", 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.270654383, 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.198535302, 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106229 2024-08-29T15:31:08.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542883+0000", 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542883+0000", 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542884+0000", 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.59e-06 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542907+0000", 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2853e-05 2024-08-29T15:31:08.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542924+0000", 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6844999999999999e-05 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542932+0000", 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9820000000000008e-06 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555831+0000", 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012899305 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555959+0000", 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012763 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556090+0000", 2024-08-29T15:31:08.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013128200000000001 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.563808+0000", 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007717313 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.567585+0000", 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037777039999999998 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741329+0000", 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17374320900000001 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741379+0000", 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0288999999999999e-05 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741418+0000", 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.93e-05 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106230 7.a 7:5c20eea8:::benchmark_data_smithi027_271924_object106229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.542947+0000", 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.270590460999998, 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19833519799999999, 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.827 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106230 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542947+0000", 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542947+0000", 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542947+0000", 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5699999999999998e-07 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.542999+0000", 2024-08-29T15:31:08.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2111000000000003e-05 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.543007+0000", 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8569999999999995e-06 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.543013+0000", 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4489999999999998e-06 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556136+0000", 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013123022 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556251+0000", 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114966 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556374+0000", 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000123008 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.563796+0000", 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074225070000000001 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.563812+0000", 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5868000000000001e-05 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741159+0000", 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.177347386 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741228+0000", 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8990000000000002e-05 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741282+0000", 2024-08-29T15:31:08.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3576999999999999e-05 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106244 7.2 7:423093e5:::benchmark_data_smithi027_271924_object106243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.555776+0000", 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.257761045000002, 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.198205623, 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.834 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106244 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555776+0000", 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555776+0000", 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555777+0000", 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.218e-06 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555798+0000", 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0431000000000001e-05 2024-08-29T15:31:08.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555810+0000", 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1807999999999999e-05 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.555818+0000", 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7970000000000001e-06 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.559849+0000", 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040317110000000003 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.559982+0000", 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013287100000000001 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.560111+0000", 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128462 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572503+0000", 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012392004 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577687+0000", 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051844899999999999 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.753885+0000", 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.176198302 2024-08-29T15:31:08.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.753942+0000", 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6496000000000003e-05 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.753982+0000", 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0033000000000001e-05 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106247 7.2 7:437d6680:::benchmark_data_smithi027_271924_object106246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.556209+0000", 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.257328053999998, 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.197887337, 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106247 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.841 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556209+0000", 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556209+0000", 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556210+0000", 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6499999999999998e-07 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556229+0000", 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8550999999999999e-05 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556240+0000", 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0866999999999999e-05 2024-08-29T15:31:08.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.556246+0000", 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7669999999999998e-06 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.563846+0000", 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075996500000000003 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.563966+0000", 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119992 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.564094+0000", 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012852 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572508+0000", 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00841305 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577745+0000", 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052371880000000003 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754024+0000", 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.176279669 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754066+0000", 2024-08-29T15:31:08.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1553e-05 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754097+0000", 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0664999999999999e-05 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106259 7.3 7:cc514a32:::benchmark_data_smithi027_271924_object106258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.572937+0000", 2024-08-29T15:31:08.847 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.240600563000001, 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17040417199999999, 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106259 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572937+0000", 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572937+0000", 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572939+0000", 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1430000000000001e-06 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572962+0000", 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2702999999999999e-05 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572974+0000", 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2592000000000001e-05 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572981+0000", 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0840000000000003e-06 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573041+0000", 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9596999999999999e-05 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573179+0000", 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013816300000000001 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573409+0000", 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023047599999999999 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577782+0000", 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043725370000000001 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578133+0000", 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035077799999999999 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743242+0000", 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.165109162 2024-08-29T15:31:08.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743298+0000", 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.588e-05 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743341+0000", 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3056999999999998e-05 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106265 7.2 7:4db730ac:::benchmark_data_smithi027_271924_object106264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.580002+0000", 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.233535181999997, 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.1743788, 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106265 2024-08-29T15:31:08.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580002+0000", 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580002+0000", 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580004+0000", 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7999999999999999e-06 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580053+0000", 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9521999999999998e-05 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580069+0000", 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5152e-05 2024-08-29T15:31:08.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580076+0000", 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6990000000000003e-06 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580136+0000", 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9777e-05 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580274+0000", 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137627 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580418+0000", 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014434800000000001 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.586685+0000", 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062671899999999997 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754132+0000", 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.167446331 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754285+0000", 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000153549 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754340+0000", 2024-08-29T15:31:08.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4635000000000001e-05 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754381+0000", 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1170000000000001e-05 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106266 7.2 7:46226f6b:::benchmark_data_smithi027_271924_object106265:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.580101+0000", 2024-08-29T15:31:08.861 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.233436261999998, 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17452821299999999, 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106266 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580101+0000", 2024-08-29T15:31:08.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580101+0000", 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580102+0000", 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7299999999999996e-07 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580157+0000", 2024-08-29T15:31:08.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4968e-05 2024-08-29T15:31:08.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580166+0000", 2024-08-29T15:31:08.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2900000000000008e-06 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.580172+0000", 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7370000000000003e-06 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.583428+0000", 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032568940000000002 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.583550+0000", 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121438 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.583693+0000", 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014257300000000001 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.586816+0000", 2024-08-29T15:31:08.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031235659999999999 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754495+0000", 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16767902600000001 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754538+0000", 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3018e-05 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754591+0000", 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2448999999999999e-05 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.754629+0000", 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8680999999999998e-05 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.868 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107327 7.a 7:5d62826e:::benchmark_data_smithi027_271924_object107326:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.025418+0000", 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 37.788118867000001, 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19663731600000001, 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107327 2024-08-29T15:31:08.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025418+0000", 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025418+0000", 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025420+0000", 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8729999999999999e-06 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025443+0000", 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2962000000000001e-05 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025463+0000", 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9386999999999999e-05 2024-08-29T15:31:08.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025471+0000", 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7649999999999999e-06 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025540+0000", 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8714999999999995e-05 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025680+0000", 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013994699999999999 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.025814+0000", 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133712 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.034975+0000", 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0091607219999999996 2024-08-29T15:31:08.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.034986+0000", 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1313e-05 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.221953+0000", 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18696742599999999 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222011+0000", 2024-08-29T15:31:08.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7867e-05 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222056+0000", 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4626999999999998e-05 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107328 7.a 7:504b982a:::benchmark_data_smithi027_271924_object107327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.029296+0000", 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 37.784240971999999, 2024-08-29T15:31:08.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19287316700000001, 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107328 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.029296+0000", 2024-08-29T15:31:08.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.029296+0000", 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.029298+0000", 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.547e-06 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.029327+0000", 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8929999999999999e-05 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.029345+0000", 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8012e-05 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.029353+0000", 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9610000000000005e-06 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030278+0000", 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00092487299999999999 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030411+0000", 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013379700000000001 2024-08-29T15:31:08.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:08.880 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:08.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:08.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030541+0000", 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012929400000000001 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035020+0000", 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044794889999999997 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.042481+0000", 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074611649999999996 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222090+0000", 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.179609091 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222137+0000", 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6752999999999998e-05 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222169+0000", 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2255000000000002e-05 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107331 7.3 7:c9a9d930:::benchmark_data_smithi027_271924_object107330:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.030751+0000", 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 37.782786035999997, 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.187354521, 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.884 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107331 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030751+0000", 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030751+0000", 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030752+0000", 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.096e-06 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030773+0000", 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0381999999999999e-05 2024-08-29T15:31:08.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030783+0000", 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0390999999999999e-05 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030799+0000", 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6327000000000001e-05 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030974+0000", 2024-08-29T15:31:08.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017466000000000001 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031094+0000", 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119858 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031217+0000", 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012284199999999999 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035295+0000", 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040780870000000002 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.036940+0000", 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001645117 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.217996+0000", 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18105571000000001 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.218058+0000", 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2532000000000004e-05 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.218106+0000", 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7518999999999997e-05 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.890 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107335 7.a 7:5cfd1d0d:::benchmark_data_smithi027_271924_object107334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.031069+0000", 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 37.782468543999997, 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19120510900000001, 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107335 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031069+0000", 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031069+0000", 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031070+0000", 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5000000000000001e-07 2024-08-29T15:31:08.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031090+0000", 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0781e-05 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031101+0000", 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0427e-05 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031108+0000", 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9240000000000001e-06 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035048+0000", 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039402309999999998 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035186+0000", 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137622 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035328+0000", 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014223400000000001 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.042539+0000", 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0072109499999999998 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.042582+0000", 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3000999999999997e-05 2024-08-29T15:31:08.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222201+0000", 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.179618954 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222244+0000", 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2784000000000002e-05 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222274+0000", 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0250999999999999e-05 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107337 7.a 7:50f00dfa:::benchmark_data_smithi027_271924_object107336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.031671+0000", 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 37.781866139999998, 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.190704436, 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.897 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107337 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031671+0000", 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031671+0000", 2024-08-29T15:31:08.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031672+0000", 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0299999999999997e-07 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031694+0000", 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2187e-05 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031705+0000", 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1209999999999999e-05 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031714+0000", 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5690000000000005e-06 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.039201+0000", 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074871820000000002 2024-08-29T15:31:08.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.039337+0000", 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135506 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.039490+0000", 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000153017 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.042575+0000", 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030853769999999998 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.042650+0000", 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5178000000000004e-05 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222305+0000", 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17965513999999999 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222346+0000", 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0917000000000001e-05 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.222376+0000", 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9249999999999999e-05 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.903 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110025 7.2 7:4fa9aa4d:::benchmark_data_smithi027_271924_object110024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.573623+0000", 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.239914181000003, 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29751161300000001, 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.904 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110025 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573623+0000", 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573623+0000", 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573624+0000", 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8599999999999998e-07 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573699+0000", 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5284000000000006e-05 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573708+0000", 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8189999999999998e-06 2024-08-29T15:31:08.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573714+0000", 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5350000000000001e-06 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600325+0000", 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0266109 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600826+0000", 2024-08-29T15:31:08.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00050131999999999998 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600960+0000", 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133497 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614386+0000", 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013426301999999999 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614495+0000", 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109065 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.870954+0000", 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25645894800000002 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.871081+0000", 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126985 2024-08-29T15:31:08.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.871135+0000", 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3471999999999999e-05 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110051 7.a 7:5df6ce9d:::benchmark_data_smithi027_271924_object110050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.600656+0000", 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.212881351999997, 2024-08-29T15:31:08.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35938326700000001, 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110051 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600656+0000", 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600656+0000", 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600657+0000", 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1459999999999999e-06 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600683+0000", 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6390999999999999e-05 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600694+0000", 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0438999999999999e-05 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600701+0000", 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4240000000000004e-06 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614640+0000", 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013938259 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614772+0000", 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013251300000000001 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614917+0000", 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000145067 2024-08-29T15:31:08.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622870+0000", 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079524539999999994 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626735+0000", 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038654359999999999 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.959899+0000", 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.333164397 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.959984+0000", 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4740000000000005e-05 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.960039+0000", 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5000999999999997e-05 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.916 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110064 7.a 7:52f01eaa:::benchmark_data_smithi027_271924_object110063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.614498+0000", 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.199039378000002, 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.359004622, 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.917 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110064 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614498+0000", 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614498+0000", 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614499+0000", 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.266e-06 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614519+0000", 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0137999999999999e-05 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614531+0000", 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1411e-05 2024-08-29T15:31:08.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614538+0000", 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9879999999999998e-06 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618649+0000", 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041114539999999996 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618765+0000", 2024-08-29T15:31:08.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115796 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618923+0000", 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015772599999999999 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622901+0000", 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039787269999999996 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626740+0000", 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038387320000000001 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973426+0000", 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34668595400000002 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973470+0000", 2024-08-29T15:31:08.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4051999999999998e-05 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973503+0000", 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2378000000000001e-05 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.923 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110066 7.2 7:420ab7a3:::benchmark_data_smithi027_271924_object110065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:08.924 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.615195+0000", 2024-08-29T15:31:08.924 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.198342558, 2024-08-29T15:31:08.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35819247199999998, 2024-08-29T15:31:08.924 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.924 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110066 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615195+0000", 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615195+0000", 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615196+0000", 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8299999999999995e-07 2024-08-29T15:31:08.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615214+0000", 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8365000000000001e-05 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615225+0000", 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0729e-05 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615231+0000", 2024-08-29T15:31:08.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7039999999999997e-06 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618426+0000", 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031946800000000001 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.619142+0000", 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00071609799999999997 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.619460+0000", 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031777800000000001 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.629806+0000", 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010345896 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.881698+0000", 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25189198099999999 2024-08-29T15:31:08.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973276+0000", 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091578520999999996 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973339+0000", 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2898000000000007e-05 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973387+0000", 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7939000000000003e-05 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110068 7.2 7:4efeab30:::benchmark_data_smithi027_271924_object110067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.618779+0000", 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.194757926000001, 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35482978100000001, 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.931 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110068 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618779+0000", 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618779+0000", 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618781+0000", 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.217e-06 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618802+0000", 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0936000000000001e-05 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618813+0000", 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1375999999999999e-05 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618842+0000", 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.923e-05 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.623102+0000", 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042601360000000003 2024-08-29T15:31:08.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.623259+0000", 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015687200000000001 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.623393+0000", 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013349799999999999 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.629814+0000", 2024-08-29T15:31:08.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064215679999999999 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.881890+0000", 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25207546600000003 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973534+0000", 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091644728999999994 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973577+0000", 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2110999999999997e-05 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973609+0000", 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2641999999999997e-05 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110074 7.2 7:47f6c0a9:::benchmark_data_smithi027_271924_object110073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:08.937 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.622690+0000", 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.190847681999998, 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35112648899999999, 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110074 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622690+0000", 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622690+0000", 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622691+0000", 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6500000000000001e-06 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622730+0000", 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9138999999999999e-05 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622742+0000", 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1659e-05 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622756+0000", 2024-08-29T15:31:08.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4297000000000001e-05 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626764+0000", 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040075370000000003 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626931+0000", 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000166976 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.627109+0000", 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017861299999999999 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.629980+0000", 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028709130000000001 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.882057+0000", 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25207687000000001 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973744+0000", 2024-08-29T15:31:08.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091687025000000005 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973785+0000", 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1072000000000002e-05 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973816+0000", 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0738000000000003e-05 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110076 7.3 7:c7ddcbee:::benchmark_data_smithi027_271924_object110075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.626845+0000", 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.186692790000002, 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33320111000000002, 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110076 2024-08-29T15:31:08.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626845+0000", 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626845+0000", 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626846+0000", 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.624e-06 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626872+0000", 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5880999999999998e-05 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626886+0000", 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4426999999999999e-05 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626896+0000", 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1859999999999998e-06 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626947+0000", 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1385999999999999e-05 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.627035+0000", 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7575000000000001e-05 2024-08-29T15:31:08.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.627123+0000", 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8832000000000006e-05 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.630447+0000", 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033235719999999999 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.630990+0000", 2024-08-29T15:31:08.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00054312800000000004 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.959956+0000", 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32896541400000001 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.960007+0000", 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1001000000000002e-05 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.960046+0000", 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9084000000000001e-05 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:131751 7.2 7:4dc739e1:::benchmark_data_smithi027_271924_object15410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:00.729385+0000", 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 8.0841524640000006, 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17094867, 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:31:08.953 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 131751 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.729385+0000", 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.729385+0000", 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.729386+0000", 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.564e-06 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.729391+0000", 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4619999999999998e-06 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.729408+0000", 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7088e-05 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.729417+0000", 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1740000000000006e-06 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:31:08.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.737464+0000", 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080467060000000007 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.737789+0000", 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000325655 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.740490+0000", 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027002419999999998 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.765589+0000", 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.025099109000000001 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.768702+0000", 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003113177 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.900183+0000", 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.131480931 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.900284+0000", 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101141 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:00.900333+0000", 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9421e-05 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:31:08.959 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:31:08.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:31:08.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:09.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:09.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:09.103 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:31:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:31:09] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.51.0" 2024-08-29T15:31:09.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:09.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:09.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:09.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:09.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:09.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:09.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:09.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:09.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:09.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:09.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:09.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:09.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:09.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:09.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:09.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:09.996 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:09.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:09.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:10.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:10.128 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:10.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:10.215 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:31:10.216 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:31:10.218 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:09 smithi027 ceph-mon[93921]: pgmap v1111: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 5941/297207 objects misplaced (1.999%); 1.3 MiB/s, 20 objects/s recovering 2024-08-29T15:31:10.219 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:09 smithi027 ceph-mon[98970]: pgmap v1111: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 5941/297207 objects misplaced (1.999%); 1.3 MiB/s, 20 objects/s recovering 2024-08-29T15:31:10.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:10.257 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:31:10.258 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:31:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:09 smithi171 ceph-mon[90031]: pgmap v1111: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 5941/297207 objects misplaced (1.999%); 1.3 MiB/s, 20 objects/s recovering 2024-08-29T15:31:10.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:10.285 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:10.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:10.302 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:31:10.303 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:31:10.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:31:10.335 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:31:10.371 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:31:10.372 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:31:10.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:10.405 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:31:10.406 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:31:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:10.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:10.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:10.436 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:31:10.436 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:31:10.473 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:31:10.473 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:31:10.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:10.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:10.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:10.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:10.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:10.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:10.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:10.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:10.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:10.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:10.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:10.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:10.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:10.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:10.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:11.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:11.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:11.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:11.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:11.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:11.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:11.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3012275486' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:11.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3012275486' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3012275486' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:11.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:11.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:11.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:11.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:11.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:11.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:11.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:11.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:11.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:11.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:11.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:11.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:11.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:11.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:11.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:11.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:11.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:12.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:11 smithi027 ceph-mon[98970]: pgmap v1112: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 5231/290190 objects misplaced (1.803%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:31:12.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:11 smithi027 ceph-mon[93921]: pgmap v1112: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 5231/290190 objects misplaced (1.803%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:31:12.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:12.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:12.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:12.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:12.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:12.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:11 smithi171 ceph-mon[90031]: pgmap v1112: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 5231/290190 objects misplaced (1.803%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:31:12.325 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:12.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:12.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:12.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:12.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:31:12.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:12.514 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:12.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:12.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:12.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:12.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:12.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:12.728 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:12.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:12.803 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:12.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:12.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:12.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:12.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:12.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:12.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:12.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:13.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:13.076 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:13.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:13.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:13.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:13.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:13.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:13.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:13.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:13.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:13.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:13.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:13.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:13.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:13.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:13.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:13.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:13.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:13.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:13.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:13.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:13.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:13.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:31:13.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:13.914 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:13.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:14.016 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:14.044 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:14.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:14.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:14.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:14.175 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:14.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:14.195 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:13 smithi171 ceph-mon[90031]: pgmap v1113: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5231/285810 objects misplaced (1.830%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:31:14.195 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:14.195 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:14.277 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:14.288 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:13 smithi027 ceph-mon[93921]: pgmap v1113: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5231/285810 objects misplaced (1.830%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:31:14.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:14.289 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:14.289 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:13 smithi027 ceph-mon[98970]: pgmap v1113: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 5231/285810 objects misplaced (1.830%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:31:14.289 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:14.289 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:moved 0 / 267 (0%) 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:avg 33.375 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 4.21122 -> 4.21122 (expected baseline 5.40399) 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.5 with 25 -> 25 pgs (0.749064 -> 0.749064 * mean) 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.1 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:oload 1000 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 0.05 2024-08-29T15:31:14.295 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:31:14.296 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0284 2024-08-29T15:31:14.296 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.2844 2024-08-29T15:31:14.298 INFO:teuthology.orchestra.run.smithi027.stderr:no change 2024-08-29T15:31:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:14.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:14.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:14.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:14.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:14.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:14.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:14.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:14.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:14.619 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:31:14.619 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:31:14.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:14.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:14.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:14.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:14.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:14.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:14.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.001 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:15.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:15.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:14 smithi027 ceph-mon[98970]: from='client.36381 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:15.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:14 smithi027 ceph-mon[93921]: from='client.36381 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:15.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:15.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:15.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:15.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:15.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:14 smithi171 ceph-mon[90031]: from='client.36381 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:15.267 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:15.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:15.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:15.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:15.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:15.431 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:31:15.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:15.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:15.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:15.698 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:15.709 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:15.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:15.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:15.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:15.964 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:15.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:16.065 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:16.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:16.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:16.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:16.101 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:15 smithi171 ceph-mon[90031]: pgmap v1114: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5231/282315 objects misplaced (1.853%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:31:16.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:16.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:16.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:16.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:15 smithi027 ceph-mon[93921]: pgmap v1114: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5231/282315 objects misplaced (1.853%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:31:16.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:15 smithi027 ceph-mon[98970]: pgmap v1114: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 5231/282315 objects misplaced (1.853%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:31:16.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:16.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:16.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:16.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:16.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:16.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:16.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:16.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:16.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:16.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:16.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:16.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:16.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:16.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:16.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:16.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:16.988 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:17.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:17.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:17.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:17.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:17.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:17.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:17.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:17.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:17.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:17.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:17.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:17.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:17.538 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:31:17.538 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:31:17.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:17.600 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:17.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:17.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:17.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:17.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:17.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:17.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:17.872 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:17.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:17.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:17.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:18.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:18.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:18.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:18.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:18.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:18.236 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:17 smithi171 ceph-mon[90031]: pgmap v1115: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4586/277212 objects misplaced (1.654%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:31:18.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:18.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:31:18.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:17 smithi027 ceph-mon[93921]: pgmap v1115: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4586/277212 objects misplaced (1.654%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:31:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:17 smithi027 ceph-mon[98970]: pgmap v1115: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4586/277212 objects misplaced (1.654%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:31:18.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:18.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:18.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:18.544 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:18.552 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-29T15:31:18.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 110 3.0 2024-08-29T15:31:18.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:18.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:18.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:18.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.685 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:18.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:18.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:18.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:18.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:18.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:18.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:18.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:19.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:19.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:19.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:19.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:19.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:19.214 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:19.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:19.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:19.315 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:31:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:31:19] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-08-29T15:31:19.363 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:19.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:19.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:19.472 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:19.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:19.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:19.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:19.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:19.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:19.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:19.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:19.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:19.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:19.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:19.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:19.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:19.989 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:31:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:19 smithi171 ceph-mon[90031]: pgmap v1116: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 4586/274533 objects misplaced (1.670%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:31:20.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3503978297' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:20.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:19 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:20.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:20.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:20.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:20.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:20.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:20.211 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:20.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:20.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:19 smithi027 ceph-mon[98970]: pgmap v1116: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 4586/274533 objects misplaced (1.670%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:31:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3503978297' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:19 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:20.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:19 smithi027 ceph-mon[93921]: pgmap v1116: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 4586/274533 objects misplaced (1.670%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:31:20.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3503978297' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:20.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:19 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:20.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:20.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:20.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:20.477 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:20.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:20.566 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:31:20.566 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:31:20.579 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:20.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:20.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:20.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:20.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:20.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:20.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:20.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:20.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:20.888 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:20.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:20.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:20.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:21.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:21.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:21.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:21.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:21.149 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[98970]: osdmap e309: 8 total, 8 up, 8 in 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[93921]: osdmap e309: 8 total, 8 up, 8 in 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:21.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:20 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:21.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:21.183 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:20 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:21.183 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:20 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:31:21.183 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:20 smithi171 ceph-mon[90031]: osdmap e309: 8 total, 8 up, 8 in 2024-08-29T15:31:21.183 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:21.183 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:20 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:21.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:21.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:21.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:21.216 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:31:21.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:21.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:21.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:21.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:21.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:21.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:21.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:21.511 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:21.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:21.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:21.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:21.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:21.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:21.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:21.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:21.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:21.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:21.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:21.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:21.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:22.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:22.046 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:22.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:22.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:22.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:22.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:22.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:22 smithi171 ceph-mon[90031]: pgmap v1118: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 3913/267444 objects misplaced (1.463%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:31:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:31:22.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:22.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[93921]: pgmap v1118: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 3913/267444 objects misplaced (1.463%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:22.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:31:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:31:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[98970]: pgmap v1118: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 3913/267444 objects misplaced (1.463%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:31:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:31:22.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:22.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:22.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:22.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:22.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:22.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:22.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:22.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:22.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:22.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:22.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:22.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:22.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:22.750 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:22.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:22.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:22.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:22.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:22.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:22.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:22.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:31:22.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:23.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:23.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:23.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:23.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:23.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:23.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:23.377 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:31:23.377 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:31:23.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:23.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:23.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:23.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:23.570 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:23.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:23.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:23.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:23.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.864 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:23.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:23.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:23.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:23.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:23.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:24.023 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:31:24.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:24.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:24.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:24.246 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:24.247 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:24 smithi027 ceph-mon[93921]: pgmap v1119: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3913/263328 objects misplaced (1.486%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:24.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:24 smithi027 ceph-mon[98970]: pgmap v1119: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3913/263328 objects misplaced (1.486%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:24 smithi171 ceph-mon[90031]: pgmap v1119: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3913/263328 objects misplaced (1.486%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:24.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:24.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:24.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:24.323 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:24.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:24.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:24.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:24.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:24.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:24.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:24.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:24.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:24.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:24.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:24.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:24.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:24.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:24.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:24.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:24.992 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:24.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:24.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:25.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:25.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:25.156 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:31:25.156 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:31:25.167 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:25.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:31:25.175 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:31:25.178 INFO:teuthology.orchestra.run.smithi027.stdout:moved 23 / 267 (8.61423%) 2024-08-29T15:31:25.178 INFO:teuthology.orchestra.run.smithi027.stdout:avg 33.375 2024-08-29T15:31:25.178 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 4.21122 -> 7.63115 (expected baseline 5.40399) 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.5 with 25 -> 28 pgs (0.749064 -> 0.838951 * mean) 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.1 with 39 -> 47 pgs (1.16854 -> 1.40824 * mean) 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 3 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0265 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0291 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.9500 -> 0.6448 2024-08-29T15:31:25.179 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 0.9500 -> 0.7485 2024-08-29T15:31:25.208 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:31:25.208 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:31:25.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:25.229 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:31:25.229 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:31:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:25.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:25.249 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:31:25.249 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:31:25.268 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:31:25.268 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:31:25.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1243542454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:25.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'49056','3':'42256'}"} : dispatch 2024-08-29T15:31:25.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1243542454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:25.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'49056','3':'42256'}"} : dispatch 2024-08-29T15:31:25.286 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:31:25.286 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:31:25.326 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:31:25.327 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:31:25.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:25.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:25.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:25.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1243542454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'49056','3':'42256'}"} : dispatch 2024-08-29T15:31:25.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:25.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:25.721 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:25.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:25.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:25.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:25.866 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:25.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:25.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:25.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:25.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:26.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:26.115 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:26.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:26.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:26.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:26.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:26.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:26.468 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:26.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:26 smithi171 ceph-mon[90031]: pgmap v1120: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 3913/259974 objects misplaced (1.505%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:26 smithi171 ceph-mon[90031]: from='client.46301 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'49056','3':'42256'}"}]': finished 2024-08-29T15:31:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:26 smithi171 ceph-mon[90031]: osdmap e310: 8 total, 8 up, 8 in 2024-08-29T15:31:26.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[98970]: pgmap v1120: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 3913/259974 objects misplaced (1.505%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:26.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[98970]: from='client.46301 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:26.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'49056','3':'42256'}"}]': finished 2024-08-29T15:31:26.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[98970]: osdmap e310: 8 total, 8 up, 8 in 2024-08-29T15:31:26.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[93921]: pgmap v1120: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 3913/259974 objects misplaced (1.505%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:31:26.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[93921]: from='client.46301 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:26.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'49056','3':'42256'}"}]': finished 2024-08-29T15:31:26.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:26 smithi027 ceph-mon[93921]: osdmap e310: 8 total, 8 up, 8 in 2024-08-29T15:31:26.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:26.614 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:26.716 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:26.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:26.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:26.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:26.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:26.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:26.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:26.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:26.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:26.996 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-08-29T15:31:26.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 1.0 2024-08-29T15:31:27.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:27.108 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:27.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:27.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:27.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:27.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:27.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:27.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:27 smithi027 ceph-mon[98970]: osdmap e311: 8 total, 8 up, 8 in 2024-08-29T15:31:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:27 smithi027 ceph-mon[98970]: osdmap e312: 8 total, 8 up, 8 in 2024-08-29T15:31:27.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:27 smithi027 ceph-mon[93921]: osdmap e311: 8 total, 8 up, 8 in 2024-08-29T15:31:27.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:27 smithi027 ceph-mon[93921]: osdmap e312: 8 total, 8 up, 8 in 2024-08-29T15:31:27.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:27.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:27.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:27.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:27 smithi171 ceph-mon[90031]: osdmap e311: 8 total, 8 up, 8 in 2024-08-29T15:31:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:27 smithi171 ceph-mon[90031]: osdmap e312: 8 total, 8 up, 8 in 2024-08-29T15:31:27.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:27.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:27.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:27.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:27.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:27.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:27.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:27.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:27.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:27.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:27.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:27.911 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:27.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:28.013 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:28.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:28.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:28.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:28.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:28.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:28.290 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:28.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:28.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:28.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:28.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:28 smithi171 ceph-mon[90031]: pgmap v1123: 89 pgs: 1 unknown, 5 peering, 1 active+remapped+backfilling, 82 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 3267/239439 objects misplaced (1.364%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:31:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:28 smithi171 ceph-mon[90031]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 309'9269 2024-08-29T15:31:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:28 smithi171 ceph-mon[90031]: 7.7 starting backfill to osd.1 from (0'0,0'0] MAX to 309'9483 2024-08-29T15:31:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:28 smithi171 ceph-mon[90031]: 7.6 starting backfill to osd.5 from (0'0,0'0] MAX to 309'9238 2024-08-29T15:31:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:28 smithi171 ceph-mon[90031]: 7.5 starting backfill to osd.1 from (0'0,0'0] MAX to 309'9328 2024-08-29T15:31:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:28 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-29T15:31:28.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:28.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:28.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:28.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:31:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[98970]: pgmap v1123: 89 pgs: 1 unknown, 5 peering, 1 active+remapped+backfilling, 82 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 3267/239439 objects misplaced (1.364%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:31:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[98970]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 309'9269 2024-08-29T15:31:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[98970]: 7.7 starting backfill to osd.1 from (0'0,0'0] MAX to 309'9483 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[98970]: 7.6 starting backfill to osd.5 from (0'0,0'0] MAX to 309'9238 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[98970]: 7.5 starting backfill to osd.1 from (0'0,0'0] MAX to 309'9328 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[93921]: pgmap v1123: 89 pgs: 1 unknown, 5 peering, 1 active+remapped+backfilling, 82 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 3267/239439 objects misplaced (1.364%); 4.5 MiB/s, 72 objects/s recovering 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[93921]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 309'9269 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[93921]: 7.7 starting backfill to osd.1 from (0'0,0'0] MAX to 309'9483 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[93921]: 7.6 starting backfill to osd.5 from (0'0,0'0] MAX to 309'9238 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[93921]: 7.5 starting backfill to osd.1 from (0'0,0'0] MAX to 309'9328 2024-08-29T15:31:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:28 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-29T15:31:28.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:28.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.723 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:28.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:28.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:28.846 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:28.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:28.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:28.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:29.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:29.107 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:29.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:29.155 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:29.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:29.235 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:29.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:29.243 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:31:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:31:29] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-08-29T15:31:29.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:29.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:29.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:29.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:29 smithi171 ceph-mon[90031]: osdmap e313: 8 total, 8 up, 8 in 2024-08-29T15:31:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:29.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:29.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:29.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:29.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:29 smithi027 ceph-mon[93921]: osdmap e313: 8 total, 8 up, 8 in 2024-08-29T15:31:29.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:29.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:29.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:29 smithi027 ceph-mon[98970]: osdmap e313: 8 total, 8 up, 8 in 2024-08-29T15:31:29.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:29.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:29.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:29.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:29.688 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:29.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:29.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:29.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:29.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:29.848 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:31:29.848 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:31:29.848 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:31:29.848 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:31:29.848 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:31:29.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:29.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:29.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:30.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:30.115 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:30.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:30.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:30.236 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:30.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:30.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:30.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:30.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:30.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:30.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:30.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:30 smithi171 ceph-mon[90031]: pgmap v1126: 89 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 unknown, 5 peering, 1 active+remapped+backfilling, 80 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 7156/237657 objects degraded (3.011%); 3267/237657 objects misplaced (1.375%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:31:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:30 smithi171 ceph-mon[90031]: osdmap e314: 8 total, 8 up, 8 in 2024-08-29T15:31:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:30 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 7156/237657 objects degraded (3.011%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:31:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2814598554' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:31:30.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:30.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[98970]: pgmap v1126: 89 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 unknown, 5 peering, 1 active+remapped+backfilling, 80 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 7156/237657 objects degraded (3.011%); 3267/237657 objects misplaced (1.375%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:31:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[98970]: osdmap e314: 8 total, 8 up, 8 in 2024-08-29T15:31:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 7156/237657 objects degraded (3.011%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:31:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2814598554' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:31:30.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[93921]: pgmap v1126: 89 pgs: 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 unknown, 5 peering, 1 active+remapped+backfilling, 80 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 7156/237657 objects degraded (3.011%); 3267/237657 objects misplaced (1.375%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:31:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[93921]: osdmap e314: 8 total, 8 up, 8 in 2024-08-29T15:31:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 7156/237657 objects degraded (3.011%), 1 pg degraded (PG_DEGRADED) 2024-08-29T15:31:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2814598554' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:31:30.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:31:30.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:30.691 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:30.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:30.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:30.708 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:31:30.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:30.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:30.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:30.845 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:30.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:31.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:31.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:31.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:31.142 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:31.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:31.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:31.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:31.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:31.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:31.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:31 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2814598554' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:31:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:31 smithi171 ceph-mon[90031]: osdmap e315: 8 total, 8 up, 8 in 2024-08-29T15:31:31.530 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:31.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:31 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:31.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2814598554' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:31:31.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:31 smithi027 ceph-mon[98970]: osdmap e315: 8 total, 8 up, 8 in 2024-08-29T15:31:31.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:31 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:31:31.532 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2814598554' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:31:31.532 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:31 smithi027 ceph-mon[93921]: osdmap e315: 8 total, 8 up, 8 in 2024-08-29T15:31:31.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:31.655 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:31.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:31.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:31.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:31.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:31.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:31.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:31.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:31.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:32.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:32.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:32.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:32.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:32.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:32.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:32.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:32.365 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:32.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:32.466 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:32.492 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:32.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:32.503 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:32 smithi027 ceph-mon[98970]: pgmap v1129: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 remapped+peering, 5 peering, 76 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 MiB/s wr, 2.65k op/s; 14454/251637 objects degraded (5.744%); 22724/251637 objects misplaced (9.030%); 25 MiB/s, 403 objects/s recovering 2024-08-29T15:31:32.503 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:32 smithi027 ceph-mon[93921]: pgmap v1129: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 remapped+peering, 5 peering, 76 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 MiB/s wr, 2.65k op/s; 14454/251637 objects degraded (5.744%); 22724/251637 objects misplaced (9.030%); 25 MiB/s, 403 objects/s recovering 2024-08-29T15:31:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:32 smithi171 ceph-mon[90031]: pgmap v1129: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 remapped+peering, 5 peering, 76 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 MiB/s wr, 2.65k op/s; 14454/251637 objects degraded (5.744%); 22724/251637 objects misplaced (9.030%); 25 MiB/s, 403 objects/s recovering 2024-08-29T15:31:32.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:32.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:32.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:32.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:32.641 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:31:32.641 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:31:32.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:31:32.641 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:31:32.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:221963 6.3 6:c8074485:::benchmark_data_smithi027_169445_object96536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 247.39707217200001, 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.875189047, 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221963 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241082+0000", 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:32.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241090+0000", 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6760000000000007e-06 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241103+0000", 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3097e-05 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241110+0000", 2024-08-29T15:31:32.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1189999999999999e-06 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.286106+0000", 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.044995852000000003 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.289616+0000", 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035105090000000002 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.307511+0000", 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017894131000000001 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.309413+0000", 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019022659999999999 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.986973+0000", 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.67756063300000002 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.989188+0000", 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022149930000000002 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.993558+0000", 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043696730000000001 2024-08-29T15:31:32.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.022139+0000", 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028581273000000001 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.043843+0000", 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021703999000000002 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.110257+0000", 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066413278000000006 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.111466+0000", 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001209578 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.116273+0000", 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0048067459999999998 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.649 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:221967 6.3 6:c4ba2f51:::benchmark_data_smithi027_169445_object96540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 247.378443498, 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86481450699999995, 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:32.650 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221967 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259711+0000", 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259717+0000", 2024-08-29T15:31:32.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4289999999999998e-06 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259726+0000", 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8389999999999996e-06 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259731+0000", 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3930000000000001e-06 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.289939+0000", 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.03020888 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.292731+0000", 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027917939999999998 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.344052+0000", 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051320328999999998 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.345748+0000", 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016965680000000001 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.009894+0000", 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.66414622700000003 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.012206+0000", 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0023118190000000001 2024-08-29T15:31:32.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.016501+0000", 2024-08-29T15:31:32.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042943189999999996 2024-08-29T15:31:32.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.076191+0000", 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.059690875999999997 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.117656+0000", 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041464320999999998 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.118388+0000", 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00073274099999999997 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.119598+0000", 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012098269999999999 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.124527+0000", 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049288099999999996 2024-08-29T15:31:32.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:222226 6.3 6:c82e9241:::benchmark_data_smithi027_169445_object96799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 246.34580161100001, 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86422102700000003, 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:32.659 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 222226 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292353+0000", 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292360+0000", 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4229999999999999e-06 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292373+0000", 2024-08-29T15:31:32.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.291e-05 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292381+0000", 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1340000000000004e-06 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.308321+0000", 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015940122000000001 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:31:32.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.310974+0000", 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026525699999999999 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.338098+0000", 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027124519999999999 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.339549+0000", 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001450724 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.061922+0000", 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.72237274799999995 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.064096+0000", 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021742850000000002 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.068373+0000", 2024-08-29T15:31:32.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042769779999999999 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.099354+0000", 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030980508 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.100191+0000", 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00083792800000000005 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.149351+0000", 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049159714 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.150692+0000", 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013410850000000001 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.156575+0000", 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0058832499999999996 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.666 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:222756 6.3 6:c4cc6026:::benchmark_data_smithi027_169445_object97329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 244.84226240300001, 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86913315599999996, 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:32.667 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 222756 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795892+0000", 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795898+0000", 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4450000000000001e-06 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795911+0000", 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3353000000000001e-05 2024-08-29T15:31:32.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795918+0000", 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0160000000000003e-06 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.800436+0000", 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045175850000000002 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.801672+0000", 2024-08-29T15:31:32.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012355319999999999 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.890056+0000", 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088384024000000005 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.891521+0000", 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014655689999999999 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.630976+0000", 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.73945498399999998 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.633483+0000", 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025067330000000001 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.637874+0000", 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043910590000000001 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.657485+0000", 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019610737 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.658230+0000", 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00074571299999999995 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.658976+0000", 2024-08-29T15:31:32.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00074594099999999997 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.660269+0000", 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012926579999999999 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.665027+0000", 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004757809 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:230075 6.3 6:c4c54f8a:::benchmark_data_smithi027_169445_object104648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 223.64438392, 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.89382495100000003, 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.675 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 230075 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.676 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993771+0000", 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993778+0000", 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8019999999999992e-06 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993792+0000", 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3959e-05 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993799+0000", 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0040000000000002e-06 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.231044+0000", 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.237244607 2024-08-29T15:31:32.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.232235+0000", 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001191454 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.388426+0000", 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.156190521 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:31:32.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.389841+0000", 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001415531 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.868908+0000", 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.479066673 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.870800+0000", 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018920810000000001 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.874105+0000", 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033051000000000001 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.881899+0000", 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0077939180000000004 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.882530+0000", 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00063088800000000002 2024-08-29T15:31:32.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.882723+0000", 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000193396 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.883672+0000", 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00094804899999999996 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.887597+0000", 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039255740000000003 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:231480 6.3 6:c1b858f2:::benchmark_data_smithi027_169445_object106053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 220.15307030100001, 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.92456820500000003, 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.683 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 231480 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:31:32.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485084+0000", 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485091+0000", 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485105+0000", 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3947999999999999e-05 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485113+0000", 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5220000000000001e-06 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.563897+0000", 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078784891999999995 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.565127+0000", 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012290720000000001 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.667308+0000", 2024-08-29T15:31:32.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10218187500000001 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.668461+0000", 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001152882 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.377239+0000", 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.70877815099999997 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.379283+0000", 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002043703 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.383071+0000", 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037881690000000001 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.402937+0000", 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019865765000000001 2024-08-29T15:31:32.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.403638+0000", 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000700682 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.403872+0000", 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00023457 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.405059+0000", 2024-08-29T15:31:32.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00118629 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.409654+0000", 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045953820000000003 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:104849 7.d 7:b726e007:::benchmark_data_smithi027_271924_object104848:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.691 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:27.824589+0000", 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 64.813567008999996, 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12709583999999999, 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 104849 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824589+0000", 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824589+0000", 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824588+0000", 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824665+0000", 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7787999999999995e-05 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824679+0000", 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3422e-05 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824686+0000", 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1860000000000003e-06 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824768+0000", 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.229e-05 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824897+0000", 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000128787 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.825025+0000", 2024-08-29T15:31:32.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000128369 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.831752+0000", 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067265720000000001 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.833893+0000", 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021408690000000001 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.951598+0000", 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.117704964 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.951646+0000", 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8615000000000001e-05 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.951685+0000", 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8491000000000001e-05 2024-08-29T15:31:32.697 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:104850 7.d 7:bb99fcb6:::benchmark_data_smithi027_271924_object104849:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:27.824712+0000", 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 64.813444246000003, 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.130523582, 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.698 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 104850 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824712+0000", 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824712+0000", 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.699 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.700 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.700 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824711+0000", 2024-08-29T15:31:32.700 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:31:32.700 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.700 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.700 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.701 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824834+0000", 2024-08-29T15:31:32.701 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012243700000000001 2024-08-29T15:31:32.701 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.701 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.701 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824907+0000", 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3020000000000002e-05 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.824997+0000", 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0288e-05 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.828438+0000", 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034414480000000002 2024-08-29T15:31:32.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.828549+0000", 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000110641 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.828676+0000", 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012721799999999999 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.703 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.831944+0000", 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032676599999999999 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.834029+0000", 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020849509999999998 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.955128+0000", 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.121098898 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.704 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.955198+0000", 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0679000000000005e-05 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.955235+0000", 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7048999999999997e-05 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:106267 7.d 7:b9baa89f:::benchmark_data_smithi027_271924_object106266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:29.580297+0000", 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 63.057858994, 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16656731, 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 106267 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.706 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580297+0000", 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580297+0000", 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580295+0000", 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:32.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580346+0000", 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0374999999999998e-05 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580362+0000", 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5965e-05 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.708 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580369+0000", 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7000000000000008e-06 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580414+0000", 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4737000000000002e-05 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580559+0000", 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014460800000000001 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.709 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580695+0000", 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013610799999999999 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.583531+0000", 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028361409999999999 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.584462+0000", 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00093147399999999995 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.746807+0000", 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.162344087 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.746840+0000", 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3581000000000003e-05 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.746864+0000", 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4261999999999999e-05 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.711 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107281 7.7 7:edf7da9f:::benchmark_data_smithi027_271924_object107280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.862099+0000", 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 61.776057289999997, 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16102459599999999, 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107281 2024-08-29T15:31:32.712 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862099+0000", 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862099+0000", 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862097+0000", 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862145+0000", 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8124000000000002e-05 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.714 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862164+0000", 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8284999999999999e-05 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862172+0000", 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9820000000000008e-06 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862217+0000", 2024-08-29T15:31:32.715 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5518e-05 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862360+0000", 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014341899999999999 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,3", 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862497+0000", 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013688700000000001 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.716 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.866010+0000", 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035125099999999999 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.878345+0000", 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012334921 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023052+0000", 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14470682000000001 2024-08-29T15:31:32.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023086+0000", 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4050000000000001e-05 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023123+0000", 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7694999999999997e-05 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107287 7.7 7:effdeacc:::benchmark_data_smithi027_271924_object107286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.867285+0000", 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 61.770871182999997, 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15597306499999999, 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107287 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.719 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867285+0000", 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867285+0000", 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867283+0000", 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867337+0000", 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3868e-05 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867349+0000", 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1878999999999999e-05 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867356+0000", 2024-08-29T15:31:32.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3899999999999998e-06 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867396+0000", 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0698999999999998e-05 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867532+0000", 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013618699999999999 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,3", 2024-08-29T15:31:32.722 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867687+0000", 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015432600000000001 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.870851+0000", 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031643180000000002 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.878387+0000", 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075357169999999999 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.723 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023171+0000", 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.144784568 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023214+0000", 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2509999999999998e-05 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023258+0000", 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4023000000000002e-05 2024-08-29T15:31:32.724 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107291 7.c 7:3f2ceea1:::benchmark_data_smithi027_271924_object107290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.869053+0000", 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 61.769102734999997, 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13527196999999999, 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.725 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107291 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869053+0000", 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869053+0000", 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.726 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869052+0000", 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869128+0000", 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5662999999999997e-05 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869140+0000", 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2081e-05 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.727 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869147+0000", 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0269999999999998e-06 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869182+0000", 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4298000000000003e-05 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869267+0000", 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5471999999999998e-05 2024-08-29T15:31:32.728 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869345+0000", 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8379000000000003e-05 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.876566+0000", 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072205790000000004 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.729 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.876609+0000", 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3090000000000002e-05 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.004269+0000", 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12765977000000001 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.004302+0000", 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3501000000000003e-05 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.730 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.004325+0000", 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2983e-05 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107292 7.c 7:33491277:::benchmark_data_smithi027_271924_object107291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 61.768503674000002, 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.141884811, 2024-08-29T15:31:32.731 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107292 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.732 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869732+0000", 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.017e-05 2024-08-29T15:31:32.733 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869745+0000", 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3499000000000001e-05 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869752+0000", 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4799999999999998e-06 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.873032+0000", 2024-08-29T15:31:32.734 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032800160000000002 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.873166+0000", 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013469400000000001 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.873309+0000", 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014280900000000001 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.735 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.876602+0000", 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032927659999999999 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.876615+0000", 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3535e-05 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.011451+0000", 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.134835548 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.736 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.011503+0000", 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2157000000000002e-05 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.011537+0000", 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4010999999999997e-05 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.737 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107304 7.c 7:32e064a1:::benchmark_data_smithi027_271924_object107303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.882616+0000", 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 61.755540215000003, 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12986541400000001, 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107304 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.738 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882616+0000", 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882616+0000", 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882615+0000", 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.739 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882679+0000", 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3981999999999998e-05 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882689+0000", 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4599999999999992e-06 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882695+0000", 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5209999999999999e-06 2024-08-29T15:31:32.740 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882738+0000", 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3180999999999999e-05 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882846+0000", 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010777699999999999 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:31:32.741 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.882976+0000", 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013003900000000001 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.886220+0000", 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032439119999999998 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.886332+0000", 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011196699999999999 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.742 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.012408+0000", 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12607554400000001 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.012453+0000", 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5889000000000001e-05 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.012481+0000", 2024-08-29T15:31:32.743 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7827e-05 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110071 7.d 7:b7d1d271:::benchmark_data_smithi027_271924_object110070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:34.619346+0000", 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 58.018810246999998, 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.259612114, 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.744 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110071 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619346+0000", 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619346+0000", 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.745 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619345+0000", 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619412+0000", 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7166e-05 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.746 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619423+0000", 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1192000000000001e-05 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619429+0000", 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4359999999999999e-06 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619498+0000", 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8756000000000004e-05 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.747 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619639+0000", 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014112599999999999 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619786+0000", 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014683200000000001 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.625738+0000", 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0059519899999999999 2024-08-29T15:31:32.748 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.626604+0000", 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00086641699999999997 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878875+0000", 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25227093099999998 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878915+0000", 2024-08-29T15:31:32.749 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9335000000000002e-05 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878958+0000", 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3220999999999998e-05 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110075 7.d 7:bd8da132:::benchmark_data_smithi027_271924_object110074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:34.622747+0000", 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 58.015408708000002, 2024-08-29T15:31:32.750 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.256296413, 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110075 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622747+0000", 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.751 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622747+0000", 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622746+0000", 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622789+0000", 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3031000000000002e-05 2024-08-29T15:31:32.752 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622800+0000", 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0584999999999999e-05 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622807+0000", 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6760000000000001e-06 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622864+0000", 2024-08-29T15:31:32.753 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7042e-05 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622953+0000", 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9240999999999995e-05 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.623047+0000", 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3597000000000005e-05 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.754 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.625811+0000", 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027641050000000002 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.626752+0000", 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00094161500000000003 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878990+0000", 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25223777400000003 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.755 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.879014+0000", 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4241999999999999e-05 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.879044+0000", 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9402000000000001e-05 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148244 7.c 7:3ee65318:::benchmark_data_smithi027_271924_object31903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:31:32.756 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:25.315929+0000", 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 7.322227367, 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4513220809999998, 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148244 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315929+0000", 2024-08-29T15:31:32.757 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315929+0000", 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315927+0000", 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.758 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315934+0000", 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.46e-06 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315947+0000", 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3696999999999999e-05 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315956+0000", 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7849999999999997e-06 2024-08-29T15:31:32.759 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.316004+0000", 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7997999999999997e-05 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.316013+0000", 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5129999999999996e-06 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:26.185362+0000", 2024-08-29T15:31:32.760 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86934909800000004 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:26.185371+0000", 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2089999999999994e-06 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.708601+0000", 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5232299839999999 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.761 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.708754+0000", 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015327 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.709440+0000", 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000685437 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.736751+0000", 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027311275999999999 2024-08-29T15:31:32.762 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.741158+0000", 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044072649999999996 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763538+0000", 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022380015 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763587+0000", 2024-08-29T15:31:32.763 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.833e-05 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.767251+0000", 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036641159999999998 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148261 7.c 7:3a32fcb3:::benchmark_data_smithi027_271924_object31920:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:27.521671+0000", 2024-08-29T15:31:32.764 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 5.11648557, 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26602938700000001, 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148261 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521671+0000", 2024-08-29T15:31:32.765 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521671+0000", 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521668+0000", 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.766 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521677+0000", 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4840000000000004e-06 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521692+0000", 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5509e-05 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521700+0000", 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3710000000000002e-06 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.767 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521762+0000", 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2642000000000001e-05 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521775+0000", 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2765999999999999e-05 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.728975+0000", 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.20719976800000001 2024-08-29T15:31:32.768 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.729117+0000", 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014175900000000001 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.729268+0000", 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015089199999999999 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.741109+0000", 2024-08-29T15:31:32.769 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011841841000000001 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748263+0000", 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0071532009999999997 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.783481+0000", 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035218226999999998 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.770 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.783537+0000", 2024-08-29T15:31:32.771 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5933999999999997e-05 2024-08-29T15:31:32.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.787700+0000", 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041630599999999997 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.772 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148272 7.c 7:3b74bfd4:::benchmark_data_smithi027_271924_object31931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:27.620587+0000", 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 5.0175691650000003, 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17806313500000001, 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148272 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.773 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620587+0000", 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620587+0000", 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620586+0000", 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.774 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620592+0000", 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8549999999999996e-06 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620606+0000", 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4053000000000001e-05 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.775 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620612+0000", 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7490000000000004e-06 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620656+0000", 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3389000000000001e-05 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620664+0000", 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7320000000000004e-06 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.776 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.733242+0000", 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11257816599999999 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.733397+0000", 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015458400000000001 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.733523+0000", 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012611600000000001 2024-08-29T15:31:32.777 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.745008+0000", 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011484938 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748316+0000", 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033082559999999999 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.778 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.794883+0000", 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046566159000000003 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.794937+0000", 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4434000000000001e-05 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.798650+0000", 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037130890000000001 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.779 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148280 7.c 7:36520c72:::benchmark_data_smithi027_271924_object31939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:27.658741+0000", 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 4.9794147320000004, 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15069249100000001, 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148280 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:32.780 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658741+0000", 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658741+0000", 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658740+0000", 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:32.781 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658746+0000", 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3890000000000002e-06 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.665801+0000", 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070547589999999999 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:32.782 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.665817+0000", 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6188000000000002e-05 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748347+0000", 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082530021999999995 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748449+0000", 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010179999999999999 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.783 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748561+0000", 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111557 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763383+0000", 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014822343 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763501+0000", 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117484 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.784 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.805791+0000", 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042290217999999997 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.805847+0000", 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5822000000000002e-05 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.809434+0000", 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035872529999999999 2024-08-29T15:31:32.785 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.786 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.786 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.786 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:32.786 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:32.786 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:31:32.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:32.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:32.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:32.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:32.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:33.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:33.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:33.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:33.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:33.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:33.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:33.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:33.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:33.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:33.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:33.451 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:33.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:33.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:33.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:33 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-29T15:31:33.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:31:33.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:33 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-29T15:31:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:33 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-29T15:31:33.579 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:33.604 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:33.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:33.706 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:33.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:33.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:33.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:33.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:33.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:33.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:33.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:33.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:31:33.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:34.008 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:34.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:34.110 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:34.138 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:34.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:34.234 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:34.240 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:34.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:34.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:34.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:34.416 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:34.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:34.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:34.543 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:34 smithi027 ceph-mon[98970]: pgmap v1130: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.99k op/s; 7440/248778 objects degraded (2.991%); 32850/248778 objects misplaced (13.205%); 19 MiB/s, 313 objects/s recovering 2024-08-29T15:31:34.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:34 smithi027 ceph-mon[93921]: pgmap v1130: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.99k op/s; 7440/248778 objects degraded (2.991%); 32850/248778 objects misplaced (13.205%); 19 MiB/s, 313 objects/s recovering 2024-08-29T15:31:34.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:34.649 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:34 smithi171 ceph-mon[90031]: pgmap v1130: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 1 active+remapped+backfilling, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.99k op/s; 7440/248778 objects degraded (2.991%); 32850/248778 objects misplaced (13.205%); 19 MiB/s, 313 objects/s recovering 2024-08-29T15:31:34.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:34.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:34.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:34.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:34.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:34.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:34.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:35.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:35.105 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:35.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:35.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:35.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:35.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:35.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:35.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:35.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:35.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:35.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:35.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:35.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:35.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:35 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 7440/245505 objects degraded (3.030%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:35.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:35 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 7440/245505 objects degraded (3.030%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:35.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:35.627 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:35 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 7440/245505 objects degraded (3.030%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:35.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:35.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:35.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:35.773 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:35.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:35.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:35.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:35.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:35.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:35.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:36.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:36.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:36.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:36.100 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:31:36.100 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:31:36.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.36273.0:152970 7.f e315/311 7:f11748b3:::benchmark_data_smithi027_271924_object36629:head v 315'9435, mlcod=298'2254)", 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:36.094854+0000", 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0039198599999999998, 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039448019999999999, 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094854+0000", 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:36.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094854+0000", 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094853+0000", 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094861+0000", 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1219999999999995e-06 2024-08-29T15:31:36.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094876+0000", 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5160000000000001e-05 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094882+0000", 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8590000000000001e-06 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094940+0000", 2024-08-29T15:31:36.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8576999999999998e-05 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:36.094967+0000", 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6063000000000001e-05 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 1 2024-08-29T15:31:36.104 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:31:36.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:36.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:36.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:36.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:36.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:36.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:36.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:36.424 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:36 smithi027 ceph-mon[98970]: pgmap v1131: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.84k op/s; 7440/245505 objects degraded (3.030%); 31809/245505 objects misplaced (12.957%); 21 MiB/s, 337 objects/s recovering 2024-08-29T15:31:36.425 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:36 smithi027 ceph-mon[93921]: pgmap v1131: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.84k op/s; 7440/245505 objects degraded (3.030%); 31809/245505 objects misplaced (12.957%); 21 MiB/s, 337 objects/s recovering 2024-08-29T15:31:36.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:36.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:36.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:36.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:36.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:36.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:36.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:36.654 INFO:teuthology.orchestra.run.smithi027.stdout:moved 0 / 267 (0%) 2024-08-29T15:31:36.654 INFO:teuthology.orchestra.run.smithi027.stdout:avg 33.375 2024-08-29T15:31:36.654 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 7.63115 -> 7.63115 (expected baseline 5.40399) 2024-08-29T15:31:36.654 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.0 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2024-08-29T15:31:36.654 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.1 with 47 -> 47 pgs (1.40824 -> 1.40824 * mean) 2024-08-29T15:31:36.654 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:31:36.655 INFO:teuthology.orchestra.run.smithi027.stdout:oload 1000 2024-08-29T15:31:36.655 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 1 2024-08-29T15:31:36.655 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:31:36.655 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0255 2024-08-29T15:31:36.655 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.2546 2024-08-29T15:31:36.655 INFO:teuthology.orchestra.run.smithi027.stderr:no change 2024-08-29T15:31:36.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:36.727 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:36.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:36 smithi171 ceph-mon[90031]: pgmap v1131: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.84k op/s; 7440/245505 objects degraded (3.030%); 31809/245505 objects misplaced (12.957%); 21 MiB/s, 337 objects/s recovering 2024-08-29T15:31:36.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:31:36.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:36.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:36.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:36.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:37.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:37.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:37.134 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:37.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:37.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:37.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:37.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:37.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:37.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:37 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/539623806' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:37.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:37.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:37.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:37.617 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:37 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/539623806' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:37.617 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:37 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/539623806' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:37.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:37.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:37.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:37.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:37.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:31:37.850 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:31:37.851 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:31:37.851 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:31:37.851 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:31:37.851 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:31:37.851 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:31:37.851 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:31:37.851 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:31:37.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:37.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:37.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:38.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:38.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:38.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:38.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:38.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:38.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:38.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:38.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:38.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:38.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:38.474 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:38.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:38.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:38 smithi027 ceph-mon[98970]: from='client.46313 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:38 smithi027 ceph-mon[98970]: pgmap v1132: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 1.61k op/s; 7440/242940 objects degraded (3.062%); 31624/242940 objects misplaced (13.017%); 17 MiB/s, 276 objects/s recovering 2024-08-29T15:31:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:38 smithi027 ceph-mon[93921]: from='client.46313 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:38 smithi027 ceph-mon[93921]: pgmap v1132: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 1.61k op/s; 7440/242940 objects degraded (3.062%); 31624/242940 objects misplaced (13.017%); 17 MiB/s, 276 objects/s recovering 2024-08-29T15:31:38.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:38.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:38.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:38.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:38.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:38 smithi171 ceph-mon[90031]: from='client.46313 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:31:38.694 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:38 smithi171 ceph-mon[90031]: pgmap v1132: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 1.61k op/s; 7440/242940 objects degraded (3.062%); 31624/242940 objects misplaced (13.017%); 17 MiB/s, 276 objects/s recovering 2024-08-29T15:31:38.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:38.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:38.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:38.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:38.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:38.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:38.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:38.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:38.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:39.033 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:39.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:39.135 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:39.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:39.178 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:31:39.178 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:31:39.236 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:39.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:39.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:39.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:39.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:31:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:31:39] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-08-29T15:31:39.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:39.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:39.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:39.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.742 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:39.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:39.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:39.869 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:39.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:39.971 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:39.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:39.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:40.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:40.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:40.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:40.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:40.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:40.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:40.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:40.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:40.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:40.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:40.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:40.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:40.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:40.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:40.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:31:40.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:40.681 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:40 smithi027 ceph-mon[93921]: pgmap v1133: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 1.43k op/s; 7687/240090 objects degraded (3.202%); 31238/240090 objects misplaced (13.011%); 5.2 MiB/s, 86 objects/s recovering 2024-08-29T15:31:40.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:40 smithi027 ceph-mon[98970]: pgmap v1133: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 1.43k op/s; 7687/240090 objects degraded (3.202%); 31238/240090 objects misplaced (13.011%); 5.2 MiB/s, 86 objects/s recovering 2024-08-29T15:31:40.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:40.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:40 smithi171 ceph-mon[90031]: pgmap v1133: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 1.43k op/s; 7687/240090 objects degraded (3.202%); 31238/240090 objects misplaced (13.011%); 5.2 MiB/s, 86 objects/s recovering 2024-08-29T15:31:40.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:40.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:31:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:40.912 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:40.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:41.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:41.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:41.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:41.180 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:41.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:41.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:41.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:41.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:41.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:41.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:41.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:41.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:41 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 7687/235428 objects degraded (3.265%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:41.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:41 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 7687/235428 objects degraded (3.265%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:41.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:41.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:41.604 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:41 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 7687/235428 objects degraded (3.265%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:41.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:41.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:41.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:41.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:41.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:41.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:41.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:41.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:42.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:42.079 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:42.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:42.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:42.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:42.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:42.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:42.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:42.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:42.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:42.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:42.436 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:31:42.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:42.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:42.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:42.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:42.626 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:42 smithi171 ceph-mon[90031]: pgmap v1134: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 7687/235428 objects degraded (3.265%); 29906/235428 objects misplaced (12.703%); 8.3 MiB/s, 133 objects/s recovering 2024-08-29T15:31:42.626 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2900019341' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:42.626 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:42 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:42.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:42.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:42.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:42.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:42.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:42.815 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:42.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:42 smithi027 ceph-mon[98970]: pgmap v1134: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 7687/235428 objects degraded (3.265%); 29906/235428 objects misplaced (12.703%); 8.3 MiB/s, 133 objects/s recovering 2024-08-29T15:31:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2900019341' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:42 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:42.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:42 smithi027 ceph-mon[93921]: pgmap v1134: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 7687/235428 objects degraded (3.265%); 29906/235428 objects misplaced (12.703%); 8.3 MiB/s, 133 objects/s recovering 2024-08-29T15:31:42.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2900019341' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:42.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:42 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:31:42.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:42.939 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:42.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:42.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:43.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:43.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:43.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:43.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:43.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:43.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:43.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:43.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:43.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:43.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:43.468 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:43.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:43.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:43.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:43.594 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:43.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:43.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:43.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:43.706 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:43 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:31:43.706 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:31:43.706 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:43 smithi027 ceph-mon[93921]: osdmap e316: 8 total, 8 up, 8 in 2024-08-29T15:31:43.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:43 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:31:43.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:31:43.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:43 smithi027 ceph-mon[98970]: osdmap e316: 8 total, 8 up, 8 in 2024-08-29T15:31:43.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:43.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:43 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:31:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:31:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:43 smithi171 ceph-mon[90031]: osdmap e316: 8 total, 8 up, 8 in 2024-08-29T15:31:43.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:43.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:43.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:43.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:43.967 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:31:43.968 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:31:43.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:44.007 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:44.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:44.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:44.200 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:31:44.200 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:31:44.201 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-29T15:31:44.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:31:44.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:44.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:44.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:44.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:44.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:44.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:44.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:44 smithi027 ceph-mon[93921]: pgmap v1136: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 7863/231687 objects degraded (3.394%); 29583/231687 objects misplaced (12.769%); 8.5 MiB/s, 135 objects/s recovering 2024-08-29T15:31:44.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:44.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:44.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:44.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:44.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:44.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:44.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:44.730 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:44 smithi171 ceph-mon[90031]: pgmap v1136: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 7863/231687 objects degraded (3.394%); 29583/231687 objects misplaced (12.769%); 8.5 MiB/s, 135 objects/s recovering 2024-08-29T15:31:44.730 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:44.730 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:44.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:31:44.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:44 smithi027 ceph-mon[98970]: pgmap v1136: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 7863/231687 objects degraded (3.394%); 29583/231687 objects misplaced (12.769%); 8.5 MiB/s, 135 objects/s recovering 2024-08-29T15:31:44.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:31:44.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:44.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:44.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:44.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:44.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:44.999 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:44.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:45.055 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:45.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:45.138 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:45.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:45.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:45.275 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:45.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:45.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:45.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:45.414 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:45.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:45.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:45.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:45.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:45.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:45.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:45.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:45.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:45.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:45 smithi171 ceph-mon[90031]: pgmap v1137: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 7863/228498 objects degraded (3.441%); 28581/228498 objects misplaced (12.508%); 8.6 MiB/s, 137 objects/s recovering 2024-08-29T15:31:45.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:45.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:45 smithi027 ceph-mon[98970]: pgmap v1137: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 7863/228498 objects degraded (3.441%); 28581/228498 objects misplaced (12.508%); 8.6 MiB/s, 137 objects/s recovering 2024-08-29T15:31:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:45 smithi027 ceph-mon[93921]: pgmap v1137: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 7863/228498 objects degraded (3.441%); 28581/228498 objects misplaced (12.508%); 8.6 MiB/s, 137 objects/s recovering 2024-08-29T15:31:45.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:45.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:45.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:45.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:45.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:45.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:45.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:46.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:46.083 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:46.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:46.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:46.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:46.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:46.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:46.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:46.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:46.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:46.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:46.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:46.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:46 smithi027 ceph-mon[98970]: osdmap e317: 8 total, 8 up, 8 in 2024-08-29T15:31:46.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:46.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:46.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:46.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:46.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:46.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:46.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:46.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:46 smithi171 ceph-mon[90031]: osdmap e317: 8 total, 8 up, 8 in 2024-08-29T15:31:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:46 smithi027 ceph-mon[93921]: osdmap e317: 8 total, 8 up, 8 in 2024-08-29T15:31:46.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:46.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:46.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:46.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:46.959 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:46.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:47.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:47.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:47.231 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:47.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:47.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:47.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:47.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:47.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:47.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:47.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:47.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:47.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:47.732 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:47 smithi171 ceph-mon[90031]: osdmap e318: 8 total, 8 up, 8 in 2024-08-29T15:31:47.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:47 smithi171 ceph-mon[90031]: pgmap v1140: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 7863/226110 objects degraded (3.478%); 28395/226110 objects misplaced (12.558%); 6.8 MiB/s, 109 objects/s recovering 2024-08-29T15:31:47.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 7863/228498 objects degraded (3.441%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:47.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:47.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:47.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:47 smithi027 ceph-mon[98970]: osdmap e318: 8 total, 8 up, 8 in 2024-08-29T15:31:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:47 smithi027 ceph-mon[98970]: pgmap v1140: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 7863/226110 objects degraded (3.478%); 28395/226110 objects misplaced (12.558%); 6.8 MiB/s, 109 objects/s recovering 2024-08-29T15:31:47.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 7863/228498 objects degraded (3.441%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:47.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:47 smithi027 ceph-mon[93921]: osdmap e318: 8 total, 8 up, 8 in 2024-08-29T15:31:47.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:47 smithi027 ceph-mon[93921]: pgmap v1140: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 7863/226110 objects degraded (3.478%); 28395/226110 objects misplaced (12.558%); 6.8 MiB/s, 109 objects/s recovering 2024-08-29T15:31:47.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 7863/228498 objects degraded (3.441%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:47.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:47.920 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:47.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:48.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:48.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:48.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:48.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:48.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:48.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:48.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:48.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:48.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:48.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:48.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:48.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:48.450 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:48.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:48.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:48.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:48.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:48.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:48.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:48.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:48.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:48.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:48.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:48.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:48.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:48.889 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:48.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:48.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:48.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:49.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:49.120 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:49.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.147 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:49.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:49.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:49.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:31:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:31:49] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-29T15:31:49.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:49.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:49.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:49.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:49.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:49.593 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:49.622 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:31:49.622 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:31:49.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:49.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:49.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:49.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:49.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:49.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:49.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:49 smithi027 ceph-mon[98970]: pgmap v1141: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 7973/223833 objects degraded (3.562%); 28068/223833 objects misplaced (12.540%); 7.8 MiB/s, 125 objects/s recovering 2024-08-29T15:31:49.935 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:49 smithi027 ceph-mon[93921]: pgmap v1141: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 7973/223833 objects degraded (3.562%); 28068/223833 objects misplaced (12.540%); 7.8 MiB/s, 125 objects/s recovering 2024-08-29T15:31:49.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:49.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:49.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:49.977 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:49 smithi171 ceph-mon[90031]: pgmap v1141: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 7973/223833 objects degraded (3.562%); 28068/223833 objects misplaced (12.540%); 7.8 MiB/s, 125 objects/s recovering 2024-08-29T15:31:50.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:50.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:50.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:50.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:50.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:50.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:50.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:50.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:50.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:31:50.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:50.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:50.447 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:50.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:50.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:50.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:50.590 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:50.627 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:50.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:50.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:50.639 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:50.729 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:50.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:50.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:50.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:50.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:50.896 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:31:50.896 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:31:50.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:50.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:50.938 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:31:50.938 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:31:50.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:31:50.993 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:31:50.993 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:31:51.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:51.050 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:31:51.051 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:31:51.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:31:51.086 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:31:51.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:51.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:51.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:31:51.119 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:31:51.154 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:31:51.154 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:31:51.201 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:31:51.201 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:31:51.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:51.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:51.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:51.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:51.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:51.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:51.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:51.462 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:51.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:51.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:51.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:51.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:51.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:51.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:51.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:51.787 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:51.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:51.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:51.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:51.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:51.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:51.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:52.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:51 smithi027 ceph-mon[93921]: pgmap v1142: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 7973/219747 objects degraded (3.628%); 25036/219747 objects misplaced (11.393%); 25 MiB/s, 399 objects/s recovering 2024-08-29T15:31:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3780732746' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:51 smithi027 ceph-mon[98970]: pgmap v1142: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 7973/219747 objects degraded (3.628%); 25036/219747 objects misplaced (11.393%); 25 MiB/s, 399 objects/s recovering 2024-08-29T15:31:52.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3780732746' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:52.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:52.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:52.101 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:51 smithi171 ceph-mon[90031]: pgmap v1142: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 7973/219747 objects degraded (3.628%); 25036/219747 objects misplaced (11.393%); 25 MiB/s, 399 objects/s recovering 2024-08-29T15:31:52.101 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3780732746' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:31:52.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:52.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:52.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:52.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:52.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:52.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:52.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:52.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:52.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:52.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:52.618 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:31:52.618 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:31:52.618 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:31:52.618 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:31:52.618 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:31:52.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:52.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:52.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:52.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:52.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:52.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:52.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:52.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:52.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:52.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:53.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:53.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:53.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:53.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:53.186 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 7973/219747 objects degraded (3.628%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:53.187 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 7973/219747 objects degraded (3.628%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:53.240 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 7973/219747 objects degraded (3.628%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:53.292 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:53.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:53.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:53.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:31:53.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:53.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:53.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:53.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:53.455 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:53.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:53.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:53.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:53.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:53.607 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:31:53.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:53.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:53.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:53.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:53.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:53.876 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:31:53.877 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:31:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:53.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:53.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:53.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:54.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:53 smithi171 ceph-mon[90031]: pgmap v1143: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 8134/216015 objects degraded (3.765%); 24722/216015 objects misplaced (11.445%); 22 MiB/s, 359 objects/s recovering 2024-08-29T15:31:54.020 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:53 smithi027 ceph-mon[98970]: pgmap v1143: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 8134/216015 objects degraded (3.765%); 24722/216015 objects misplaced (11.445%); 22 MiB/s, 359 objects/s recovering 2024-08-29T15:31:54.020 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:53 smithi027 ceph-mon[93921]: pgmap v1143: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 8134/216015 objects degraded (3.765%); 24722/216015 objects misplaced (11.445%); 22 MiB/s, 359 objects/s recovering 2024-08-29T15:31:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:54.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:54.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:54.123 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:54.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:54.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:54.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:54.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:54.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:54.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:54.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:54.426 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:54.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:54.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:54.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:54.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:54.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:54.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:54.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:54.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:54.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:54.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:54.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:54.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:54.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:31:54.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:55.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:55.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:55.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:55.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:54 smithi171 ceph-mon[90031]: from='client.56101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:31:55.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:55.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:55.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:54 smithi027 ceph-mon[93921]: from='client.56101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:31:55.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:54 smithi027 ceph-mon[98970]: from='client.56101 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:31:55.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:55.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:55.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:55.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:55.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:55.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:55.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:55.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:31:55.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:55.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:31:55.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:55.984 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:55.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:56.086 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:56.115 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:56.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:56.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:56.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:56.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:55 smithi171 ceph-mon[90031]: pgmap v1144: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 8134/212829 objects degraded (3.822%); 22625/212829 objects misplaced (10.631%); 30 MiB/s, 480 objects/s recovering 2024-08-29T15:31:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:55 smithi027 ceph-mon[98970]: pgmap v1144: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 8134/212829 objects degraded (3.822%); 22625/212829 objects misplaced (10.631%); 30 MiB/s, 480 objects/s recovering 2024-08-29T15:31:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:55 smithi027 ceph-mon[93921]: pgmap v1144: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 8134/212829 objects degraded (3.822%); 22625/212829 objects misplaced (10.631%); 30 MiB/s, 480 objects/s recovering 2024-08-29T15:31:56.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:56.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:56.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:56.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:56.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:56.521 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:56.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:56.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:56.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:56.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:56.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:56.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:56.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:56.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:56.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:56.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:57.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:57.078 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:57.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:57.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:57.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:57.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:57.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:57.367 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:57.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:57.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:57.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:57.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:57.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:57.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:57.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:57.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:57.766 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:57.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:57.867 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:31:57.902 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:57.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:31:58.004 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:57 smithi171 ceph-mon[90031]: pgmap v1145: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 8134/210231 objects degraded (3.869%); 21879/210231 objects misplaced (10.407%); 31 MiB/s, 495 objects/s recovering 2024-08-29T15:31:58.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 8134/210231 objects degraded (3.869%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:58.035 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:58.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:58.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:58.137 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:58.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:58.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:58.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:58.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:31:58.304 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:58.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:58.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:31:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:57 smithi027 ceph-mon[98970]: pgmap v1145: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 8134/210231 objects degraded (3.869%); 21879/210231 objects misplaced (10.407%); 31 MiB/s, 495 objects/s recovering 2024-08-29T15:31:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 8134/210231 objects degraded (3.869%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:57 smithi027 ceph-mon[93921]: pgmap v1145: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 8134/210231 objects degraded (3.869%); 21879/210231 objects misplaced (10.407%); 31 MiB/s, 495 objects/s recovering 2024-08-29T15:31:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 8134/210231 objects degraded (3.869%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:31:58.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:58.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:58.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:58.495 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:31:58.495 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:31:58.495 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:31:58.495 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:31:58.495 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:240733 6.7 6:ee8bf952:::benchmark_data_smithi027_169445_object115306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:14.072526+0000", 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 224.41811845199999, 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066965819999999995, 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 240733 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.496 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072526+0000", 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072526+0000", 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072525+0000", 2024-08-29T15:31:58.497 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072531+0000", 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5799999999999999e-06 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072541+0000", 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0453e-05 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.072548+0000", 2024-08-29T15:31:58.498 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4400000000000002e-06 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.104570+0000", 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032022238000000001 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.104758+0000", 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018819900000000001 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.499 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.104904+0000", 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014563600000000001 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.118915+0000", 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014011288 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.119079+0000", 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000163528 2024-08-29T15:31:58.500 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.139379+0000", 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020299762999999998 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.139461+0000", 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2114999999999996e-05 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:14.139492+0000", 2024-08-29T15:31:58.501 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.154e-05 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:243473 6.7 6:edff5081:::benchmark_data_smithi027_169445_object118046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 219.62311561800001, 2024-08-29T15:31:58.502 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062441970999999999, 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 243473 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.503 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867529+0000", 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867532+0000", 2024-08-29T15:31:58.504 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1389999999999999e-06 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867539+0000", 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9530000000000002e-06 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.867544+0000", 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5389999999999998e-06 2024-08-29T15:31:58.505 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.871915+0000", 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043704920000000001 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.872038+0000", 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000123638 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.872149+0000", 2024-08-29T15:31:58.506 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000110525 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.877166+0000", 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050165799999999997 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.878416+0000", 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001250082 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.507 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929870+0000", 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051454518999999997 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929919+0000", 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9165000000000001e-05 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:28:18.929971+0000", 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1830999999999999e-05 2024-08-29T15:31:58.508 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:84709 7.0 7:055127a7:::benchmark_data_smithi027_271924_object84708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:07.154645+0000", 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 111.335999633, 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072125712999999994, 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.509 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 84709 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.154645+0000", 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.154645+0000", 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.510 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.154644+0000", 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.154691+0000", 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6882000000000003e-05 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.154700+0000", 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9930000000000001e-06 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.511 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.154769+0000", 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8853e-05 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.154984+0000", 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021580699999999999 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.155126+0000", 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014138099999999999 2024-08-29T15:31:58.512 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.155277+0000", 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000151337 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.158772+0000", 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003494507 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.158869+0000", 2024-08-29T15:31:58.513 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6940000000000004e-05 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.226695+0000", 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067826866 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.226741+0000", 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5748000000000003e-05 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.514 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.226771+0000", 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9697999999999999e-05 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:84718 7.0 7:085c3b9a:::benchmark_data_smithi027_271924_object84717:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:07.163431+0000", 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 111.327214107, 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067692593999999995, 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.515 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 84718 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163431+0000", 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163431+0000", 2024-08-29T15:31:58.516 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163429+0000", 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163497+0000", 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8232000000000007e-05 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.517 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163517+0000", 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0375999999999999e-05 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163527+0000", 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9329999999999996e-06 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163572+0000", 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4184999999999998e-05 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.518 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163674+0000", 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000101984 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.163781+0000", 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000107009 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.165911+0000", 2024-08-29T15:31:58.519 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021304560000000002 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.166230+0000", 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00031941000000000001 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.231029+0000", 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.064798897999999994 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.520 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.231084+0000", 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4208e-05 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:07.231123+0000", 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9782e-05 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:104856 7.0 7:02355ccf:::benchmark_data_smithi027_271924_object104855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:27.829831+0000", 2024-08-29T15:31:58.521 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 90.660813391000005, 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.121693916, 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 104856 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.829831+0000", 2024-08-29T15:31:58.522 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.829831+0000", 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.829830+0000", 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.523 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.524 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.829937+0000", 2024-08-29T15:31:58.524 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000107218 2024-08-29T15:31:58.524 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.829949+0000", 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1187e-05 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.829954+0000", 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2909999999999998e-06 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.829987+0000", 2024-08-29T15:31:58.525 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3099999999999998e-05 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.830105+0000", 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117905 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.830259+0000", 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015367899999999999 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.526 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.833272+0000", 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030133489999999998 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.833379+0000", 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000107046 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.951440+0000", 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.118061332 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.527 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.951488+0000", 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7868000000000003e-05 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:27.951525+0000", 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7251e-05 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:105271 7.0 7:0ff7bc74:::benchmark_data_smithi027_271924_object105270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:58.528 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:28.349574+0000", 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 90.141070384000002, 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.073062641999999997, 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 105271 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.349574+0000", 2024-08-29T15:31:58.529 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.349574+0000", 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.349574+0000", 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.530 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.349606+0000", 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2092999999999999e-05 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.349613+0000", 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.413e-06 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.349618+0000", 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1309999999999997e-06 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.531 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.353055+0000", 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034364819999999998 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.353208+0000", 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000152989 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.353381+0000", 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017272500000000001 2024-08-29T15:31:58.532 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.356290+0000", 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029095409999999999 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.356310+0000", 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9731000000000001e-05 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.422569+0000", 2024-08-29T15:31:58.533 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066259389000000002 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.422609+0000", 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9873000000000003e-05 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:28.422637+0000", 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7917000000000001e-05 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.534 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:106260 7.0 7:0f42b86f:::benchmark_data_smithi027_271924_object106259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:29.573314+0000", 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 88.917330348999997, 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17217152899999999, 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 106260 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.535 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573314+0000", 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573314+0000", 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573313+0000", 2024-08-29T15:31:58.536 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573369+0000", 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6079999999999998e-05 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573384+0000", 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5588000000000001e-05 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.537 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573392+0000", 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5440000000000001e-06 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573436+0000", 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3860999999999999e-05 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573578+0000", 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000141862 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.538 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.573779+0000", 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000201926 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.576580+0000", 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028003920000000001 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.582602+0000", 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060219059999999996 2024-08-29T15:31:58.539 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.745407+0000", 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16280504200000001 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.745444+0000", 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7277999999999998e-05 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.540 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.745486+0000", 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1955000000000001e-05 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110062 7.0 7:0907c5f0:::benchmark_data_smithi027_271924_object110061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:34.612538+0000", 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.878106857999995, 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26528133100000001, 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.541 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110062 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612538+0000", 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612538+0000", 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.542 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612528+0000", 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.99999 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612596+0000", 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8072999999999997e-05 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.543 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612618+0000", 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2056e-05 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612626+0000", 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1950000000000003e-06 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612679+0000", 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2617999999999997e-05 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.544 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612805+0000", 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012649800000000001 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.612943+0000", 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013800300000000001 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.616394+0000", 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003450884 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.545 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.618687+0000", 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002292765 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.877717+0000", 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25903025200000002 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.877770+0000", 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3279999999999998e-05 2024-08-29T15:31:58.546 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.877819+0000", 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8878000000000002e-05 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110077 7.0 7:0d10ab45:::benchmark_data_smithi027_271924_object110076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:34.626969+0000", 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.863676256999995, 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25100664900000003, 2024-08-29T15:31:58.547 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110077 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.626969+0000", 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.548 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.626969+0000", 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.626967+0000", 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.627017+0000", 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0275000000000002e-05 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.549 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.627033+0000", 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5409000000000001e-05 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.627039+0000", 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2319999999999998e-06 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.627079+0000", 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9471999999999997e-05 2024-08-29T15:31:58.550 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.627171+0000", 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2044000000000002e-05 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.627267+0000", 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5946999999999995e-05 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.551 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.629998+0000", 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027313559999999999 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.631159+0000", 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0011613699999999999 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.877887+0000", 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.24672767300000001 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.552 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.877937+0000", 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0518e-05 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.877975+0000", 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7764e-05 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.36273.0:110167 7.b e303/294 7:db4f3e81:::benchmark_data_smithi027_271924_object110166:head v 303'6925, mlcod=303'6923)", 2024-08-29T15:31:58.553 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:35.029355+0000", 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.461290274999996, 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.129876255, 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029355+0000", 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029355+0000", 2024-08-29T15:31:58.554 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029354+0000", 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029439+0000", 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4999000000000004e-05 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.555 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029464+0000", 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5329000000000001e-05 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029474+0000", 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0481e-05 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029516+0000", 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1445e-05 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.556 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.029538+0000", 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2586999999999999e-05 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159189+0000", 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12965033100000001 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159231+0000", 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2091000000000001e-05 2024-08-29T15:31:58.557 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110170 7.0 7:028d1809:::benchmark_data_smithi027_271924_object110169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:35.031180+0000", 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.459464988999997, 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13424767100000001, 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.558 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110170 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031180+0000", 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031180+0000", 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.559 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031178+0000", 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031224+0000", 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5705999999999999e-05 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031235+0000", 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1443e-05 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.560 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031241+0000", 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3380000000000004e-06 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031286+0000", 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5262000000000001e-05 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031413+0000", 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012702199999999999 2024-08-29T15:31:58.561 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.031535+0000", 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012208400000000001 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165331+0000", 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.133795475 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165365+0000", 2024-08-29T15:31:58.562 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4096e-05 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165372+0000", 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6359999999999994e-06 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165403+0000", 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0366e-05 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.563 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165428+0000", 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4879999999999999e-05 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110173 7.0 7:0bc47068:::benchmark_data_smithi027_271924_object110172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:35.035410+0000", 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.455234731999994, 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13711548600000001, 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.564 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110173 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035410+0000", 2024-08-29T15:31:58.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:58.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035410+0000", 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035408+0000", 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035465+0000", 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6889999999999999e-05 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.158802+0000", 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12333701800000001 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.158819+0000", 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6718000000000002e-05 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.161913+0000", 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030939840000000001 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.162050+0000", 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136668 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.162196+0000", 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014592699999999999 2024-08-29T15:31:58.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.172373+0000", 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010177465 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.172381+0000", 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1179999999999992e-06 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.172440+0000", 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8900000000000002e-05 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.172489+0000", 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9401999999999999e-05 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.172526+0000", 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6146999999999998e-05 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.36273.0:110170 7.0 e303/294 ondisk, result = 0)", 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:35.035750+0000", 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.454895256, 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12968253299999999, 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035750+0000", 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035750+0000", 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035749+0000", 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.035755+0000", 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9909999999999999e-06 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159366+0000", 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12361114199999999 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159372+0000", 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9859999999999999e-06 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165358+0000", 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00598614 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165362+0000", 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7780000000000001e-06 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.165432+0000", 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0320000000000004e-05 2024-08-29T15:31:58.575 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.36273.0:110191 7.e e303/294 7:7a501db4:::benchmark_data_smithi027_271924_object110190:head v 303'6775, mlcod=303'6773)", 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:35.053583+0000", 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.437061400999994, 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.113565132, 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.053583+0000", 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.053583+0000", 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.053582+0000", 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.053625+0000", 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3130000000000002e-05 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159464+0000", 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10583888900000001 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159469+0000", 2024-08-29T15:31:58.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.874e-06 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159501+0000", 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1853999999999997e-05 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.159516+0000", 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4656e-05 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.579 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.167111+0000", 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075945600000000002 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:35.167149+0000", 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8031000000000002e-05 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:151588 7.0 7:0ae37d63:::benchmark_data_smithi027_271924_object35247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-08-29T15:31:58.580 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:33.655270+0000", 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 24.835374438999999, 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.116068083, 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 151588 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655270+0000", 2024-08-29T15:31:58.581 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655270+0000", 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655269+0000", 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.582 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655277+0000", 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6430000000000001e-06 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655294+0000", 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6595e-05 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655308+0000", 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.396e-05 2024-08-29T15:31:58.583 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655348+0000", 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9616000000000002e-05 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655479+0000", 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000131786 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.655569+0000", 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9853999999999997e-05 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.659157+0000", 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035876520000000002 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.734991+0000", 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.075833796999999994 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.771249+0000", 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036258258000000002 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.771297+0000", 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8495000000000002e-05 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.771338+0000", 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1053999999999997e-05 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.586 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:153659 7.0 7:02026f02:::benchmark_data_smithi027_271924_object37318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:37.324393+0000", 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 21.166251425999999, 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10636656799999999, 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 153659 2024-08-29T15:31:58.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324393+0000", 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324393+0000", 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324392+0000", 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324398+0000", 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1260000000000003e-06 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324411+0000", 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3241000000000001e-05 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324419+0000", 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8550000000000002e-06 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324463+0000", 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3954e-05 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324564+0000", 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000101192 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.324645+0000", 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0729000000000006e-05 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.329376+0000", 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0047313169999999996 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.336370+0000", 2024-08-29T15:31:58.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0069939950000000002 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.430697+0000", 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.094326525999999994 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.430733+0000", 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6149000000000002e-05 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.430760+0000", 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6923999999999998e-05 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:153663 7.0 7:02a0e120:::benchmark_data_smithi027_271924_object37322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:37.330039+0000", 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 21.160605716999999, 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100811951, 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.593 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 153663 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330039+0000", 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330039+0000", 2024-08-29T15:31:58.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330037+0000", 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330044+0000", 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2259999999999999e-06 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330057+0000", 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.257e-05 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330065+0000", 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.729e-06 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330095+0000", 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0685999999999997e-05 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330189+0000", 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3762000000000001e-05 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.330267+0000", 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7860000000000003e-05 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.333828+0000", 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035613569999999998 2024-08-29T15:31:58.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.336608+0000", 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027791399999999998 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.430795+0000", 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.094187147999999998 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.430826+0000", 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1139000000000001e-05 2024-08-29T15:31:58.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:37.430851+0000", 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5236999999999999e-05 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:163459 7.0 7:0fd6f904:::benchmark_data_smithi027_271924_object47118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:55.556139+0000", 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.934506238, 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.097764342000000004, 2024-08-29T15:31:58.599 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 163459 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.556139+0000", 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.556139+0000", 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.556137+0000", 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.556145+0000", 2024-08-29T15:31:58.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7999999999999999e-06 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.556159+0000", 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4052e-05 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.556166+0000", 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3490000000000003e-06 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.557225+0000", 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001058706 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.557335+0000", 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000110517 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.557439+0000", 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000103807 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.562463+0000", 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0050241219999999998 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.651768+0000", 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.089304474999999994 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.653801+0000", 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020332010000000001 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.653857+0000", 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6166999999999999e-05 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.653903+0000", 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5961000000000003e-05 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:164846 7.0 7:0eae711b:::benchmark_data_smithi027_271924_object48505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:58.148173+0000", 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.34247192199999998, 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072687647999999994, 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 164846 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.606 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148173+0000", 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148173+0000", 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148171+0000", 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148178+0000", 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9280000000000004e-06 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148192+0000", 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4008999999999999e-05 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148199+0000", 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0140000000000001e-06 2024-08-29T15:31:58.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148237+0000", 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7499000000000001e-05 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148364+0000", 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012778299999999999 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.148474+0000", 2024-08-29T15:31:58.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000109867 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.153296+0000", 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004821278 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.165161+0000", 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011865904 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.220766+0000", 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.055604383 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.220819+0000", 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3449999999999998e-05 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.612 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.220861+0000", 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1223000000000002e-05 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:164853 7.0 7:08931efc:::benchmark_data_smithi027_271924_object48512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:58.160021+0000", 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.33062335999999998, 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.061995726000000001, 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:31:58.613 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 164853 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160021+0000", 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160021+0000", 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:31:58.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160019+0000", 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160029+0000", 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.414e-06 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:31:58.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160051+0000", 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2623999999999999e-05 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160060+0000", 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7609999999999996e-06 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160130+0000", 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9581999999999999e-05 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.616 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160283+0000", 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015317999999999999 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.160420+0000", 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013726999999999999 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.165212+0000", 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0047922720000000002 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.176523+0000", 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011311026 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.221939+0000", 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.045415469 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.221984+0000", 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4648999999999999e-05 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:31:58.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:58.222017+0000", 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3593999999999997e-05 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:31:58.619 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:31:58.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:58.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:58.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:58.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:58.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:58.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:58.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:31:58.919 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1143,"stamp":"2024-08-29T15:31:52.763951+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4721809405,"num_objects":72005,"num_object_clones":0,"num_object_copies":216015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8134,"num_objects_misplaced":24722,"num_objects_unfound":0,"num_objects_dirty":72005,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":161824,"num_write_kb":7460674,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8686,"num_bytes_recovered":568401929,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141459,"ondisk_log_size":141459,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":277,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17551840,"kb_used_data":14557280,"kb_used_omap":43,"kb_used_meta":2994452,"kb_avail":732409376,"statfs":{"total":767960285184,"available":749987201024,"internally_reserved":0,"allocated":14906654720,"data_stored":14891393414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":3066319340},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,3,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-272695296,"num_objects":-4161,"num_object_clones":0,"num_object_copies":-12483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":271,"num_objects_misplaced":-3859,"num_objects_unfound":0,"num_objects_dirty":-4161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4161,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2878,"num_bytes_recovered":188612608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002171"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":523,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.468499+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:31:46.468499+0000","last_peered":"2024-08-29T15:31:46.468499+0000","last_clean":"2024-08-29T15:31:46.468499+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:31:46.468499+0000","last_undegraded":"2024-08-29T15:31:46.468499+0000","last_fullsized":"2024-08-29T15:31:46.468499+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":107,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471647+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:31:45.471647+0000","last_peered":"2024-08-29T15:31:45.471647+0000","last_clean":"2024-08-29T15:31:45.471647+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:31:45.471647+0000","last_undegraded":"2024-08-29T15:31:45.471647+0000","last_fullsized":"2024-08-29T15:31:45.471647+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":188,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.464077+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:31:45.464077+0000","last_peered":"2024-08-29T15:31:45.464077+0000","last_clean":"2024-08-29T15:31:45.464077+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:31:45.464077+0000","last_undegraded":"2024-08-29T15:31:45.464077+0000","last_fullsized":"2024-08-29T15:31:45.464077+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":497,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491387+0000","last_change":"2024-08-29T15:31:26.199672+0000","last_active":"2024-08-29T15:31:46.491387+0000","last_peered":"2024-08-29T15:31:46.491387+0000","last_clean":"2024-08-29T15:31:46.491387+0000","last_became_active":"2024-08-29T15:31:26.199057+0000","last_became_peered":"2024-08-29T15:31:26.199057+0000","last_unstale":"2024-08-29T15:31:46.491387+0000","last_undegraded":"2024-08-29T15:31:46.491387+0000","last_fullsized":"2024-08-29T15:31:46.491387+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:21:31.006988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":490,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.492009+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:31:46.492009+0000","last_peered":"2024-08-29T15:31:46.492009+0000","last_clean":"2024-08-29T15:31:46.492009+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:31:46.492009+0000","last_undegraded":"2024-08-29T15:31:46.492009+0000","last_fullsized":"2024-08-29T15:31:46.492009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":100,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471638+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:31:45.471638+0000","last_peered":"2024-08-29T15:31:45.471638+0000","last_clean":"2024-08-29T15:31:45.471638+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:31:45.471638+0000","last_undegraded":"2024-08-29T15:31:45.471638+0000","last_fullsized":"2024-08-29T15:31:45.471638+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":308,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.471146+0000","last_change":"2024-08-29T15:31:26.189733+0000","last_active":"2024-08-29T15:31:46.471146+0000","last_peered":"2024-08-29T15:31:46.471146+0000","last_clean":"2024-08-29T15:31:46.471146+0000","last_became_active":"2024-08-29T15:31:26.189414+0000","last_became_peered":"2024-08-29T15:31:26.189414+0000","last_unstale":"2024-08-29T15:31:46.471146+0000","last_undegraded":"2024-08-29T15:31:46.471146+0000","last_fullsized":"2024-08-29T15:31:46.471146+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:12:46.739048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":209,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.473781+0000","last_change":"2024-08-29T15:31:26.202033+0000","last_active":"2024-08-29T15:31:45.473781+0000","last_peered":"2024-08-29T15:31:45.473781+0000","last_clean":"2024-08-29T15:31:45.473781+0000","last_became_active":"2024-08-29T15:31:26.192847+0000","last_became_peered":"2024-08-29T15:31:26.192847+0000","last_unstale":"2024-08-29T15:31:45.473781+0000","last_undegraded":"2024-08-29T15:31:45.473781+0000","last_fullsized":"2024-08-29T15:31:45.473781+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:14:14.891828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"318'10183","reported_seq":10243,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:50.908764+0000","last_change":"2024-08-29T15:28:57.922512+0000","last_active":"2024-08-29T15:31:50.908764+0000","last_peered":"2024-08-29T15:31:50.908764+0000","last_clean":"2024-08-29T15:31:50.908764+0000","last_became_active":"2024-08-29T15:28:57.922093+0000","last_became_peered":"2024-08-29T15:28:57.922093+0000","last_unstale":"2024-08-29T15:31:50.908764+0000","last_undegraded":"2024-08-29T15:31:50.908764+0000","last_fullsized":"2024-08-29T15:31:50.908764+0000","mapping_epoch":294,"log_start":"299'3844","ondisk_log_start":"299'3844","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":6339,"log_dups_size":0,"ondisk_log_size":6339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:26.218110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291307520,"num_objects":4445,"num_object_clones":0,"num_object_copies":13335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10183,"num_write_kb":468096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":117,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.472004+0000","last_change":"2024-08-29T15:31:26.215108+0000","last_active":"2024-08-29T15:31:45.472004+0000","last_peered":"2024-08-29T15:31:45.472004+0000","last_clean":"2024-08-29T15:31:45.472004+0000","last_became_active":"2024-08-29T15:31:26.197355+0000","last_became_peered":"2024-08-29T15:31:26.197355+0000","last_unstale":"2024-08-29T15:31:45.472004+0000","last_undegraded":"2024-08-29T15:31:45.472004+0000","last_fullsized":"2024-08-29T15:31:45.472004+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:10:37.515092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":207,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.463652+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:31:45.463652+0000","last_peered":"2024-08-29T15:31:45.463652+0000","last_clean":"2024-08-29T15:31:45.463652+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:31:45.463652+0000","last_undegraded":"2024-08-29T15:31:45.463652+0000","last_fullsized":"2024-08-29T15:31:45.463652+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"318'10199","reported_seq":10259,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:52.045140+0000","last_change":"2024-08-29T15:28:57.922760+0000","last_active":"2024-08-29T15:31:52.045140+0000","last_peered":"2024-08-29T15:31:52.045140+0000","last_clean":"2024-08-29T15:31:52.045140+0000","last_became_active":"2024-08-29T15:28:57.922490+0000","last_became_peered":"2024-08-29T15:28:57.922490+0000","last_unstale":"2024-08-29T15:31:52.045140+0000","last_undegraded":"2024-08-29T15:31:52.045140+0000","last_fullsized":"2024-08-29T15:31:52.045140+0000","mapping_epoch":294,"log_start":"297'1100","ondisk_log_start":"297'1100","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9099,"log_dups_size":0,"ondisk_log_size":9099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:46:47.703614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286851072,"num_objects":4377,"num_object_clones":0,"num_object_copies":13131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10199,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":104,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.472480+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:31:45.472480+0000","last_peered":"2024-08-29T15:31:45.472480+0000","last_clean":"2024-08-29T15:31:45.472480+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:31:45.472480+0000","last_undegraded":"2024-08-29T15:31:45.472480+0000","last_fullsized":"2024-08-29T15:31:45.472480+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":109,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471402+0000","last_change":"2024-08-29T15:31:26.195631+0000","last_active":"2024-08-29T15:31:45.471402+0000","last_peered":"2024-08-29T15:31:45.471402+0000","last_clean":"2024-08-29T15:31:45.471402+0000","last_became_active":"2024-08-29T15:31:26.195293+0000","last_became_peered":"2024-08-29T15:31:26.195293+0000","last_unstale":"2024-08-29T15:31:45.471402+0000","last_undegraded":"2024-08-29T15:31:45.471402+0000","last_fullsized":"2024-08-29T15:31:45.471402+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:52:28.638575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"318'9775","reported_seq":9835,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:49.330763+0000","last_change":"2024-08-29T15:28:57.926748+0000","last_active":"2024-08-29T15:31:49.330763+0000","last_peered":"2024-08-29T15:31:49.330763+0000","last_clean":"2024-08-29T15:31:49.330763+0000","last_became_active":"2024-08-29T15:28:57.926391+0000","last_became_peered":"2024-08-29T15:28:57.926391+0000","last_unstale":"2024-08-29T15:31:49.330763+0000","last_undegraded":"2024-08-29T15:31:49.330763+0000","last_fullsized":"2024-08-29T15:31:49.330763+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9775,"log_dups_size":0,"ondisk_log_size":9775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:30:16.060709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295239680,"num_objects":4505,"num_object_clones":0,"num_object_copies":13515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9775,"num_write_kb":456960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":306,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491814+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:31:46.491814+0000","last_peered":"2024-08-29T15:31:46.491814+0000","last_clean":"2024-08-29T15:31:46.491814+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:31:46.491814+0000","last_undegraded":"2024-08-29T15:31:46.491814+0000","last_fullsized":"2024-08-29T15:31:46.491814+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":145,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.473910+0000","last_change":"2024-08-29T15:16:59.033264+0000","last_active":"2024-08-29T15:31:45.473910+0000","last_peered":"2024-08-29T15:31:45.473910+0000","last_clean":"2024-08-29T15:31:45.473910+0000","last_became_active":"2024-08-29T15:16:59.032518+0000","last_became_peered":"2024-08-29T15:16:59.032518+0000","last_unstale":"2024-08-29T15:31:45.473910+0000","last_undegraded":"2024-08-29T15:31:45.473910+0000","last_fullsized":"2024-08-29T15:31:45.473910+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:35.122502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"318'10224","reported_seq":10258,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:49.188933+0000","last_change":"2024-08-29T15:28:58.126108+0000","last_active":"2024-08-29T15:31:49.188933+0000","last_peered":"2024-08-29T15:31:49.188933+0000","last_clean":"2024-08-29T15:31:49.188933+0000","last_became_active":"2024-08-29T15:28:58.125784+0000","last_became_peered":"2024-08-29T15:28:58.125784+0000","last_unstale":"2024-08-29T15:31:49.188933+0000","last_undegraded":"2024-08-29T15:31:49.188933+0000","last_fullsized":"2024-08-29T15:31:49.188933+0000","mapping_epoch":294,"log_start":"298'2885","ondisk_log_start":"298'2885","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":7339,"log_dups_size":0,"ondisk_log_size":7339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:15:58.792157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301203456,"num_objects":4596,"num_object_clones":0,"num_object_copies":13788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10224,"num_write_kb":474240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":207,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.463947+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:31:45.463947+0000","last_peered":"2024-08-29T15:31:45.463947+0000","last_clean":"2024-08-29T15:31:45.463947+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:31:45.463947+0000","last_undegraded":"2024-08-29T15:31:45.463947+0000","last_fullsized":"2024-08-29T15:31:45.463947+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":504,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491609+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:31:46.491609+0000","last_peered":"2024-08-29T15:31:46.491609+0000","last_clean":"2024-08-29T15:31:46.491609+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:31:46.491609+0000","last_undegraded":"2024-08-29T15:31:46.491609+0000","last_fullsized":"2024-08-29T15:31:46.491609+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"318'9941","reported_seq":10814,"reported_epoch":318,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:31:52.080421+0000","last_change":"2024-08-29T15:31:27.723493+0000","last_active":"2024-08-29T15:31:52.080421+0000","last_peered":"2024-08-29T15:31:52.080421+0000","last_clean":"2024-08-29T15:31:25.035300+0000","last_became_active":"2024-08-29T15:31:27.546384+0000","last_became_peered":"2024-08-29T15:31:27.546384+0000","last_unstale":"2024-08-29T15:31:52.080421+0000","last_undegraded":"2024-08-29T15:31:27.529900+0000","last_fullsized":"2024-08-29T15:31:27.182235+0000","mapping_epoch":311,"log_start":"297'800","ondisk_log_start":"297'800","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9141,"log_dups_size":0,"ondisk_log_size":9141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289603584,"num_objects":4419,"num_object_clones":0,"num_object_copies":13257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":384,"num_objects_misplaced":4419,"num_objects_unfound":0,"num_objects_dirty":4419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9941,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":404,"num_bytes_recovered":26279936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":4419},{"shards":"3,7","objects":-384}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":509,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.468325+0000","last_change":"2024-08-29T15:31:26.213123+0000","last_active":"2024-08-29T15:31:46.468325+0000","last_peered":"2024-08-29T15:31:46.468325+0000","last_clean":"2024-08-29T15:31:46.468325+0000","last_became_active":"2024-08-29T15:31:26.196584+0000","last_became_peered":"2024-08-29T15:31:26.196584+0000","last_unstale":"2024-08-29T15:31:46.468325+0000","last_undegraded":"2024-08-29T15:31:46.468325+0000","last_fullsized":"2024-08-29T15:31:46.468325+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:51:23.571768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.472215+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:31:45.472215+0000","last_peered":"2024-08-29T15:31:45.472215+0000","last_clean":"2024-08-29T15:31:45.472215+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:31:45.472215+0000","last_undegraded":"2024-08-29T15:31:45.472215+0000","last_fullsized":"2024-08-29T15:31:45.472215+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"318'9954","reported_seq":10790,"reported_epoch":318,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:31:52.719756+0000","last_change":"2024-08-29T15:31:27.673874+0000","last_active":"2024-08-29T15:31:52.719756+0000","last_peered":"2024-08-29T15:31:52.719756+0000","last_clean":"2024-08-29T15:31:25.089216+0000","last_became_active":"2024-08-29T15:31:27.631816+0000","last_became_peered":"2024-08-29T15:31:27.631816+0000","last_unstale":"2024-08-29T15:31:52.719756+0000","last_undegraded":"2024-08-29T15:31:27.238701+0000","last_fullsized":"2024-08-29T15:31:27.187311+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9954,"log_dups_size":0,"ondisk_log_size":9954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285605888,"num_objects":4358,"num_object_clones":0,"num_object_copies":13074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":452,"num_objects_misplaced":4357,"num_objects_unfound":0,"num_objects_dirty":4358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9954,"num_write_kb":457984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":386,"num_bytes_recovered":25296896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":4357},{"shards":"2,5","objects":-451}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":163,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.473826+0000","last_change":"2024-08-29T15:19:46.807606+0000","last_active":"2024-08-29T15:31:45.473826+0000","last_peered":"2024-08-29T15:31:45.473826+0000","last_clean":"2024-08-29T15:31:45.473826+0000","last_became_active":"2024-08-29T15:19:46.807181+0000","last_became_peered":"2024-08-29T15:19:46.807181+0000","last_unstale":"2024-08-29T15:31:45.473826+0000","last_undegraded":"2024-08-29T15:31:45.473826+0000","last_fullsized":"2024-08-29T15:31:45.473826+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:21:54.628978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":100,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471268+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:31:45.471268+0000","last_peered":"2024-08-29T15:31:45.471268+0000","last_clean":"2024-08-29T15:31:45.471268+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:31:45.471268+0000","last_undegraded":"2024-08-29T15:31:45.471268+0000","last_fullsized":"2024-08-29T15:31:45.471268+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"318'10019","reported_seq":10054,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:50.007190+0000","last_change":"2024-08-29T15:28:57.926441+0000","last_active":"2024-08-29T15:31:50.007190+0000","last_peered":"2024-08-29T15:31:50.007190+0000","last_clean":"2024-08-29T15:31:50.007190+0000","last_became_active":"2024-08-29T15:28:57.926173+0000","last_became_peered":"2024-08-29T15:28:57.926173+0000","last_unstale":"2024-08-29T15:31:50.007190+0000","last_undegraded":"2024-08-29T15:31:50.007190+0000","last_fullsized":"2024-08-29T15:31:50.007190+0000","mapping_epoch":294,"log_start":"297'1412","ondisk_log_start":"297'1412","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":8607,"log_dups_size":0,"ondisk_log_size":8607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:43.091728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294387736,"num_objects":4493,"num_object_clones":0,"num_object_copies":13479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4493,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10019,"num_write_kb":464321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":100,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471485+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:31:45.471485+0000","last_peered":"2024-08-29T15:31:45.471485+0000","last_clean":"2024-08-29T15:31:45.471485+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:31:45.471485+0000","last_undegraded":"2024-08-29T15:31:45.471485+0000","last_fullsized":"2024-08-29T15:31:45.471485+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":306,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.472239+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:31:45.472239+0000","last_peered":"2024-08-29T15:31:45.472239+0000","last_clean":"2024-08-29T15:31:45.472239+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:31:45.472239+0000","last_undegraded":"2024-08-29T15:31:45.472239+0000","last_fullsized":"2024-08-29T15:31:45.472239+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"318'10025","reported_seq":10078,"reported_epoch":318,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:31:49.935080+0000","last_change":"2024-08-29T15:31:27.703062+0000","last_active":"2024-08-29T15:31:49.935080+0000","last_peered":"2024-08-29T15:31:49.935080+0000","last_clean":"2024-08-29T15:31:24.997719+0000","last_became_active":"2024-08-29T15:31:27.640040+0000","last_became_peered":"2024-08-29T15:31:27.640040+0000","last_unstale":"2024-08-29T15:31:49.935080+0000","last_undegraded":"2024-08-29T15:31:27.347018+0000","last_fullsized":"2024-08-29T15:31:27.183404+0000","mapping_epoch":311,"log_start":"297'1399","ondisk_log_start":"297'1399","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":8626,"log_dups_size":0,"ondisk_log_size":8626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:10:33.501013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7298,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10025,"num_write_kb":467456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":4583},{"shards":"6,7","objects":-755}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470472+0000","last_change":"2024-08-29T15:31:26.192931+0000","last_active":"2024-08-29T15:31:46.470472+0000","last_peered":"2024-08-29T15:31:46.470472+0000","last_clean":"2024-08-29T15:31:46.470472+0000","last_became_active":"2024-08-29T15:31:26.191104+0000","last_became_peered":"2024-08-29T15:31:26.191104+0000","last_unstale":"2024-08-29T15:31:46.470472+0000","last_undegraded":"2024-08-29T15:31:46.470472+0000","last_fullsized":"2024-08-29T15:31:46.470472+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:06:43.150576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":494,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.471164+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:31:46.471164+0000","last_peered":"2024-08-29T15:31:46.471164+0000","last_clean":"2024-08-29T15:31:46.471164+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:31:46.471164+0000","last_undegraded":"2024-08-29T15:31:46.471164+0000","last_fullsized":"2024-08-29T15:31:46.471164+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"318'10005","reported_seq":10204,"reported_epoch":318,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:31:49.327627+0000","last_change":"2024-08-29T15:31:27.673917+0000","last_active":"2024-08-29T15:31:49.327627+0000","last_peered":"2024-08-29T15:31:49.327627+0000","last_clean":"2024-08-29T15:31:25.045495+0000","last_became_active":"2024-08-29T15:31:27.344646+0000","last_became_peered":"2024-08-29T15:31:27.344646+0000","last_unstale":"2024-08-29T15:31:49.327627+0000","last_undegraded":"2024-08-29T15:31:49.327627+0000","last_fullsized":"2024-08-29T15:31:49.327627+0000","mapping_epoch":311,"log_start":"297'100","ondisk_log_start":"297'100","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9905,"log_dups_size":0,"ondisk_log_size":9905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4562,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10005,"num_write_kb":467968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":4619}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":594,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491299+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:31:46.491299+0000","last_peered":"2024-08-29T15:31:46.491299+0000","last_clean":"2024-08-29T15:31:46.491299+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:31:46.491299+0000","last_undegraded":"2024-08-29T15:31:46.491299+0000","last_fullsized":"2024-08-29T15:31:46.491299+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:04:06.642650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":791,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.463841+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:31:45.463841+0000","last_peered":"2024-08-29T15:31:45.463841+0000","last_clean":"2024-08-29T15:31:45.463841+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:31:45.463841+0000","last_undegraded":"2024-08-29T15:31:45.463841+0000","last_fullsized":"2024-08-29T15:31:45.463841+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":308,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.463729+0000","last_change":"2024-08-29T15:31:26.189487+0000","last_active":"2024-08-29T15:31:45.463729+0000","last_peered":"2024-08-29T15:31:45.463729+0000","last_clean":"2024-08-29T15:31:45.463729+0000","last_became_active":"2024-08-29T15:31:26.188972+0000","last_became_peered":"2024-08-29T15:31:26.188972+0000","last_unstale":"2024-08-29T15:31:45.463729+0000","last_undegraded":"2024-08-29T15:31:45.463729+0000","last_fullsized":"2024-08-29T15:31:45.463729+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:46:31.878072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":162,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.474003+0000","last_change":"2024-08-29T15:19:46.808175+0000","last_active":"2024-08-29T15:31:45.474003+0000","last_peered":"2024-08-29T15:31:45.474003+0000","last_clean":"2024-08-29T15:31:45.474003+0000","last_became_active":"2024-08-29T15:19:46.807085+0000","last_became_peered":"2024-08-29T15:19:46.807085+0000","last_unstale":"2024-08-29T15:31:45.474003+0000","last_undegraded":"2024-08-29T15:31:45.474003+0000","last_fullsized":"2024-08-29T15:31:45.474003+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:24.589124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"318'10043","reported_seq":10106,"reported_epoch":318,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:31:49.437018+0000","last_change":"2024-08-29T15:31:27.475678+0000","last_active":"2024-08-29T15:31:49.437018+0000","last_peered":"2024-08-29T15:31:49.437018+0000","last_clean":"2024-08-29T15:31:25.054089+0000","last_became_active":"2024-08-29T15:31:27.447747+0000","last_became_peered":"2024-08-29T15:31:27.447747+0000","last_unstale":"2024-08-29T15:31:49.437018+0000","last_undegraded":"2024-08-29T15:31:49.437018+0000","last_fullsized":"2024-08-29T15:31:49.437018+0000","mapping_epoch":311,"log_start":"297'200","ondisk_log_start":"297'200","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9843,"log_dups_size":0,"ondisk_log_size":9843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303890432,"num_objects":4637,"num_object_clones":0,"num_object_copies":13911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4637,"num_objects_unfound":0,"num_objects_dirty":4637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10043,"num_write_kb":469760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":4637}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":967,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470407+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:31:46.470407+0000","last_peered":"2024-08-29T15:31:46.470407+0000","last_clean":"2024-08-29T15:31:46.470407+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:31:46.470407+0000","last_undegraded":"2024-08-29T15:31:46.470407+0000","last_fullsized":"2024-08-29T15:31:46.470407+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":512,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491338+0000","last_change":"2024-08-29T15:31:26.193377+0000","last_active":"2024-08-29T15:31:46.491338+0000","last_peered":"2024-08-29T15:31:46.491338+0000","last_clean":"2024-08-29T15:31:46.491338+0000","last_became_active":"2024-08-29T15:31:26.193034+0000","last_became_peered":"2024-08-29T15:31:26.193034+0000","last_unstale":"2024-08-29T15:31:46.491338+0000","last_undegraded":"2024-08-29T15:31:46.491338+0000","last_fullsized":"2024-08-29T15:31:46.491338+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:51.414695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":324,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.471201+0000","last_change":"2024-08-29T15:31:26.189633+0000","last_active":"2024-08-29T15:31:46.471201+0000","last_peered":"2024-08-29T15:31:46.471201+0000","last_clean":"2024-08-29T15:31:46.471201+0000","last_became_active":"2024-08-29T15:31:26.189312+0000","last_became_peered":"2024-08-29T15:31:26.189312+0000","last_unstale":"2024-08-29T15:31:46.471201+0000","last_undegraded":"2024-08-29T15:31:46.471201+0000","last_fullsized":"2024-08-29T15:31:46.471201+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:53:49.593868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"318'9977","reported_seq":12953,"reported_epoch":318,"state":"active+remapped+backfilling","last_fresh":"2024-08-29T15:31:49.231916+0000","last_change":"2024-08-29T15:31:44.806560+0000","last_active":"2024-08-29T15:31:49.231916+0000","last_peered":"2024-08-29T15:31:49.231916+0000","last_clean":"2024-08-29T15:31:25.002456+0000","last_became_active":"2024-08-29T15:31:27.459770+0000","last_became_peered":"2024-08-29T15:31:27.459770+0000","last_unstale":"2024-08-29T15:31:49.231916+0000","last_undegraded":"2024-08-29T15:31:49.231916+0000","last_fullsized":"2024-08-29T15:31:49.231916+0000","mapping_epoch":311,"log_start":"298'2652","ondisk_log_start":"298'2652","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:26:49.921369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296026112,"num_objects":4517,"num_object_clones":0,"num_object_copies":13551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3079,"num_objects_unfound":0,"num_objects_dirty":4517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9977,"num_write_kb":463808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1458,"num_bytes_recovered":95551488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":4517}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":531,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491475+0000","last_change":"2024-08-29T15:31:29.300646+0000","last_active":"2024-08-29T15:31:46.491475+0000","last_peered":"2024-08-29T15:31:46.491475+0000","last_clean":"2024-08-29T15:31:46.491475+0000","last_became_active":"2024-08-29T15:31:29.296994+0000","last_became_peered":"2024-08-29T15:31:29.296994+0000","last_unstale":"2024-08-29T15:31:46.491475+0000","last_undegraded":"2024-08-29T15:31:46.491475+0000","last_fullsized":"2024-08-29T15:31:46.491475+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:04:42.548079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":152,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.473610+0000","last_change":"2024-08-29T15:18:34.987357+0000","last_active":"2024-08-29T15:31:45.473610+0000","last_peered":"2024-08-29T15:31:45.473610+0000","last_clean":"2024-08-29T15:31:45.473610+0000","last_became_active":"2024-08-29T15:18:34.987045+0000","last_became_peered":"2024-08-29T15:18:34.987045+0000","last_unstale":"2024-08-29T15:31:45.473610+0000","last_undegraded":"2024-08-29T15:31:45.473610+0000","last_fullsized":"2024-08-29T15:31:45.473610+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:34.204621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":540,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.471064+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:31:46.471064+0000","last_peered":"2024-08-29T15:31:46.471064+0000","last_clean":"2024-08-29T15:31:46.471064+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:31:46.471064+0000","last_undegraded":"2024-08-29T15:31:46.471064+0000","last_fullsized":"2024-08-29T15:31:46.471064+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"318'10201","reported_seq":12194,"reported_epoch":318,"state":"active+remapped+backfilling","last_fresh":"2024-08-29T15:31:50.075685+0000","last_change":"2024-08-29T15:31:44.812689+0000","last_active":"2024-08-29T15:31:50.075685+0000","last_peered":"2024-08-29T15:31:50.075685+0000","last_clean":"2024-08-29T15:31:25.062237+0000","last_became_active":"2024-08-29T15:31:27.541472+0000","last_became_peered":"2024-08-29T15:31:27.541472+0000","last_unstale":"2024-08-29T15:31:50.075685+0000","last_undegraded":"2024-08-29T15:31:50.075685+0000","last_fullsized":"2024-08-29T15:31:50.075685+0000","mapping_epoch":311,"log_start":"297'1612","ondisk_log_start":"297'1612","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":8589,"log_dups_size":0,"ondisk_log_size":8589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:28.885904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303497216,"num_objects":4631,"num_object_clones":0,"num_object_copies":13893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3668,"num_objects_unfound":0,"num_objects_dirty":4631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10201,"num_write_kb":474624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":974,"num_bytes_recovered":63832064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,3],"avail_no_missing":["6","0","3"],"object_location_counts":[{"shards":"0,3,6","objects":4631}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":259,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.463627+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:31:45.463627+0000","last_peered":"2024-08-29T15:31:45.463627+0000","last_clean":"2024-08-29T15:31:45.463627+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:31:45.463627+0000","last_undegraded":"2024-08-29T15:31:45.463627+0000","last_fullsized":"2024-08-29T15:31:45.463627+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":100,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471559+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:31:45.471559+0000","last_peered":"2024-08-29T15:31:45.471559+0000","last_clean":"2024-08-29T15:31:45.471559+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:31:45.471559+0000","last_undegraded":"2024-08-29T15:31:45.471559+0000","last_fullsized":"2024-08-29T15:31:45.471559+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":166,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.463824+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:31:45.463824+0000","last_peered":"2024-08-29T15:31:45.463824+0000","last_clean":"2024-08-29T15:31:45.463824+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:31:45.463824+0000","last_undegraded":"2024-08-29T15:31:45.463824+0000","last_fullsized":"2024-08-29T15:31:45.463824+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.1","version":"318'9933","reported_seq":20465,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:49.207544+0000","last_change":"2024-08-29T15:31:46.490013+0000","last_active":"2024-08-29T15:31:49.207544+0000","last_peered":"2024-08-29T15:31:49.207544+0000","last_clean":"2024-08-29T15:31:49.207544+0000","last_became_active":"2024-08-29T15:31:46.471674+0000","last_became_peered":"2024-08-29T15:31:46.471674+0000","last_unstale":"2024-08-29T15:31:49.207544+0000","last_undegraded":"2024-08-29T15:31:49.207544+0000","last_fullsized":"2024-08-29T15:31:49.207544+0000","mapping_epoch":317,"log_start":"298'2520","ondisk_log_start":"298'2520","created":294,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":7413,"log_dups_size":0,"ondisk_log_size":7413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:32:03.986269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291962880,"num_objects":4455,"num_object_clones":0,"num_object_copies":13365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9933,"num_write_kb":460416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5235,"num_bytes_recovered":343080960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":191,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.464042+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:31:45.464042+0000","last_peered":"2024-08-29T15:31:45.464042+0000","last_clean":"2024-08-29T15:31:45.464042+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:31:45.464042+0000","last_undegraded":"2024-08-29T15:31:45.464042+0000","last_fullsized":"2024-08-29T15:31:45.464042+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":932,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470838+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:31:46.470838+0000","last_peered":"2024-08-29T15:31:46.470838+0000","last_clean":"2024-08-29T15:31:46.470838+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:31:46.470838+0000","last_undegraded":"2024-08-29T15:31:46.470838+0000","last_fullsized":"2024-08-29T15:31:46.470838+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":509,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470650+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:31:46.470650+0000","last_peered":"2024-08-29T15:31:46.470650+0000","last_clean":"2024-08-29T15:31:46.470650+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:31:46.470650+0000","last_undegraded":"2024-08-29T15:31:46.470650+0000","last_fullsized":"2024-08-29T15:31:46.470650+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"318'10008","reported_seq":10042,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:51.370269+0000","last_change":"2024-08-29T15:28:57.922386+0000","last_active":"2024-08-29T15:31:51.370269+0000","last_peered":"2024-08-29T15:31:51.370269+0000","last_clean":"2024-08-29T15:31:51.370269+0000","last_became_active":"2024-08-29T15:28:57.922067+0000","last_became_peered":"2024-08-29T15:28:57.922067+0000","last_unstale":"2024-08-29T15:31:51.370269+0000","last_undegraded":"2024-08-29T15:31:51.370269+0000","last_fullsized":"2024-08-29T15:31:51.370269+0000","mapping_epoch":294,"log_start":"0'0","ondisk_log_start":"0'0","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:11:27.444379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289800192,"num_objects":4422,"num_object_clones":0,"num_object_copies":13266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10008,"num_write_kb":461760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":514,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470689+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:31:46.470689+0000","last_peered":"2024-08-29T15:31:46.470689+0000","last_clean":"2024-08-29T15:31:46.470689+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:31:46.470689+0000","last_undegraded":"2024-08-29T15:31:46.470689+0000","last_fullsized":"2024-08-29T15:31:46.470689+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":771,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.476003+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:31:45.476003+0000","last_peered":"2024-08-29T15:31:45.476003+0000","last_clean":"2024-08-29T15:31:45.476003+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:31:45.476003+0000","last_undegraded":"2024-08-29T15:31:45.476003+0000","last_fullsized":"2024-08-29T15:31:45.476003+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":505,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491528+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:31:46.491528+0000","last_peered":"2024-08-29T15:31:46.491528+0000","last_clean":"2024-08-29T15:31:46.491528+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:31:46.491528+0000","last_undegraded":"2024-08-29T15:31:46.491528+0000","last_fullsized":"2024-08-29T15:31:46.491528+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"318'10255","reported_seq":10315,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:52.075875+0000","last_change":"2024-08-29T15:28:57.924025+0000","last_active":"2024-08-29T15:31:52.075875+0000","last_peered":"2024-08-29T15:31:52.075875+0000","last_clean":"2024-08-29T15:31:52.075875+0000","last_became_active":"2024-08-29T15:28:57.923719+0000","last_became_peered":"2024-08-29T15:28:57.923719+0000","last_unstale":"2024-08-29T15:31:52.075875+0000","last_undegraded":"2024-08-29T15:31:52.075875+0000","last_fullsized":"2024-08-29T15:31:52.075875+0000","mapping_epoch":294,"log_start":"297'1100","ondisk_log_start":"297'1100","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9155,"log_dups_size":0,"ondisk_log_size":9155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:46.870950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10255,"num_write_kb":473408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":498,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.475410+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:31:46.475410+0000","last_peered":"2024-08-29T15:31:46.475410+0000","last_clean":"2024-08-29T15:31:46.475410+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:31:46.475410+0000","last_undegraded":"2024-08-29T15:31:46.475410+0000","last_fullsized":"2024-08-29T15:31:46.475410+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1151,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.471293+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:31:46.471293+0000","last_peered":"2024-08-29T15:31:46.471293+0000","last_clean":"2024-08-29T15:31:46.471293+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:31:46.471293+0000","last_undegraded":"2024-08-29T15:31:46.471293+0000","last_fullsized":"2024-08-29T15:31:46.471293+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":26,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470230+0000","last_change":"2024-08-29T15:31:26.189321+0000","last_active":"2024-08-29T15:31:46.470230+0000","last_peered":"2024-08-29T15:31:46.470230+0000","last_clean":"2024-08-29T15:31:46.470230+0000","last_became_active":"2024-08-29T15:31:26.188778+0000","last_became_peered":"2024-08-29T15:31:26.188778+0000","last_unstale":"2024-08-29T15:31:46.470230+0000","last_undegraded":"2024-08-29T15:31:46.470230+0000","last_fullsized":"2024-08-29T15:31:46.470230+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:11:57.644178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.2","version":"318'9959","reported_seq":10019,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:52.077595+0000","last_change":"2024-08-29T15:28:57.923922+0000","last_active":"2024-08-29T15:31:52.077595+0000","last_peered":"2024-08-29T15:31:52.077595+0000","last_clean":"2024-08-29T15:31:52.077595+0000","last_became_active":"2024-08-29T15:28:57.923593+0000","last_became_peered":"2024-08-29T15:28:57.923593+0000","last_unstale":"2024-08-29T15:31:52.077595+0000","last_undegraded":"2024-08-29T15:31:52.077595+0000","last_fullsized":"2024-08-29T15:31:52.077595+0000","mapping_epoch":294,"log_start":"297'800","ondisk_log_start":"297'800","created":294,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:28:56.906146+0000","last_clean_scrub_stamp":"2024-08-29T15:28:56.906146+0000","objects_scrubbed":0,"log_size":9159,"log_dups_size":0,"ondisk_log_size":9159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:48:59.077875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287113216,"num_objects":4381,"num_object_clones":0,"num_object_copies":13143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9959,"num_write_kb":458880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":52,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470960+0000","last_change":"2024-08-29T15:31:26.211099+0000","last_active":"2024-08-29T15:31:46.470960+0000","last_peered":"2024-08-29T15:31:46.470960+0000","last_clean":"2024-08-29T15:31:46.470960+0000","last_became_active":"2024-08-29T15:31:26.196998+0000","last_became_peered":"2024-08-29T15:31:26.196998+0000","last_unstale":"2024-08-29T15:31:46.470960+0000","last_undegraded":"2024-08-29T15:31:46.470960+0000","last_fullsized":"2024-08-29T15:31:46.470960+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:07:19.391161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":172,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.474070+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:31:45.474070+0000","last_peered":"2024-08-29T15:31:45.474070+0000","last_clean":"2024-08-29T15:31:45.474070+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:31:45.474070+0000","last_undegraded":"2024-08-29T15:31:45.474070+0000","last_fullsized":"2024-08-29T15:31:45.474070+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":100,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471560+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:31:45.471560+0000","last_peered":"2024-08-29T15:31:45.471560+0000","last_clean":"2024-08-29T15:31:45.471560+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:31:45.471560+0000","last_undegraded":"2024-08-29T15:31:45.471560+0000","last_fullsized":"2024-08-29T15:31:45.471560+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471729+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:31:45.471729+0000","last_peered":"2024-08-29T15:31:45.471729+0000","last_clean":"2024-08-29T15:31:45.471729+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:31:45.471729+0000","last_undegraded":"2024-08-29T15:31:45.471729+0000","last_fullsized":"2024-08-29T15:31:45.471729+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":110,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.472302+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:31:45.472302+0000","last_peered":"2024-08-29T15:31:45.472302+0000","last_clean":"2024-08-29T15:31:45.472302+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:31:45.472302+0000","last_undegraded":"2024-08-29T15:31:45.472302+0000","last_fullsized":"2024-08-29T15:31:45.472302+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":503,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470537+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:31:46.470537+0000","last_peered":"2024-08-29T15:31:46.470537+0000","last_clean":"2024-08-29T15:31:46.470537+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:31:46.470537+0000","last_undegraded":"2024-08-29T15:31:46.470537+0000","last_fullsized":"2024-08-29T15:31:46.470537+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":261,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.469247+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:31:46.469247+0000","last_peered":"2024-08-29T15:31:46.469247+0000","last_clean":"2024-08-29T15:31:46.469247+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:31:46.469247+0000","last_undegraded":"2024-08-29T15:31:46.469247+0000","last_fullsized":"2024-08-29T15:31:46.469247+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":188,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.468247+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:31:45.468247+0000","last_peered":"2024-08-29T15:31:45.468247+0000","last_clean":"2024-08-29T15:31:45.468247+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:31:45.468247+0000","last_undegraded":"2024-08-29T15:31:45.468247+0000","last_fullsized":"2024-08-29T15:31:45.468247+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":461,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.467933+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:31:46.467933+0000","last_peered":"2024-08-29T15:31:46.467933+0000","last_clean":"2024-08-29T15:31:46.467933+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:31:46.467933+0000","last_undegraded":"2024-08-29T15:31:46.467933+0000","last_fullsized":"2024-08-29T15:31:46.467933+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":516,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.491943+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:31:46.491943+0000","last_peered":"2024-08-29T15:31:46.491943+0000","last_clean":"2024-08-29T15:31:46.491943+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:31:46.491943+0000","last_undegraded":"2024-08-29T15:31:46.491943+0000","last_fullsized":"2024-08-29T15:31:46.491943+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":513,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.467835+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:31:46.467835+0000","last_peered":"2024-08-29T15:31:46.467835+0000","last_clean":"2024-08-29T15:31:46.467835+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:31:46.467835+0000","last_undegraded":"2024-08-29T15:31:46.467835+0000","last_fullsized":"2024-08-29T15:31:46.467835+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":161,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.474145+0000","last_change":"2024-08-29T15:17:40.739801+0000","last_active":"2024-08-29T15:31:45.474145+0000","last_peered":"2024-08-29T15:31:45.474145+0000","last_clean":"2024-08-29T15:31:45.474145+0000","last_became_active":"2024-08-29T15:17:40.739507+0000","last_became_peered":"2024-08-29T15:17:40.739507+0000","last_unstale":"2024-08-29T15:31:45.474145+0000","last_undegraded":"2024-08-29T15:31:45.474145+0000","last_fullsized":"2024-08-29T15:31:45.474145+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:36:07.069301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":26,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.467710+0000","last_change":"2024-08-29T15:31:26.195212+0000","last_active":"2024-08-29T15:31:46.467710+0000","last_peered":"2024-08-29T15:31:46.467710+0000","last_clean":"2024-08-29T15:31:46.467710+0000","last_became_active":"2024-08-29T15:31:26.194866+0000","last_became_peered":"2024-08-29T15:31:26.194866+0000","last_unstale":"2024-08-29T15:31:46.467710+0000","last_undegraded":"2024-08-29T15:31:46.467710+0000","last_fullsized":"2024-08-29T15:31:46.467710+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:50:45.933437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":499,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.492049+0000","last_change":"2024-08-29T15:31:26.199464+0000","last_active":"2024-08-29T15:31:46.492049+0000","last_peered":"2024-08-29T15:31:46.492049+0000","last_clean":"2024-08-29T15:31:46.492049+0000","last_became_active":"2024-08-29T15:31:26.199059+0000","last_became_peered":"2024-08-29T15:31:26.199059+0000","last_unstale":"2024-08-29T15:31:46.492049+0000","last_undegraded":"2024-08-29T15:31:46.492049+0000","last_fullsized":"2024-08-29T15:31:46.492049+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:48:51.926606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":494,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470600+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:31:46.470600+0000","last_peered":"2024-08-29T15:31:46.470600+0000","last_clean":"2024-08-29T15:31:46.470600+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:31:46.470600+0000","last_undegraded":"2024-08-29T15:31:46.470600+0000","last_fullsized":"2024-08-29T15:31:46.470600+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":498,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.469046+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:31:46.469046+0000","last_peered":"2024-08-29T15:31:46.469046+0000","last_clean":"2024-08-29T15:31:46.469046+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:31:46.469046+0000","last_undegraded":"2024-08-29T15:31:46.469046+0000","last_fullsized":"2024-08-29T15:31:46.469046+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":99,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.470802+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:31:45.470802+0000","last_peered":"2024-08-29T15:31:45.470802+0000","last_clean":"2024-08-29T15:31:45.470802+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:31:45.470802+0000","last_undegraded":"2024-08-29T15:31:45.470802+0000","last_fullsized":"2024-08-29T15:31:45.470802+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":519,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.470878+0000","last_change":"2024-08-29T15:18:34.053256+0000","last_active":"2024-08-29T15:31:46.470878+0000","last_peered":"2024-08-29T15:31:46.470878+0000","last_clean":"2024-08-29T15:31:46.470878+0000","last_became_active":"2024-08-29T15:18:34.052919+0000","last_became_peered":"2024-08-29T15:18:34.052919+0000","last_unstale":"2024-08-29T15:31:46.470878+0000","last_undegraded":"2024-08-29T15:31:46.470878+0000","last_fullsized":"2024-08-29T15:31:46.470878+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:02.034129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":34,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471816+0000","last_change":"2024-08-29T15:31:01.734785+0000","last_active":"2024-08-29T15:31:45.471816+0000","last_peered":"2024-08-29T15:31:45.471816+0000","last_clean":"2024-08-29T15:31:45.471816+0000","last_became_active":"2024-08-29T15:31:01.734449+0000","last_became_peered":"2024-08-29T15:31:01.734449+0000","last_unstale":"2024-08-29T15:31:45.471816+0000","last_undegraded":"2024-08-29T15:31:45.471816+0000","last_fullsized":"2024-08-29T15:31:45.471816+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:18:36.797427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":99,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.471478+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:31:45.471478+0000","last_peered":"2024-08-29T15:31:45.471478+0000","last_clean":"2024-08-29T15:31:45.471478+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:31:45.471478+0000","last_undegraded":"2024-08-29T15:31:45.471478+0000","last_fullsized":"2024-08-29T15:31:45.471478+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":218,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.463950+0000","last_change":"2024-08-29T15:28:35.519733+0000","last_active":"2024-08-29T15:31:45.463950+0000","last_peered":"2024-08-29T15:31:45.463950+0000","last_clean":"2024-08-29T15:31:45.463950+0000","last_became_active":"2024-08-29T15:19:46.804301+0000","last_became_peered":"2024-08-29T15:19:46.804301+0000","last_unstale":"2024-08-29T15:31:45.463950+0000","last_undegraded":"2024-08-29T15:31:45.463950+0000","last_fullsized":"2024-08-29T15:31:45.463950+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:14:02.143420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":99,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.470892+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:31:45.470892+0000","last_peered":"2024-08-29T15:31:45.470892+0000","last_clean":"2024-08-29T15:31:45.470892+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:31:45.470892+0000","last_undegraded":"2024-08-29T15:31:45.470892+0000","last_fullsized":"2024-08-29T15:31:45.470892+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":218,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.464120+0000","last_change":"2024-08-29T15:17:40.719139+0000","last_active":"2024-08-29T15:31:45.464120+0000","last_peered":"2024-08-29T15:31:45.464120+0000","last_clean":"2024-08-29T15:31:45.464120+0000","last_became_active":"2024-08-29T15:17:40.718591+0000","last_became_peered":"2024-08-29T15:17:40.718591+0000","last_unstale":"2024-08-29T15:31:45.464120+0000","last_undegraded":"2024-08-29T15:31:45.464120+0000","last_fullsized":"2024-08-29T15:31:45.464120+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:33:21.321334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":529,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.468083+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:31:46.468083+0000","last_peered":"2024-08-29T15:31:46.468083+0000","last_clean":"2024-08-29T15:31:46.468083+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:31:46.468083+0000","last_undegraded":"2024-08-29T15:31:46.468083+0000","last_fullsized":"2024-08-29T15:31:46.468083+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":531,"reported_epoch":318,"state":"active+clean","last_fresh":"2024-08-29T15:31:46.469764+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:31:46.469764+0000","last_peered":"2024-08-29T15:31:46.469764+0000","last_clean":"2024-08-29T15:31:46.469764+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:31:46.469764+0000","last_undegraded":"2024-08-29T15:31:46.469764+0000","last_fullsized":"2024-08-29T15:31:46.469764+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":99,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.470659+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:31:45.470659+0000","last_peered":"2024-08-29T15:31:45.470659+0000","last_clean":"2024-08-29T15:31:45.470659+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:31:45.470659+0000","last_undegraded":"2024-08-29T15:31:45.470659+0000","last_fullsized":"2024-08-29T15:31:45.470659+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":182,"reported_epoch":317,"state":"active+clean","last_fresh":"2024-08-29T15:31:45.464190+0000","last_change":"2024-08-29T15:31:26.195903+0000","last_active":"2024-08-29T15:31:45.464190+0000","last_peered":"2024-08-29T15:31:45.464190+0000","last_clean":"2024-08-29T15:31:45.464190+0000","last_became_active":"2024-08-29T15:31:26.190997+0000","last_became_peered":"2024-08-29T15:31:26.190997+0000","last_unstale":"2024-08-29T15:31:45.464190+0000","last_undegraded":"2024-08-29T15:31:45.464190+0000","last_fullsized":"2024-08-29T15:31:45.464190+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:22.980377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4717019160,"num_objects":71977,"num_object_clones":0,"num_object_copies":215931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8134,"num_objects_misplaced":24722,"num_objects_unfound":0,"num_objects_dirty":71977,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160701,"num_write_kb":7445633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8528,"num_bytes_recovered":558235648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14768877568,"data_stored":14768865352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140277,"ondisk_log_size":140277,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":6550,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234446,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1919020,"kb_used_data":1482660,"kb_used_omap":1,"kb_used_meta":436350,"kb_avail":91826132,"statfs":{"total":95995035648,"available":94029959168,"internally_reserved":0,"allocated":1518243840,"data_stored":1516323519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":446822859},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496077,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1903980,"kb_used_data":1557860,"kb_used_omap":1,"kb_used_meta":346110,"kb_avail":91841172,"statfs":{"total":95995035648,"available":94045360128,"internally_reserved":0,"allocated":1595248640,"data_stored":1593328595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":354417095},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426664,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292180,"kb_used_data":1138636,"kb_used_omap":8,"kb_used_meta":153527,"kb_avail":92452972,"statfs":{"total":95995035648,"available":94671843328,"internally_reserved":0,"allocated":1165963264,"data_stored":1164062343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":157212554},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019356,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1869732,"kb_used_data":1463708,"kb_used_omap":1,"kb_used_meta":406014,"kb_avail":91875420,"statfs":{"total":95995035648,"available":94080430080,"internally_reserved":0,"allocated":1498836992,"data_stored":1496924046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":415758790},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435541,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3166804,"kb_used_data":2643724,"kb_used_omap":4,"kb_used_meta":523067,"kb_avail":90578348,"statfs":{"total":95995035648,"available":92752228352,"internally_reserved":0,"allocated":2707173376,"data_stored":2705272287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":535620873},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903393,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1992400,"kb_used_data":1712136,"kb_used_omap":10,"kb_used_meta":280245,"kb_avail":91752752,"statfs":{"total":95995035648,"available":93954818048,"internally_reserved":0,"allocated":1753227264,"data_stored":1751329617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":286971408},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,3,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762594,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2529304,"kb_used_data":2122768,"kb_used_omap":7,"kb_used_meta":406520,"kb_avail":91215848,"statfs":{"total":95995035648,"available":93405028352,"internally_reserved":0,"allocated":2173714432,"data_stored":2171808077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":416276834},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926120,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2878420,"kb_used_data":2435788,"kb_used_omap":7,"kb_used_meta":442616,"kb_avail":90866732,"statfs":{"total":95995035648,"available":93047533568,"internally_reserved":0,"allocated":2494246912,"data_stored":2492344930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":453238927},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2478968832,"data_stored":2478964760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2106331136,"data_stored":2106327064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1742340096,"data_stored":1742340096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2692874240,"data_stored":2692874240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1489567744,"data_stored":1489567744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1159856128,"data_stored":1159856128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1592201216,"data_stored":1592197144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1506738176,"data_stored":1506738176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:31:58.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-29T15:31:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:58.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:58.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:31:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:58.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:59.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:59.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:59.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:59.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:59.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:31:59.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:59.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:59.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:31:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:31:59.311 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:31:59] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-29T15:31:59.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:31:59.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:59.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:59.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:31:59.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:59.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:59.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:59.573 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:59.575 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:31:59.617 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:59.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:59.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:59.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:31:59.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:31:59.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:59.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:31:59.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:31:59.891 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:31:59.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:31:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:31:59.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:31:59.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:31:59.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:00.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[93921]: pgmap v1146: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 8308/207732 objects degraded (3.999%); 21549/207732 objects misplaced (10.373%); 27 MiB/s, 437 objects/s recovering 2024-08-29T15:32:00.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[93921]: osdmap e319: 8 total, 8 up, 8 in 2024-08-29T15:32:00.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2540359743' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:32:00.003 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:32:00.004 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[98970]: pgmap v1146: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 8308/207732 objects degraded (3.999%); 21549/207732 objects misplaced (10.373%); 27 MiB/s, 437 objects/s recovering 2024-08-29T15:32:00.004 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[98970]: osdmap e319: 8 total, 8 up, 8 in 2024-08-29T15:32:00.004 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2540359743' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:32:00.004 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:31:59 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:32:00.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:00.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:00.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:00.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:00.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:59 smithi171 ceph-mon[90031]: pgmap v1146: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 8308/207732 objects degraded (3.999%); 21549/207732 objects misplaced (10.373%); 27 MiB/s, 437 objects/s recovering 2024-08-29T15:32:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:59 smithi171 ceph-mon[90031]: osdmap e319: 8 total, 8 up, 8 in 2024-08-29T15:32:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2540359743' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:32:00.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:31:59 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:32:00.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:00.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:00.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:00.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:00.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:00.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:00.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:00.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:00.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:00.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:00.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:00.959 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:00.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:00.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:01.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:01.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:01.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:01.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:01.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:01.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:01.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:01.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:00 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:01.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:00 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:32:01.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:00 smithi171 ceph-mon[90031]: osdmap e320: 8 total, 8 up, 8 in 2024-08-29T15:32:01.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:00 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:00 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:32:01.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:00 smithi027 ceph-mon[98970]: osdmap e320: 8 total, 8 up, 8 in 2024-08-29T15:32:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:00 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:00 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:32:01.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:00 smithi027 ceph-mon[93921]: osdmap e320: 8 total, 8 up, 8 in 2024-08-29T15:32:01.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:01.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:01.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:01.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:01.474 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:01.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:01.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:01.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:01.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:01.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:01.715 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:01.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:01.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:01.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:01.882 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:01.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:01.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:01.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:02.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:02.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:02.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:02.219 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:01 smithi027 ceph-mon[93921]: pgmap v1149: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 8308/203487 objects degraded (4.083%); 18878/203487 objects misplaced (9.277%); 36 MiB/s, 578 objects/s recovering 2024-08-29T15:32:02.219 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:01 smithi027 ceph-mon[98970]: pgmap v1149: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 8308/203487 objects degraded (4.083%); 18878/203487 objects misplaced (9.277%); 36 MiB/s, 578 objects/s recovering 2024-08-29T15:32:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:01 smithi171 ceph-mon[90031]: pgmap v1149: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 8308/203487 objects degraded (4.083%); 18878/203487 objects misplaced (9.277%); 36 MiB/s, 578 objects/s recovering 2024-08-29T15:32:02.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:02.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:02.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:02.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:02.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:02.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:02.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:02.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:02.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:02.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:02.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:02.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:02.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:02.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:02.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:02.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:02.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:03.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:03.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:03.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:03.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:03.206 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 8308/203487 objects degraded (4.083%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:03.206 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 8308/203487 objects degraded (4.083%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:03.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:03.252 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:03.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 8308/203487 objects degraded (4.083%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:03.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:03.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:03.391 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:03.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:03.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:03.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:03.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:03.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:03.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:03.672 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:03.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:03.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:03.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:03.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:03.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:03.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:03.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:04.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:04.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:04.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:04.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:04.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:04.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:04.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:03 smithi171 ceph-mon[90031]: pgmap v1150: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 8348/200775 objects degraded (4.158%); 18731/200775 objects misplaced (9.329%); 24 MiB/s, 388 objects/s recovering 2024-08-29T15:32:04.267 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:32:04 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=infra.usagestats t=2024-08-29T15:32:04.218623884Z level=info msg="Usage stats are ready to report" 2024-08-29T15:32:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:03 smithi027 ceph-mon[98970]: pgmap v1150: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 8348/200775 objects degraded (4.158%); 18731/200775 objects misplaced (9.329%); 24 MiB/s, 388 objects/s recovering 2024-08-29T15:32:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:03 smithi027 ceph-mon[93921]: pgmap v1150: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 82 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 8348/200775 objects degraded (4.158%); 18731/200775 objects misplaced (9.329%); 24 MiB/s, 388 objects/s recovering 2024-08-29T15:32:04.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:04.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:04.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:04.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:04.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:04.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:04.530 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:04.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:04.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:04.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:04.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:04.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:04.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:04.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:04.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:04.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:04.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:04.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:04.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:05.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:05.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:05.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:05.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:05.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:05.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:05.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:05.296 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:05.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:05.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:05.427 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:05.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:05.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:05.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:05.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:05.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:05.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:05.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:05.771 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:05.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:05.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:05.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:05.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:05.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:05.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:06.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:06.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:06.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:06.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:06.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:06.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:05 smithi171 ceph-mon[90031]: pgmap v1151: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 8414/197727 objects degraded (4.255%); 18297/197727 objects misplaced (9.254%); 20 MiB/s, 321 objects/s recovering 2024-08-29T15:32:06.307 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:05 smithi027 ceph-mon[98970]: pgmap v1151: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 8414/197727 objects degraded (4.255%); 18297/197727 objects misplaced (9.254%); 20 MiB/s, 321 objects/s recovering 2024-08-29T15:32:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:05 smithi027 ceph-mon[93921]: pgmap v1151: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 8414/197727 objects degraded (4.255%); 18297/197727 objects misplaced (9.254%); 20 MiB/s, 321 objects/s recovering 2024-08-29T15:32:06.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:06.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:06.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:06.457 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:06.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:32:06.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:06.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:06.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:06.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:06.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:06.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:06.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:06.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:06.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:06.921 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:06.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:07.006 INFO:teuthology.orchestra.run.smithi027.stderr:set pool 1 pgp_num to 1 2024-08-29T15:32:07.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:07.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:07.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:07.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:07.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:07.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:07.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2072332503' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-29T15:32:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 8414/197727 objects degraded (4.255%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2072332503' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-29T15:32:07.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 8414/197727 objects degraded (4.255%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:07.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2072332503' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-29T15:32:07.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 8414/197727 objects degraded (4.255%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:07.323 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:07.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:07.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:07.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:07.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:07.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:07.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:07.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:07.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:07.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:07.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:07.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:07.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:07.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:07.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:07.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:07.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:07.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:07.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:08.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:08.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:08 smithi027 ceph-mon[98970]: pgmap v1152: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 8414/195282 objects degraded (4.309%); 17566/195282 objects misplaced (8.995%); 24 MiB/s, 386 objects/s recovering 2024-08-29T15:32:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2072332503' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-29T15:32:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:08 smithi027 ceph-mon[98970]: osdmap e321: 8 total, 8 up, 8 in 2024-08-29T15:32:08.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:08 smithi027 ceph-mon[93921]: pgmap v1152: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 8414/195282 objects degraded (4.309%); 17566/195282 objects misplaced (8.995%); 24 MiB/s, 386 objects/s recovering 2024-08-29T15:32:08.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2072332503' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-29T15:32:08.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:08 smithi027 ceph-mon[93921]: osdmap e321: 8 total, 8 up, 8 in 2024-08-29T15:32:08.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:08.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:08.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:08 smithi171 ceph-mon[90031]: pgmap v1152: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 8414/195282 objects degraded (4.309%); 17566/195282 objects misplaced (8.995%); 24 MiB/s, 386 objects/s recovering 2024-08-29T15:32:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2072332503' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-29T15:32:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:08 smithi171 ceph-mon[90031]: osdmap e321: 8 total, 8 up, 8 in 2024-08-29T15:32:08.309 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:08.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:08.443 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:08.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:08.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:08.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:08.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:08.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:08.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:08.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:08.857 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:08.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:08.959 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:08.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:08.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:09.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:09.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:09.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:09.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:09.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:09.261 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:09.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:09.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:09.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:32:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:32:09] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-29T15:32:09.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:09.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:09.415 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:09.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:09.517 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:09.541 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:09.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:09.571 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:32:09.571 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:32:09.606 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:32:09.606 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:32:09.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:09.656 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:32:09.657 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:32:09.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:09.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:09.690 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:32:09.691 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:32:09.724 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:32:09.725 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:32:09.761 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:32:09.762 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:32:09.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:09.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:09.799 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:32:09.799 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:32:09.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:09.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:09.837 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:32:09.837 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:32:09.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:09.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:09.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:10.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:10.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:10.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:10.129 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:32:10.129 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:32:10.129 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:10 smithi027 ceph-mon[98970]: pgmap v1154: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 8579/192633 objects degraded (4.454%); 17247/192633 objects misplaced (8.953%); 14 MiB/s, 217 objects/s recovering 2024-08-29T15:32:10.129 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/476710197' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:10.129 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:10 smithi027 ceph-mon[93921]: pgmap v1154: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 8579/192633 objects degraded (4.454%); 17247/192633 objects misplaced (8.953%); 14 MiB/s, 217 objects/s recovering 2024-08-29T15:32:10.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/476710197' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:10.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:10.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:10.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:10.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:10.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:10.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:10.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:10.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:10.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:10.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:10 smithi171 ceph-mon[90031]: pgmap v1154: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 8579/192633 objects degraded (4.454%); 17247/192633 objects misplaced (8.953%); 14 MiB/s, 217 objects/s recovering 2024-08-29T15:32:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/476710197' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:10.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:10.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:10.629 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:10.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:10.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:10.757 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:10.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:10.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:10.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:10.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:10.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:11.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:11.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:11.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:11.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:11.178 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:11.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:11.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:11.337 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:11.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:11.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:11.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:11.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:11.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:11.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:11.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:11.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:11.678 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:11.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:11.717 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:11.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:11.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:11.876 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:11.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:11.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:12.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:12.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:12 smithi027 ceph-mon[98970]: pgmap v1155: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8579/188016 objects degraded (4.563%); 16254/188016 objects misplaced (8.645%); 10 MiB/s, 164 objects/s recovering 2024-08-29T15:32:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 8579/188016 objects degraded (4.563%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:12 smithi027 ceph-mon[93921]: pgmap v1155: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8579/188016 objects degraded (4.563%); 16254/188016 objects misplaced (8.645%); 10 MiB/s, 164 objects/s recovering 2024-08-29T15:32:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 8579/188016 objects degraded (4.563%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:12.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:12.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:12.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:12.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:32:12.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:12.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:12.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:12.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:12.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:12.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:12.407 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:12.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:12.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:12 smithi171 ceph-mon[90031]: pgmap v1155: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8579/188016 objects degraded (4.563%); 16254/188016 objects misplaced (8.645%); 10 MiB/s, 164 objects/s recovering 2024-08-29T15:32:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 8579/188016 objects degraded (4.563%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:12.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:12.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:12.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:12.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:12.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:12.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:12.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:12.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:12.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:13.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:13.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:13.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:13.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.175 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:13.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:13.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:13.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:13.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:13.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:13.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:32:13.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:13.557 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:13.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:13.674 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:32:13.674 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:32:13.674 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-29T15:32:13.674 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2024-08-29T15:32:13.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 6 0 2024-08-29T15:32:13.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:13.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:13.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:13.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:13.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:13.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:13.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:13.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:14.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:14.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:14.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:14.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:14.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:14.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:14.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:14.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:14 smithi027 ceph-mon[93921]: pgmap v1156: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 8690/184689 objects degraded (4.705%); 16058/184689 objects misplaced (8.695%); 10 MiB/s, 162 objects/s recovering 2024-08-29T15:32:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:14.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:14 smithi027 ceph-mon[98970]: pgmap v1156: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 8690/184689 objects degraded (4.705%); 16058/184689 objects misplaced (8.695%); 10 MiB/s, 162 objects/s recovering 2024-08-29T15:32:14.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:14.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:14.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:14.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:14.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:14.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:14.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:14 smithi171 ceph-mon[90031]: pgmap v1156: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 83 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 8690/184689 objects degraded (4.705%); 16058/184689 objects misplaced (8.695%); 10 MiB/s, 162 objects/s recovering 2024-08-29T15:32:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:14.518 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:14.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:14.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:14.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:14.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:14.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:14.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:14.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:14.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:14.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:14.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:15.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:15.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:15.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:15.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:15.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:15.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:15.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:15.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:15.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:15.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:15.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:15.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:15.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:15.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:15.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:15.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:15.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:15.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:15.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:15.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:16.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:16.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:16.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:16.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:16.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:16.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:16.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:16 smithi027 ceph-mon[98970]: pgmap v1157: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8786/180981 objects degraded (4.855%); 15480/180981 objects misplaced (8.553%); 10 MiB/s, 164 objects/s recovering 2024-08-29T15:32:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:16 smithi027 ceph-mon[98970]: osdmap e322: 8 total, 8 up, 8 in 2024-08-29T15:32:16.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:16 smithi027 ceph-mon[93921]: pgmap v1157: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8786/180981 objects degraded (4.855%); 15480/180981 objects misplaced (8.553%); 10 MiB/s, 164 objects/s recovering 2024-08-29T15:32:16.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:16 smithi027 ceph-mon[93921]: osdmap e322: 8 total, 8 up, 8 in 2024-08-29T15:32:16.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:16.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:16.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:16.444 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:16.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:16.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:16 smithi171 ceph-mon[90031]: pgmap v1157: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8786/180981 objects degraded (4.855%); 15480/180981 objects misplaced (8.553%); 10 MiB/s, 164 objects/s recovering 2024-08-29T15:32:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:16 smithi171 ceph-mon[90031]: osdmap e322: 8 total, 8 up, 8 in 2024-08-29T15:32:16.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:16.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:16.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:16.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:16.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:16.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:16.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:16.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:16.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:16.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:16.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:16.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:17.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:17.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:17.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:17.250 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:17.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:17.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:17.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:17 smithi027 ceph-mon[93921]: osdmap e323: 8 total, 8 up, 8 in 2024-08-29T15:32:17.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 8786/180981 objects degraded (4.855%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:17.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:17 smithi027 ceph-mon[98970]: osdmap e323: 8 total, 8 up, 8 in 2024-08-29T15:32:17.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 8786/180981 objects degraded (4.855%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:17.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:17.413 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:17.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:17.515 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:17 smithi171 ceph-mon[90031]: osdmap e323: 8 total, 8 up, 8 in 2024-08-29T15:32:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 8786/180981 objects degraded (4.855%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:17.539 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:17.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:17.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:17.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:17.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:17.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:17.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:17.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:17.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:18.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:18.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:18.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:18.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:18.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:18.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:18.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:18.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:18.388 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:18.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:18.489 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:18.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:18.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:18 smithi171 ceph-mon[90031]: pgmap v1160: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 8786/178482 objects degraded (4.923%); 14934/178482 objects misplaced (8.367%); 11 MiB/s, 168 objects/s recovering 2024-08-29T15:32:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:18 smithi027 ceph-mon[93921]: pgmap v1160: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 8786/178482 objects degraded (4.923%); 14934/178482 objects misplaced (8.367%); 11 MiB/s, 168 objects/s recovering 2024-08-29T15:32:18.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:18 smithi027 ceph-mon[98970]: pgmap v1160: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 8786/178482 objects degraded (4.923%); 14934/178482 objects misplaced (8.367%); 11 MiB/s, 168 objects/s recovering 2024-08-29T15:32:18.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:18.641 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:18.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:18.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:18.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:18.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:18.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:18.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:18.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:18.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:19.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:19.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:19.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:19.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:19.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:19.172 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:19.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:19.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:19.257 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:19.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:19.315 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:19.318 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:19.320 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:32:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:32:19] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-08-29T15:32:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:19.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:19.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:19.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:19.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:19.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:19.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:19.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:19.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:19.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:19.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:19.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:19.737 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:19.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:19.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:19.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:19.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:19.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:20.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:20.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:20.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:20.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:20.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:20.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:20.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:20.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:20.370 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:20 smithi027 ceph-mon[98970]: pgmap v1161: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 8868/176364 objects degraded (5.028%); 14642/176364 objects misplaced (8.302%); 7.3 MiB/s, 117 objects/s recovering 2024-08-29T15:32:20.370 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:20 smithi027 ceph-mon[93921]: pgmap v1161: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 8868/176364 objects degraded (5.028%); 14642/176364 objects misplaced (8.302%); 7.3 MiB/s, 117 objects/s recovering 2024-08-29T15:32:20.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:20.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:20 smithi171 ceph-mon[90031]: pgmap v1161: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 8868/176364 objects degraded (5.028%); 14642/176364 objects misplaced (8.302%); 7.3 MiB/s, 117 objects/s recovering 2024-08-29T15:32:20.556 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:20.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:20.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:20.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:20.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:20.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:20.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:20.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:20.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:20.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:20.965 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:20.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:21.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:21.102 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:21.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:21.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:21.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:21.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:21.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:21.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:21.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:21.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:21.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:21.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:21.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:21.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:21.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:21.659 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:21.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:21.761 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:21.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:21.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:21.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:21.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:21.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:21.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:22.020 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:22.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:22.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:22.082 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:32:22.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:22.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:22.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:22.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:22.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:22.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:22.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:22.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:22.488 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:22.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:22 smithi171 ceph-mon[90031]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8868/172323 objects degraded (5.146%); 13666/172323 objects misplaced (7.930%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:32:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2432878695' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:32:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:32:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:32:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 8868/172323 objects degraded (5.146%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:22.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[98970]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8868/172323 objects degraded (5.146%); 13666/172323 objects misplaced (7.930%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2432878695' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 8868/172323 objects degraded (5.146%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[93921]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 8868/172323 objects degraded (5.146%); 13666/172323 objects misplaced (7.930%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2432878695' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:32:22.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 8868/172323 objects degraded (5.146%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:22.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:22.627 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:22.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:22.728 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:22.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:22.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:22.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:22.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:22.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:22.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:22.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:22.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:23.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:23.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:23.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:23.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:23.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:23.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:23.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:32:23.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[98970]: osdmap e324: 8 total, 8 up, 8 in 2024-08-29T15:32:23.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:23.248 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:23.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:23.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:32:23.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[93921]: osdmap e324: 8 total, 8 up, 8 in 2024-08-29T15:32:23.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:23.248 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:23.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:23.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:23.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:23.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:23.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:23.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:23.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:23 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:32:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:23 smithi171 ceph-mon[90031]: osdmap e324: 8 total, 8 up, 8 in 2024-08-29T15:32:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:23.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:23.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:23.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:23.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:23.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:23.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:23.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:23.884 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:23.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:23.986 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:24.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:24.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:24.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:24.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:24.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:24 smithi027 ceph-mon[93921]: pgmap v1164: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 8933/169248 objects degraded (5.278%); 13504/169248 objects misplaced (7.979%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:32:24.177 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:24 smithi027 ceph-mon[98970]: pgmap v1164: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 8933/169248 objects degraded (5.278%); 13504/169248 objects misplaced (7.979%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:32:24.244 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:24.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:24.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:24.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:24.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:24.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:24.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:24 smithi171 ceph-mon[90031]: pgmap v1164: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 8933/169248 objects degraded (5.278%); 13504/169248 objects misplaced (7.979%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:32:24.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:24.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:24.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:24.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:24.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:24.774 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:24.799 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:24.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:24.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:24.949 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:24.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:25.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:25.101 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:25.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:25.133 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.6 primary-affinity to 0 (802) 2024-08-29T15:32:25.203 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:25.226 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:25.312 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1902037194' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.0} : dispatch 2024-08-29T15:32:25.313 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1902037194' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.0} : dispatch 2024-08-29T15:32:25.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:25.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:25.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:25.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:25.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:25.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1902037194' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.0} : dispatch 2024-08-29T15:32:25.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:25.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:25.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:25.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:32:25.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:25.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:25.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:25.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:25.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:25.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:25.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:25.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:25.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:26.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:26.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:26.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:26.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:26.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:26.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:26.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:26.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:26.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:26.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:26.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:26 smithi171 ceph-mon[90031]: pgmap v1165: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 9049/165354 objects degraded (5.473%); 12636/165354 objects misplaced (7.642%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:32:26.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1902037194' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2024-08-29T15:32:26.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:26 smithi171 ceph-mon[90031]: osdmap e325: 8 total, 8 up, 8 in 2024-08-29T15:32:26.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:26 smithi171 ceph-mon[90031]: osdmap e326: 8 total, 8 up, 8 in 2024-08-29T15:32:26.471 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:26.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[93921]: pgmap v1165: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 9049/165354 objects degraded (5.473%); 12636/165354 objects misplaced (7.642%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:32:26.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1902037194' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2024-08-29T15:32:26.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[93921]: osdmap e325: 8 total, 8 up, 8 in 2024-08-29T15:32:26.491 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[93921]: osdmap e326: 8 total, 8 up, 8 in 2024-08-29T15:32:26.491 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[98970]: pgmap v1165: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 9049/165354 objects degraded (5.473%); 12636/165354 objects misplaced (7.642%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:32:26.491 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1902037194' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2024-08-29T15:32:26.491 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[98970]: osdmap e325: 8 total, 8 up, 8 in 2024-08-29T15:32:26.492 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:26 smithi027 ceph-mon[98970]: osdmap e326: 8 total, 8 up, 8 in 2024-08-29T15:32:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:26.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:26.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:26.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:26.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:26.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:26.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:26.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:26.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:26.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:26.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:26.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:26.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:26.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:27.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:27.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.150 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:27.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:27.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:27.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:27.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 9049/165354 objects degraded (5.473%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:27.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:27 smithi027 ceph-mon[98970]: osdmap e327: 8 total, 8 up, 8 in 2024-08-29T15:32:27.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 9049/165354 objects degraded (5.473%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:27.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:27 smithi027 ceph-mon[93921]: osdmap e327: 8 total, 8 up, 8 in 2024-08-29T15:32:27.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:27.407 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:27.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 9049/165354 objects degraded (5.473%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:32:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:27 smithi171 ceph-mon[90031]: osdmap e327: 8 total, 8 up, 8 in 2024-08-29T15:32:27.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:27.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:27.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.676 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:27.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:27.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:27.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:27.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:27.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:27.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:27.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:28.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:28.086 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:28.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:28.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:28.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:28.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:28.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:28.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:28.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:28.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:28.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:28.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:28.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:28.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:28.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:28 smithi171 ceph-mon[90031]: pgmap v1168: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 9049/162975 objects degraded (5.552%); 12636/162975 objects misplaced (7.753%); 5.1 MiB/s, 79 objects/s recovering 2024-08-29T15:32:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:28 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:32:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:28.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[98970]: pgmap v1168: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 9049/162975 objects degraded (5.552%); 12636/162975 objects misplaced (7.753%); 5.1 MiB/s, 79 objects/s recovering 2024-08-29T15:32:28.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:32:28.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:28.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:28.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[93921]: pgmap v1168: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 9049/162975 objects degraded (5.552%); 12636/162975 objects misplaced (7.753%); 5.1 MiB/s, 79 objects/s recovering 2024-08-29T15:32:28.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:32:28.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:28.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:28.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:28.620 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:28.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:28.722 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:28.747 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:28.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:28.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:28.879 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:28.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:28.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:28.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:29.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:29.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:29.053 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:32:29] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-29T15:32:29.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:29.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:29.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:29.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:29.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:29.322 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:29.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:29.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:29.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:29.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:29.450 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:29.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:29.511 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:32:29.511 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:32:29.547 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:32:29.547 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:32:29.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:29.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:29.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:29.586 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:32:29.587 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:32:29.629 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:32:29.629 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:32:29.659 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:32:29.659 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:32:29.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:29.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:32:29.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:32:29.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:29.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:29.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:29.735 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:32:29.736 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:32:29.775 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:32:29.775 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:32:29.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:29.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:29.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:29.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.005 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:30.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:29 smithi171 ceph-mon[90031]: pgmap v1170: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 9140/161082 objects degraded (5.674%); 12237/161082 objects misplaced (7.597%); 7.3 MiB/s, 114 objects/s recovering 2024-08-29T15:32:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1970156403' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:30.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[98970]: pgmap v1170: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 9140/161082 objects degraded (5.674%); 12237/161082 objects misplaced (7.597%); 7.3 MiB/s, 114 objects/s recovering 2024-08-29T15:32:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1970156403' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[93921]: pgmap v1170: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 9140/161082 objects degraded (5.674%); 12237/161082 objects misplaced (7.597%); 7.3 MiB/s, 114 objects/s recovering 2024-08-29T15:32:30.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1970156403' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:30.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:30.146 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:30.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:30.300 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:30.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:30.402 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:30.408 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:30.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:30.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:30.470 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:32:30.471 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:32:30.471 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:32:30.471 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:32:30.471 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:32:30.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:30.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:30.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:30.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:30.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:30.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:30.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:30.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:30.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:30.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:30.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:31.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:31.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:31.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:31.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:31.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:31.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:31.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:31.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:31.355 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:31.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:31.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:31.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:31.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:31.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:31.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:31.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:31.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:31.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:31.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:31.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:31.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:31.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:31.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:31.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:32.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:31 smithi027 ceph-mon[98970]: pgmap v1171: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 peering, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 1842/158553 objects degraded (1.162%); 14368/158553 objects misplaced (9.062%); 14 MiB/s, 229 objects/s recovering 2024-08-29T15:32:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:31 smithi027 ceph-mon[93921]: pgmap v1171: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 peering, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 1842/158553 objects degraded (1.162%); 14368/158553 objects misplaced (9.062%); 14 MiB/s, 229 objects/s recovering 2024-08-29T15:32:32.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:32.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:31 smithi171 ceph-mon[90031]: pgmap v1171: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 peering, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 377 op/s; 1842/158553 objects degraded (1.162%); 14368/158553 objects misplaced (9.062%); 14 MiB/s, 229 objects/s recovering 2024-08-29T15:32:32.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:32.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:32.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:32.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:32.425 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:32.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:32.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:32.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:32.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:32.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:32.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:32.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:32.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:32.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:32.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:32.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:32.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:32.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:33.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:33.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:33.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:33.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:33.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:33.239 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:33.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 1842/158553 objects degraded (1.162%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:33.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 1842/158553 objects degraded (1.162%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:33.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 1842/158553 objects degraded (1.162%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:33.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:33.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:33.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:33.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:33.490 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:33.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:33.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:33.619 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:33.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:33.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:33.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:33.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:33.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:33.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:33.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:33.821 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:32:33.821 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:32:33.821 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.7975436216705227 2024-08-29T15:32:33.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 2 0.7975436216705227 2024-08-29T15:32:33.888 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:33.911 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:33.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:34.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:34.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:33 smithi027 ceph-mon[93921]: pgmap v1172: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 1842/155196 objects degraded (1.187%); 14276/155196 objects misplaced (9.199%); 12 MiB/s, 192 objects/s recovering 2024-08-29T15:32:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:33 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:32:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:33 smithi027 ceph-mon[98970]: pgmap v1172: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 1842/155196 objects degraded (1.187%); 14276/155196 objects misplaced (9.199%); 12 MiB/s, 192 objects/s recovering 2024-08-29T15:32:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:33 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:32:34.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:34.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:34.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:33 smithi171 ceph-mon[90031]: pgmap v1172: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 1842/155196 objects degraded (1.187%); 14276/155196 objects misplaced (9.199%); 12 MiB/s, 192 objects/s recovering 2024-08-29T15:32:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:33 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:32:34.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:34.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:34.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:34.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:34.472 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:34.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:34.600 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:34.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:34.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:34.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:34.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:34.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:34.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:34.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:34.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:34.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:34.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:35.096 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:35.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:35.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:35.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:35.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:35.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:35.371 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:35.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:35.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:35.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:35.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:35.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:35.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:35.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:35.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:35.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:35.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:35.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:35.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:35.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:35.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:36.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:36.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:36.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:36.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:36.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:36.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:35 smithi171 ceph-mon[90031]: pgmap v1173: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 1871/150591 objects degraded (1.242%); 13514/150591 objects misplaced (8.974%); 14 MiB/s, 228 objects/s recovering 2024-08-29T15:32:36.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:35 smithi027 ceph-mon[98970]: pgmap v1173: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 1871/150591 objects degraded (1.242%); 13514/150591 objects misplaced (8.974%); 14 MiB/s, 228 objects/s recovering 2024-08-29T15:32:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:35 smithi027 ceph-mon[93921]: pgmap v1173: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 1871/150591 objects degraded (1.242%); 13514/150591 objects misplaced (8.974%); 14 MiB/s, 228 objects/s recovering 2024-08-29T15:32:36.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:36.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:36.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:36.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:36.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:36.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:36.626 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:36.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:36.728 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:36.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:36.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:36.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:36.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:36.910 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:36.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:36.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:37.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:37.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:37.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:37.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:37.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:37.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:37.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:37.303 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:37.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:37.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:37.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:37.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:37.500 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:37.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:37.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:37.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:37.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:37.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:37.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:37.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:37.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:37.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:37.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:37.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:38.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:38.057 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:38.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:38.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:38.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:38.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:38.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:37 smithi171 ceph-mon[90031]: pgmap v1174: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 1871/149013 objects degraded (1.256%); 13514/149013 objects misplaced (9.069%); 12 MiB/s, 198 objects/s recovering 2024-08-29T15:32:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 1871/149013 objects degraded (1.256%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:38.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:38.316 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:38.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:37 smithi027 ceph-mon[93921]: pgmap v1174: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 1871/149013 objects degraded (1.256%); 13514/149013 objects misplaced (9.069%); 12 MiB/s, 198 objects/s recovering 2024-08-29T15:32:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 1871/149013 objects degraded (1.256%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:37 smithi027 ceph-mon[98970]: pgmap v1174: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 1871/149013 objects degraded (1.256%); 13514/149013 objects misplaced (9.069%); 12 MiB/s, 198 objects/s recovering 2024-08-29T15:32:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 1871/149013 objects degraded (1.256%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:38.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:38.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:38.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:38.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:38.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:38.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:38.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:38.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:38.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:38.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:38.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:38.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:38.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:38.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:38.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:39.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:39.017 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:32:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:32:39] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-29T15:32:39.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:39.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:39.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:39.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:32:39.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:39.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:39.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:39.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:39.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:39.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:39.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:39.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:39.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:39.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:39.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:39.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:39.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:39.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:39.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:39.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:39.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:39.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:39.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:39.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:32:39.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:40.022 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:40.023 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:39 smithi027 ceph-mon[93921]: pgmap v1175: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 2070/146331 objects degraded (1.415%); 13142/146331 objects misplaced (8.981%); 12 MiB/s, 185 objects/s recovering 2024-08-29T15:32:40.023 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:39 smithi027 ceph-mon[98970]: pgmap v1175: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 2070/146331 objects degraded (1.415%); 13142/146331 objects misplaced (8.981%); 12 MiB/s, 185 objects/s recovering 2024-08-29T15:32:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:40.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:40.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:40.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:40.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:40.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:39 smithi171 ceph-mon[90031]: pgmap v1175: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 2070/146331 objects degraded (1.415%); 13142/146331 objects misplaced (8.981%); 12 MiB/s, 185 objects/s recovering 2024-08-29T15:32:40.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:40.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:40.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:40.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:40.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:40.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:40.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:40.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:40.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:40.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:40.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:40.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:40.736 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:40.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:40.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:40.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:40.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:40.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:41.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:41.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:41.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:41.192 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:41.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:41.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:41.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:41.355 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:41.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:41.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:41.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:41.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:41.522 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:41.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:41.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:41.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:41.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:41.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:41.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:41.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:41.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:41.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:41.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:41.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:42.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:42.054 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:42.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:42.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:42.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:42.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:41 smithi171 ceph-mon[90031]: pgmap v1176: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 2070/141828 objects degraded (1.460%); 12380/141828 objects misplaced (8.729%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:32:42.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:41 smithi027 ceph-mon[98970]: pgmap v1176: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 2070/141828 objects degraded (1.460%); 12380/141828 objects misplaced (8.729%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:32:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:41 smithi027 ceph-mon[93921]: pgmap v1176: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 2070/141828 objects degraded (1.460%); 12380/141828 objects misplaced (8.729%); 12 MiB/s, 199 objects/s recovering 2024-08-29T15:32:42.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:42.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:42.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:42.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:42.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:42.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:42.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:42.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:42.614 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:42.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:42.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:42.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:42.769 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:42.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:42.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:42.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:42.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:43.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:43.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:43.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:43.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2070/141828 objects degraded (1.460%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:43.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2070/141828 objects degraded (1.460%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:43.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2070/141828 objects degraded (1.460%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:43.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:43.421 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:43.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:43.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:43.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:43.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:43.685 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:43.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:43.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:43.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:43.894 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:43.971 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:43.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:43.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:43.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:44.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:43 smithi027 ceph-mon[93921]: pgmap v1177: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 2070/140241 objects degraded (1.476%); 12380/140241 objects misplaced (8.828%); 6.8 MiB/s, 109 objects/s recovering 2024-08-29T15:32:44.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:44.038 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:44.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:43 smithi027 ceph-mon[98970]: pgmap v1177: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 2070/140241 objects degraded (1.476%); 12380/140241 objects misplaced (8.828%); 6.8 MiB/s, 109 objects/s recovering 2024-08-29T15:32:44.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:44.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:44.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:44.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:44.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:44.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:43 smithi171 ceph-mon[90031]: pgmap v1177: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 2070/140241 objects degraded (1.476%); 12380/140241 objects misplaced (8.828%); 6.8 MiB/s, 109 objects/s recovering 2024-08-29T15:32:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:32:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:44.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:44.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:44.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:44.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:44.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:44.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:44.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:44.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:44.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:44.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:44.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:44.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:44.687 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:44.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:44.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:44.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:44.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:44.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:44.917 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:32:44.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:44.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:44.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:44.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:45.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:45.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:45.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:45.195 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:45.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:45.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:45.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:45.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3736099977' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:32:45.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3736099977' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:32:45.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3736099977' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:32:45.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:45.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:45.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:45.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:45.485 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:45.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:45.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:45.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:45.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:45.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:45.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:45.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:45.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:45.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:45.862 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:45.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:45.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:45.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:46.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:46.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:46.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:46.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:46.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:45 smithi171 ceph-mon[90031]: pgmap v1178: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2253/133200 objects degraded (1.691%); 11280/133200 objects misplaced (8.468%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:32:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:45 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3736099977' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:32:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:45 smithi171 ceph-mon[90031]: osdmap e328: 8 total, 8 up, 8 in 2024-08-29T15:32:46.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[98970]: pgmap v1178: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2253/133200 objects degraded (1.691%); 11280/133200 objects misplaced (8.468%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:32:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3736099977' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:32:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[98970]: osdmap e328: 8 total, 8 up, 8 in 2024-08-29T15:32:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[93921]: pgmap v1178: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2253/133200 objects degraded (1.691%); 11280/133200 objects misplaced (8.468%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:32:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:32:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3736099977' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:32:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:45 smithi027 ceph-mon[93921]: osdmap e328: 8 total, 8 up, 8 in 2024-08-29T15:32:46.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:46.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:46.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:46.599 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:46.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:46.735 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:46.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:46.861 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:46.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:46.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:46.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:46.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:47.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:47.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:47.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:47.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:47.249 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:47.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:47.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:47.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:47.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:47.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:47.527 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:47.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:47.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:47.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:47.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:47.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:47.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:47.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:47.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:47.888 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:47.914 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:47.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:47.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:48.016 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:48.038 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:48.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:48.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:47 smithi171 ceph-mon[90031]: pgmap v1180: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2253/131547 objects degraded (1.713%); 11280/131547 objects misplaced (8.575%); 8.9 MiB/s, 141 objects/s recovering 2024-08-29T15:32:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2253/131547 objects degraded (1.713%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:48.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:48.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:48.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:47 smithi027 ceph-mon[98970]: pgmap v1180: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2253/131547 objects degraded (1.713%); 11280/131547 objects misplaced (8.575%); 8.9 MiB/s, 141 objects/s recovering 2024-08-29T15:32:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2253/131547 objects degraded (1.713%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:47 smithi027 ceph-mon[93921]: pgmap v1180: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2253/131547 objects degraded (1.713%); 11280/131547 objects misplaced (8.575%); 8.9 MiB/s, 141 objects/s recovering 2024-08-29T15:32:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2253/131547 objects degraded (1.713%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:48.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:48.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:48.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:48.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.594 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:48.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:48.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:48.722 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:48.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:48.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:48.848 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:48.950 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:48.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:48.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:49.072 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:32:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:32:49] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-29T15:32:49.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:49.122 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:49.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:49.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:49.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:49.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:49.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:49.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:49.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:49.380 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:49.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:49.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:49.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:49.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:49.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:49.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:49.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:49.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:49.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:49.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:49.758 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:49.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:49.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:49.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:49.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:49.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:50.014 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:50.116 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:50.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:50.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:50.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:49 smithi171 ceph-mon[90031]: pgmap v1181: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 2411/128754 objects degraded (1.873%); 10938/128754 objects misplaced (8.495%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:32:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:49 smithi171 ceph-mon[90031]: osdmap e329: 8 total, 8 up, 8 in 2024-08-29T15:32:50.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:50.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:49 smithi027 ceph-mon[98970]: pgmap v1181: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 2411/128754 objects degraded (1.873%); 10938/128754 objects misplaced (8.495%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:32:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:49 smithi027 ceph-mon[98970]: osdmap e329: 8 total, 8 up, 8 in 2024-08-29T15:32:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:49 smithi027 ceph-mon[93921]: pgmap v1181: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 2411/128754 objects degraded (1.873%); 10938/128754 objects misplaced (8.495%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:32:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:49 smithi027 ceph-mon[93921]: osdmap e329: 8 total, 8 up, 8 in 2024-08-29T15:32:50.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:50.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:50.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:50.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:50.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:50.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:50.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:50.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:50.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:50.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:50.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:50.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:50.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:50.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:51.094 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:51.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:51.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:51.224 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:51.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:51.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:50 smithi171 ceph-mon[90031]: osdmap e330: 8 total, 8 up, 8 in 2024-08-29T15:32:51.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:50 smithi027 ceph-mon[98970]: osdmap e330: 8 total, 8 up, 8 in 2024-08-29T15:32:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:50 smithi027 ceph-mon[93921]: osdmap e330: 8 total, 8 up, 8 in 2024-08-29T15:32:51.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:51.377 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:51.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:51.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:51.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:51.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:51.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:51.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:51.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:51.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:51.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:51.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:51.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:51.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:51.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:51.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:51.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:51.998 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:52.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:52.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:52.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:52.167 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:52.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:51 smithi171 ceph-mon[90031]: pgmap v1184: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 2411/124842 objects degraded (1.931%); 10559/124842 objects misplaced (8.458%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:32:52.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:52.315 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:51 smithi027 ceph-mon[98970]: pgmap v1184: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 2411/124842 objects degraded (1.931%); 10559/124842 objects misplaced (8.458%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:32:52.315 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:51 smithi027 ceph-mon[93921]: pgmap v1184: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 2411/124842 objects degraded (1.931%); 10559/124842 objects misplaced (8.458%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:32:52.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:52.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:52.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:52.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:52.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:52.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:52.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:52.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:52.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:52.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:52.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:53.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:53.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:53.067 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:32:53.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:32:53.068 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:32:53.068 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:32:53.068 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:32:53.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:32:53.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:32:53.069 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:32:53.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:32:53.070 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:32:53.070 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:32:53.070 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:32:53.071 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:32:53.071 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:32:53.071 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:32:53.072 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:32:53.104 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2411/124842 objects degraded (1.931%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:53.105 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/923764354' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:53.105 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2411/124842 objects degraded (1.931%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:53.105 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/923764354' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:53.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:53.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:53.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:53.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2411/124842 objects degraded (1.931%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:53.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/923764354' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:32:53.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:53.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:53.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:53.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:53.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:53.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:53.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:53.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:53.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:53.621 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:53.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:53.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:53.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:53.748 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:53.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:53.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:53.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:53.896 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:53.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:53.997 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:54.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:54.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:54.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:54.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:54.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:54.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:54.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:54 smithi171 ceph-mon[90031]: pgmap v1185: 89 pgs: 1 active+recovery_wait, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 436 op/s; 2411/122877 objects degraded (1.962%); 10248/122877 objects misplaced (8.340%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:32:54.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:54 smithi027 ceph-mon[93921]: pgmap v1185: 89 pgs: 1 active+recovery_wait, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 436 op/s; 2411/122877 objects degraded (1.962%); 10248/122877 objects misplaced (8.340%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:32:54.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:54 smithi027 ceph-mon[98970]: pgmap v1185: 89 pgs: 1 active+recovery_wait, 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 436 op/s; 2411/122877 objects degraded (1.962%); 10248/122877 objects misplaced (8.340%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:32:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:54.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:54.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:54.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:32:54.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:54.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:54.510 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:54.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:54.591 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:32:54.591 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:32:54.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:54.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:54.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:54.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:54.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:54.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:54.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:54.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:54.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:55.025 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.2 to 0.797544 (cc2b) 2024-08-29T15:32:55.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:55.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:55.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:55.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:55.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:55.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:55.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:55.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1930363153' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227} : dispatch 2024-08-29T15:32:55.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:55 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227} : dispatch 2024-08-29T15:32:55.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1930363153' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227} : dispatch 2024-08-29T15:32:55.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:55 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227} : dispatch 2024-08-29T15:32:55.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:32:55.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:55.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:55.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:55.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:55.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:55.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:55.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1930363153' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227} : dispatch 2024-08-29T15:32:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:55 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227} : dispatch 2024-08-29T15:32:55.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:55.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:55.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:55.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:55.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:55.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:55.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:55.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:55.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:55.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:55.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:55.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:56.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:56.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:56.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:56.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:56.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:56.300 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:56.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:56 smithi027 ceph-mon[98970]: pgmap v1186: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 8798/116862 objects degraded (7.529%); 7218/116862 objects misplaced (6.177%); 8.7 MiB/s, 138 objects/s recovering 2024-08-29T15:32:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:56 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227}]': finished 2024-08-29T15:32:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:56 smithi027 ceph-mon[98970]: osdmap e331: 8 total, 8 up, 8 in 2024-08-29T15:32:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:56 smithi027 ceph-mon[93921]: pgmap v1186: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 8798/116862 objects degraded (7.529%); 7218/116862 objects misplaced (6.177%); 8.7 MiB/s, 138 objects/s recovering 2024-08-29T15:32:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:56 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227}]': finished 2024-08-29T15:32:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:56 smithi027 ceph-mon[93921]: osdmap e331: 8 total, 8 up, 8 in 2024-08-29T15:32:56.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:56.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:56.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:56.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:32:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:56 smithi171 ceph-mon[90031]: pgmap v1186: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 8798/116862 objects degraded (7.529%); 7218/116862 objects misplaced (6.177%); 8.7 MiB/s, 138 objects/s recovering 2024-08-29T15:32:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:56 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7975436216705227}]': finished 2024-08-29T15:32:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:56 smithi171 ceph-mon[90031]: osdmap e331: 8 total, 8 up, 8 in 2024-08-29T15:32:56.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:56.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:56.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:56.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:56.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:56.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:56.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:56.943 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:56.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:56.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:56.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:57.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:57.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:57.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:57.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:57.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:57.258 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:57.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:57 smithi027 ceph-mon[98970]: osdmap e332: 8 total, 8 up, 8 in 2024-08-29T15:32:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 8798/116862 objects degraded (7.529%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:57.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:57 smithi027 ceph-mon[93921]: osdmap e332: 8 total, 8 up, 8 in 2024-08-29T15:32:57.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 8798/116862 objects degraded (7.529%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:57.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:32:57.388 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:57.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:32:57.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:57 smithi171 ceph-mon[90031]: osdmap e332: 8 total, 8 up, 8 in 2024-08-29T15:32:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 8798/116862 objects degraded (7.529%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:32:57.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:57.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:57.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:57.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:57.685 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:57.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:57.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:57.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:57.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:57.870 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:57.950 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:57.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:57.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:58.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:58.141 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:58.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:58.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:58.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:58.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:58.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:58 smithi027 ceph-mon[93921]: pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 8798/114534 objects degraded (7.682%); 7218/114534 objects misplaced (6.302%); 6.6 MiB/s, 105 objects/s recovering 2024-08-29T15:32:58.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:58 smithi027 ceph-mon[98970]: pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 8798/114534 objects degraded (7.682%); 7218/114534 objects misplaced (6.302%); 6.6 MiB/s, 105 objects/s recovering 2024-08-29T15:32:58.325 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:32:58.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:58.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:58.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:58 smithi171 ceph-mon[90031]: pgmap v1189: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 8798/114534 objects degraded (7.682%); 7218/114534 objects misplaced (6.302%); 6.6 MiB/s, 105 objects/s recovering 2024-08-29T15:32:58.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:32:58.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:58.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:58.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:32:58.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:58.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:58.720 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:58.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:58.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:58.861 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:58.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:58.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:32:59.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:32:59.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:32:59.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:32:59.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:59.285 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:59.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:32:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:32:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:32:59] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-29T15:32:59.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:32:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:59.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:32:59.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:32:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:32:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:32:59.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:59.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:32:59.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:32:59.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:59.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:32:59.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:32:59.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:32:59.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:32:59.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:32:59.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:32:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:32:59.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:00.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:00.083 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:00.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:00.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:00.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:00.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:00.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:00.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:00.351 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:00.380 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:00.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:00.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:00.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:00.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:00 smithi171 ceph-mon[90031]: pgmap v1190: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 8978/111756 objects degraded (8.034%); 6843/111756 objects misplaced (6.123%); 7.1 MiB/s, 114 objects/s recovering 2024-08-29T15:33:00.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:00 smithi027 ceph-mon[93921]: pgmap v1190: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 8978/111756 objects degraded (8.034%); 6843/111756 objects misplaced (6.123%); 7.1 MiB/s, 114 objects/s recovering 2024-08-29T15:33:00.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:00 smithi027 ceph-mon[98970]: pgmap v1190: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 8978/111756 objects degraded (8.034%); 6843/111756 objects misplaced (6.123%); 7.1 MiB/s, 114 objects/s recovering 2024-08-29T15:33:00.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:00.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:00.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:00.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:00.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:00.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:00.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:00.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:00.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:00.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:00.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:01.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:01.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:01.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:01.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:01.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:01.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:01.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:01.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:01.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:01.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:01.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:01.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:01.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:01.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:01.925 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:01.951 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:01.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:01.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:02.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:02.077 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:02.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:02.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:02.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:02.226 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:02.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:02.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:02 smithi027 ceph-mon[93921]: pgmap v1191: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 2651/107628 objects degraded (2.463%); 8852/107628 objects misplaced (8.225%); 6.3 MiB/s, 100 objects/s recovering 2024-08-29T15:33:02.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2651/107628 objects degraded (2.463%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:02.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:02 smithi027 ceph-mon[98970]: pgmap v1191: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 2651/107628 objects degraded (2.463%); 8852/107628 objects misplaced (8.225%); 6.3 MiB/s, 100 objects/s recovering 2024-08-29T15:33:02.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2651/107628 objects degraded (2.463%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:02.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:02.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:02.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:02.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:02.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:02.476 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:02.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:02.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:02 smithi171 ceph-mon[90031]: pgmap v1191: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 2651/107628 objects degraded (2.463%); 8852/107628 objects misplaced (8.225%); 6.3 MiB/s, 100 objects/s recovering 2024-08-29T15:33:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2651/107628 objects degraded (2.463%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:02.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:02.620 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:02.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:02.721 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:02.747 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:02.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:02.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:02.777 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:33:02.777 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:33:02.778 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-08-29T15:33:02.778 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:33:02.778 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:33:02.778 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-08-29T15:33:02.849 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:02.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:02.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:02.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:02.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.032 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:03.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:03.094 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:33:03.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:03.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:03.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:03.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:03.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:03.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:03.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:03.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:03.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:03.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:03.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:03.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:03.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:03.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:03.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:03.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:04.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:04.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:04.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:04.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:04.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:04.270 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:04.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:04.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:04.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:04.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:04.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:04.434 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:04 smithi171 ceph-mon[90031]: pgmap v1192: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 2651/105192 objects degraded (2.520%); 8852/105192 objects misplaced (8.415%); 3.0 MiB/s, 48 objects/s recovering 2024-08-29T15:33:04.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:04.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:04.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:04.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:04 smithi027 ceph-mon[93921]: pgmap v1192: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 2651/105192 objects degraded (2.520%); 8852/105192 objects misplaced (8.415%); 3.0 MiB/s, 48 objects/s recovering 2024-08-29T15:33:04.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:04 smithi027 ceph-mon[98970]: pgmap v1192: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 2651/105192 objects degraded (2.520%); 8852/105192 objects misplaced (8.415%); 3.0 MiB/s, 48 objects/s recovering 2024-08-29T15:33:04.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:04.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:04.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:04.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:04.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:04.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:04.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:04.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:04.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:04.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:04.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:05.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:05.031 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:33:05.031 INFO:teuthology.orchestra.run.smithi171.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:33:05.031 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:33:05.072 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:05.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:05.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:05.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:05.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:05.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:05.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:05.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:05.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:05.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:05.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:05.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:05.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:05.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:05.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:05.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:05.756 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:05.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:05.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:05.910 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:05.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:05.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:06.012 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:06.046 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:06.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:06.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:06.148 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:06.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:06.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:06.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:06.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:06 smithi027 ceph-mon[93921]: pgmap v1193: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 2808/98046 objects degraded (2.864%); 8073/98046 objects misplaced (8.234%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:06 smithi027 ceph-mon[98970]: pgmap v1193: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 2808/98046 objects degraded (2.864%); 8073/98046 objects misplaced (8.234%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:06.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:06.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:06.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:06.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:06.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:06 smithi171 ceph-mon[90031]: pgmap v1193: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 2808/98046 objects degraded (2.864%); 8073/98046 objects misplaced (8.234%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:06.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:06.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:06.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:06.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:06.751 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:06.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:06.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:06.882 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:06.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:06.984 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:07.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:07.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:07.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:07.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:07.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:07.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:07.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2808/98046 objects degraded (2.864%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:07.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2808/98046 objects degraded (2.864%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:07.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:07.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2808/98046 objects degraded (2.864%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:07.288 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:07.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:07.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:07.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:07.435 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:07.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:07.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:07.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:07.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:07.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:07.748 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:07.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:07.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:07.899 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:07.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:07.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:07.952 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:33:07.953 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:33:08.001 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:08.016 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:33:08.016 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:33:08.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:08.061 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:33:08.061 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:33:08.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:08.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:08.202 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:33:08.202 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:33:08.245 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:33:08.245 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:33:08.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:08.291 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:08 smithi027 ceph-mon[93921]: pgmap v1194: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 2808/95778 objects degraded (2.932%); 8073/95778 objects misplaced (8.429%); 4.7 MiB/s, 76 objects/s recovering 2024-08-29T15:33:08.291 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2796119778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:08.291 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:08 smithi027 ceph-mon[98970]: pgmap v1194: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 2808/95778 objects degraded (2.932%); 8073/95778 objects misplaced (8.429%); 4.7 MiB/s, 76 objects/s recovering 2024-08-29T15:33:08.292 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2796119778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:08.292 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:33:08.292 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:33:08.321 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:33:08.321 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:33:08.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:08.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:08.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:33:08.368 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:33:08.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:08.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:08 smithi171 ceph-mon[90031]: pgmap v1194: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 2808/95778 objects degraded (2.932%); 8073/95778 objects misplaced (8.429%); 4.7 MiB/s, 76 objects/s recovering 2024-08-29T15:33:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2796119778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:08.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:08.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:08.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:08.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:08.718 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:08.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:08.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:08.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:08.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:08.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:08.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:08.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:09.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:09.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:09.103 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:09.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:09.104 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:33:09.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:09.231 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:09.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:09.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:09.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3015813413' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:09.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:33:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:33:09] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-29T15:33:09.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3015813413' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:09.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:09.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:09.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:09.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:09.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:09.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:09.510 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:09.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:09.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3015813413' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:09.613 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:09.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:09.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:09.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:09.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:09.790 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:09.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:09.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:09.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:09.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:09.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:10.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:10.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:10.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:10.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:10.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:10.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:10.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:10.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:10.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:10.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:10.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:10.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:10.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:10.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:10 smithi171 ceph-mon[90031]: pgmap v1195: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 2952/93012 objects degraded (3.174%); 7714/93012 objects misplaced (8.294%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:10 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:33:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:10 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:33:10.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:10 smithi171 ceph-mon[90031]: osdmap e333: 8 total, 8 up, 8 in 2024-08-29T15:33:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[93921]: pgmap v1195: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 2952/93012 objects degraded (3.174%); 7714/93012 objects misplaced (8.294%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:33:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:33:10.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[93921]: osdmap e333: 8 total, 8 up, 8 in 2024-08-29T15:33:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[98970]: pgmap v1195: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 2952/93012 objects degraded (3.174%); 7714/93012 objects misplaced (8.294%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:33:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:33:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:10 smithi027 ceph-mon[98970]: osdmap e333: 8 total, 8 up, 8 in 2024-08-29T15:33:10.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:10.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:10.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:10.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:10.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:10.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:10.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:10.789 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:33:10.790 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:33:10.790 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-08-29T15:33:10.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-08-29T15:33:10.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:10.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:10.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:11.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:11.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:11.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:11.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:11.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:11.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:11.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:11.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:11.383 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:11.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:11.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:11.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:11.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:11.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:11.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:11.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:11.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:11.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:11.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:11.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:11.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:11.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:11.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:11.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:11.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:12.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:12.076 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:12.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:12.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:12.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:12.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:12.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:12.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:12.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:12 smithi027 ceph-mon[98970]: pgmap v1197: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 2952/89250 objects degraded (3.308%); 7347/89250 objects misplaced (8.232%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 2952/89250 objects degraded (3.308%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:12 smithi027 ceph-mon[93921]: pgmap v1197: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 2952/89250 objects degraded (3.308%); 7347/89250 objects misplaced (8.232%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:12.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 2952/89250 objects degraded (3.308%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:12.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:12.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:12.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:12.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:12.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:12.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:12 smithi171 ceph-mon[90031]: pgmap v1197: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 2952/89250 objects degraded (3.308%); 7347/89250 objects misplaced (8.232%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 2952/89250 objects degraded (3.308%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:12.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:12.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:12.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:12.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:12.745 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:12.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:12.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:12.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:12.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:12.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:12.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:12.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:13.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:13.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:13.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:13.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:13.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:13.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:13.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:13.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:13.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:13.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:13.692 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:13.722 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:13.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:13.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:13.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:13.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:13.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: pgmap v1198: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2952/87198 objects degraded (3.385%); 7347/87198 objects misplaced (8.426%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:33:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:33:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: pgmap v1198: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2952/87198 objects degraded (3.385%); 7347/87198 objects misplaced (8.426%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: pgmap v1198: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2952/87198 objects degraded (3.385%); 7347/87198 objects misplaced (8.426%); 5.2 MiB/s, 83 objects/s recovering 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:14.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:14.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:14.127 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:14.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:14.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:14.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:14.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:14.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:14.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:14.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:14.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:14.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:14.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:14.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:14.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:14.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:14.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:14.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:14.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:14.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:14.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:14.900 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:14.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:14.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:14.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:14.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:14.939 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:33:14.939 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:33:14.939 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:33:14.939 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:33:15.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:15.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:15.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:15.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:15.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:15.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:15.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:15.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:15.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:15.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:15.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:15.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:15.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:15.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:15.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:15.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:15.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:15.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:15.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:15.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:15.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:33:15.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:15.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:15.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:15.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:15.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:16.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:15 smithi027 ceph-mon[93921]: pgmap v1199: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3104/80367 objects degraded (3.862%); 6571/80367 objects misplaced (8.176%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:33:16.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:15 smithi027 ceph-mon[98970]: pgmap v1199: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3104/80367 objects degraded (3.862%); 6571/80367 objects misplaced (8.176%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:33:16.070 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:16.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:16.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:16.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:16.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:16.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:16.221 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:15 smithi171 ceph-mon[90031]: pgmap v1199: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3104/80367 objects degraded (3.862%); 6571/80367 objects misplaced (8.176%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:33:16.301 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:16.328 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:16.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:16.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:16.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:16.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:16.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:16.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:16.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:16.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:33:16.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:16.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:16.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:16.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:16.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:16.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:16.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:16.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:16.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:17.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:17.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:17.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:17.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:17.276 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:17.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:17.424 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:17.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:17.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:17.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:17.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:17.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:17.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:17.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:17.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:17.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:17.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:18.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:18.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:18.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:18.240 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:18.264 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:18.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:17 smithi171 ceph-mon[90031]: pgmap v1200: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 3104/78168 objects degraded (3.971%); 6571/78168 objects misplaced (8.406%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:33:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3104/78168 objects degraded (3.971%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:17 smithi027 ceph-mon[93921]: pgmap v1200: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 3104/78168 objects degraded (3.971%); 6571/78168 objects misplaced (8.406%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:33:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3104/78168 objects degraded (3.971%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:17 smithi027 ceph-mon[98970]: pgmap v1200: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 3104/78168 objects degraded (3.971%); 6571/78168 objects misplaced (8.406%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:33:18.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3104/78168 objects degraded (3.971%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:18.365 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:18.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:18.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:18.488 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:18.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:18.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:18.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:18.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:18.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:18.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:18.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:18.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:18.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:18.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:18.908 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:18.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:18.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:19.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:19.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:19.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:19.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:19.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:19.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:33:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:33:19] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.51.0" 2024-08-29T15:33:19.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:19.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:19.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:19.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:19.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:19.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:19.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:19.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:19.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:19.799 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:19.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:19.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:19.973 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:19.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:19 smithi027 ceph-mon[93921]: pgmap v1201: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3275/75336 objects degraded (4.347%); 6196/75336 objects misplaced (8.224%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:33:20.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:19 smithi027 ceph-mon[98970]: pgmap v1201: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3275/75336 objects degraded (4.347%); 6196/75336 objects misplaced (8.224%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:33:20.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:20.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:20.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:20.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:20.263 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:20.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:19 smithi171 ceph-mon[90031]: pgmap v1201: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3275/75336 objects degraded (4.347%); 6196/75336 objects misplaced (8.224%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:33:20.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:20.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:20.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:20.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:20.493 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:20.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:20.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:20.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:20.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:20.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:33:20.672 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:20.716 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:20.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:20.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:20.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:20.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:20.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:20.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:20.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:20.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:21.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:21.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:21.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:21.171 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:21.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:21.273 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:21.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:21.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:21.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:21.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:21.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:21.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:21.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:21.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:21.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:21.586 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:33:21.586 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:33:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:33:21.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:21.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:21.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:21.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:21.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:21.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:21.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:21.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:21.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:22.011 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:22.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:22.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:22.140 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:21 smithi027 ceph-mon[98970]: pgmap v1202: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3275/71136 objects degraded (4.604%); 5786/71136 objects misplaced (8.134%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:22.140 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:21 smithi027 ceph-mon[93921]: pgmap v1202: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3275/71136 objects degraded (4.604%); 5786/71136 objects misplaced (8.134%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:22.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:22.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:22.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:21 smithi171 ceph-mon[90031]: pgmap v1202: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3275/71136 objects degraded (4.604%); 5786/71136 objects misplaced (8.134%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:22.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:22.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:22.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:22.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:22.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:22.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:22.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:22.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:22.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:22.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:22.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:22.831 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:22.933 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:22.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:22.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:23.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:23.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:23.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:23.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:23.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:23.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:23.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3275/71136 objects degraded (4.604%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3275/71136 objects degraded (4.604%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:23.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3275/71136 objects degraded (4.604%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:23.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:23.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:23.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:23.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:23.476 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:23.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:23.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:23.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:23.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:23.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:23.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:23.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:23.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:23.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:23.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:23.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:23.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:23.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:23.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:24.036 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:24.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:24.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:24.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:23 smithi171 ceph-mon[90031]: pgmap v1203: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3275/68904 objects degraded (4.753%); 5786/68904 objects misplaced (8.397%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:33:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:24.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:24.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:23 smithi027 ceph-mon[98970]: pgmap v1203: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3275/68904 objects degraded (4.753%); 5786/68904 objects misplaced (8.397%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:33:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:23 smithi027 ceph-mon[93921]: pgmap v1203: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3275/68904 objects degraded (4.753%); 5786/68904 objects misplaced (8.397%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:33:24.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:24.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:24.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:24.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:24.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:24.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:24.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:24.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:24.822 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:24.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:24.953 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:24.988 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:24.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:25.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:25.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:25.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:25.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:25.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:25.257 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:25.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:25.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:25.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:25.396 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:25.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:25.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:25.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:25.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:25.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:25.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:25.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:25.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:25.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:25.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:25.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:25.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:25.974 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:25 smithi027 ceph-mon[93921]: pgmap v1204: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 3449/62010 objects degraded (5.562%); 5010/62010 objects misplaced (8.079%); 6.4 MiB/s, 101 objects/s recovering 2024-08-29T15:33:25.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:25 smithi027 ceph-mon[98970]: pgmap v1204: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 3449/62010 objects degraded (5.562%); 5010/62010 objects misplaced (8.079%); 6.4 MiB/s, 101 objects/s recovering 2024-08-29T15:33:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:25.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:25.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:26.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:26.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:26.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:26.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:26.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:26.242 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:33:26.242 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:33:26.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-08-29T15:33:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:25 smithi171 ceph-mon[90031]: pgmap v1204: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 3449/62010 objects degraded (5.562%); 5010/62010 objects misplaced (8.079%); 6.4 MiB/s, 101 objects/s recovering 2024-08-29T15:33:26.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:26.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:26.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:26.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:26.406 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:26.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:26.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:26.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:26.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:26.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:26.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:26.716 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:26.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:26.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:26.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:26.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:26.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:27.036 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:27.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:27.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:27.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:27.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:27.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:27.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:27.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:27.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:27.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:27.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:27.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:27.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:27.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:27.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:27.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:27.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:27.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:27.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:27.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:27.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:27.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:27.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:28.004 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:28.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:27 smithi027 ceph-mon[98970]: pgmap v1205: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3449/59727 objects degraded (5.775%); 5010/59727 objects misplaced (8.388%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:33:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3449/59727 objects degraded (5.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3884996901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:28.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:27 smithi027 ceph-mon[93921]: pgmap v1205: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3449/59727 objects degraded (5.775%); 5010/59727 objects misplaced (8.388%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:33:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3449/59727 objects degraded (5.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3884996901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:28.088 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:28.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:28.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:28.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:27 smithi171 ceph-mon[90031]: pgmap v1205: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3449/59727 objects degraded (5.775%); 5010/59727 objects misplaced (8.388%); 4.2 MiB/s, 67 objects/s recovering 2024-08-29T15:33:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3449/59727 objects degraded (5.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3884996901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:28.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:28.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:28.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:28.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:28.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:28.556 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:28.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:28.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:28.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:28.787 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:28.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:28.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:28.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:28.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:28.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:28.953 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:33:29.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:29.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:29.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:29.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:29.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:29.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:29.231 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:33:29.231 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:33:29.232 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:33:29.233 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:33:29.233 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:33:29.233 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:33:29.233 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:33:29.233 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:33:29.233 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:33:29.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:28 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2581243068' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:33:29.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:28 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2581243068' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:33:29.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:29.234 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:33:29] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-29T15:33:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:28 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2581243068' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:33:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:29.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:29.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:29.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:29.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:29.487 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:29.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:29.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:29.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:29.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:29.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:29.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:29.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:29.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:29.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:29.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:29.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:29.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:29.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:30.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:30.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:30.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:30.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:29 smithi171 ceph-mon[90031]: pgmap v1206: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3634/56970 objects degraded (6.379%); 4597/56970 objects misplaced (8.069%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:29 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:33:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2581243068' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:33:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:29 smithi171 ceph-mon[90031]: osdmap e334: 8 total, 8 up, 8 in 2024-08-29T15:33:30.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:30.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:30.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[93921]: pgmap v1206: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3634/56970 objects degraded (6.379%); 4597/56970 objects misplaced (8.069%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:30.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:33:30.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2581243068' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:33:30.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[93921]: osdmap e334: 8 total, 8 up, 8 in 2024-08-29T15:33:30.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[98970]: pgmap v1206: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3634/56970 objects degraded (6.379%); 4597/56970 objects misplaced (8.069%); 5.4 MiB/s, 86 objects/s recovering 2024-08-29T15:33:30.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:33:30.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2581243068' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:33:30.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:29 smithi027 ceph-mon[98970]: osdmap e334: 8 total, 8 up, 8 in 2024-08-29T15:33:30.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:30.402 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:30.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:30.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:30.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:30.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:30.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:30.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:30.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:30.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:30.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:30.906 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:30.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:30.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:31.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:31.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:31.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:31.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:31.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:31.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:31.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:31.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:31.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:31.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:31.341 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:31.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:31.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:31.497 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:31.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:31.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:31.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:31.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:31.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:31.776 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:31.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:31.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:31.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:31.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:31.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:31.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:32.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:32.039 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:32.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:31 smithi027 ceph-mon[98970]: pgmap v1208: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 3634/53127 objects degraded (6.840%); 4200/53127 objects misplaced (7.906%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:33:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:31 smithi027 ceph-mon[93921]: pgmap v1208: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 3634/53127 objects degraded (6.840%); 4200/53127 objects misplaced (7.906%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:33:32.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:32.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:31 smithi171 ceph-mon[90031]: pgmap v1208: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 3634/53127 objects degraded (6.840%); 4200/53127 objects misplaced (7.906%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:33:32.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:32.322 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:32.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:32.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:32.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:32.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:32.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:32.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:32.717 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:32.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:32.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:32.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:32.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:32.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:32.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:33.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:33.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:33.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:33:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:33.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:33.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:33.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:33.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:33.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3634/53127 objects degraded (6.840%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:33.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3634/53127 objects degraded (6.840%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:33.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3634/53127 objects degraded (6.840%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:33.356 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:33.382 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:33.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:33.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:33.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:33.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:33.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:33.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:33.642 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:33.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:33.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:33.769 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:33.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:33.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:33.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:33.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:33.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:34.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:34.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:34.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:34.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:34.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:33 smithi027 ceph-mon[98970]: pgmap v1209: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3634/51177 objects degraded (7.101%); 4200/51177 objects misplaced (8.207%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:33:34.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:33 smithi027 ceph-mon[93921]: pgmap v1209: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3634/51177 objects degraded (7.101%); 4200/51177 objects misplaced (8.207%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:33:34.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:34.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:34.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:34.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:33 smithi171 ceph-mon[90031]: pgmap v1209: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3634/51177 objects degraded (7.101%); 4200/51177 objects misplaced (8.207%); 5.3 MiB/s, 84 objects/s recovering 2024-08-29T15:33:34.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:34.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:34.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:34.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:34.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:34.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:34.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:34.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:34.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:34.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:34.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:34.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:34.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:34.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:34.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:34.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:34.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:35.007 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:35.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:35.151 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:35.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:35.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:35.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:35.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:35.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:35.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:35.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:35.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:35.562 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:35.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:35.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:35.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:35.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:35.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:35.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:35.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:33:35.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:35.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:35.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:36.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:36.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:36.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:36.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:36.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:36.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:36.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:35 smithi171 ceph-mon[90031]: pgmap v1210: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 923 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3761/44172 objects degraded (8.514%); 3464/44172 objects misplaced (7.842%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:33:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:35 smithi027 ceph-mon[98970]: pgmap v1210: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 923 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3761/44172 objects degraded (8.514%); 3464/44172 objects misplaced (7.842%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:33:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:35 smithi027 ceph-mon[93921]: pgmap v1210: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 923 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 3761/44172 objects degraded (8.514%); 3464/44172 objects misplaced (7.842%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:33:36.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:36.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:36.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:36.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:36.474 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:36.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:36.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:36.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:36.636 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:36.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:36.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:36.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:36.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:36.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:36.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:36.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:36.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:36.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:36.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:37.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:37.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:37.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:37.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:37.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:37.266 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:37.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:37.368 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:37.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:37.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:37.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:37.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:37.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:37.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:37.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:37.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:37.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:37.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:37.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:38.007 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:33:38.007 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:33:38.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:38.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:38.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:38.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:38.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:37 smithi171 ceph-mon[90031]: pgmap v1211: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 873 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 3761/41748 objects degraded (9.009%); 3464/41748 objects misplaced (8.297%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:33:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3761/41748 objects degraded (9.009%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:38.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:38.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:37 smithi027 ceph-mon[98970]: pgmap v1211: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 873 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 3761/41748 objects degraded (9.009%); 3464/41748 objects misplaced (8.297%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:33:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3761/41748 objects degraded (9.009%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:37 smithi027 ceph-mon[93921]: pgmap v1211: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 873 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 3761/41748 objects degraded (9.009%); 3464/41748 objects misplaced (8.297%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:33:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3761/41748 objects degraded (9.009%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:38.373 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:38.399 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:38.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:38.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:38.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:38.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:38.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:38.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:38.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:38.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:38.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:38.797 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:38.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:38.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:38.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:38.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:39.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:39.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:33:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:33:39] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-29T15:33:39.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:39.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:39.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:39.201 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:39.226 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:39.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:39.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:39.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:39.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:39.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:39.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:39.417 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:39.418 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:39.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:33:39.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:39.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:39.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:39.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:39.586 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:39.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:39.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:39.716 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:39.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:39.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:39.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:39.899 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:39.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:40.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:40.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:40.124 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:39 smithi027 ceph-mon[98970]: pgmap v1212: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 815 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 3880/38982 objects degraded (9.953%); 3111/38982 objects misplaced (7.981%); 5.6 MiB/s, 88 objects/s recovering 2024-08-29T15:33:40.124 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:39 smithi027 ceph-mon[93921]: pgmap v1212: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 815 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 3880/38982 objects degraded (9.953%); 3111/38982 objects misplaced (7.981%); 5.6 MiB/s, 88 objects/s recovering 2024-08-29T15:33:40.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:40.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:40.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:39 smithi171 ceph-mon[90031]: pgmap v1212: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 815 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 3880/38982 objects degraded (9.953%); 3111/38982 objects misplaced (7.981%); 5.6 MiB/s, 88 objects/s recovering 2024-08-29T15:33:40.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:40.396 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:40.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:40.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:40.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:40.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:40.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:40.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:40.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:40.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:40.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:40.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:40.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:40.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:40.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:41.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:41.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:41.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:41.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:41.222 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:41.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:41.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:41.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:41.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:41.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:41.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:41.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:41.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:41.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:41.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:41.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:41.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:41.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:41.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:41.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:41.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:41.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:41.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:41.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:41.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:41.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:41.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:42.007 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:42.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:42.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:42.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:42.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:42.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:42.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:42.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:41 smithi171 ceph-mon[90031]: pgmap v1213: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 733 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 3880/35034 objects degraded (11.075%); 2719/35034 objects misplaced (7.761%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:33:42.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:42.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:41 smithi027 ceph-mon[93921]: pgmap v1213: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 733 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 3880/35034 objects degraded (11.075%); 2719/35034 objects misplaced (7.761%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:33:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:41 smithi027 ceph-mon[98970]: pgmap v1213: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 733 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 3880/35034 objects degraded (11.075%); 2719/35034 objects misplaced (7.761%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:33:42.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:42.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:42.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:42.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:42.581 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:42.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:42.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:42.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:42.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:42.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:42.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:42.877 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:42.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:42.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:42.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:43.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:43.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:43.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:43.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:43.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:43.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3880/35034 objects degraded (11.075%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:43.290 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:43.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3880/35034 objects degraded (11.075%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:43.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3880/35034 objects degraded (11.075%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:43.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:43.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:43.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:43.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:43.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:43.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:43.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:43.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:43.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:43.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:43.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:43.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:43.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:43 smithi027 ceph-mon[93921]: pgmap v1214: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 688 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3880/32898 objects degraded (11.794%); 2719/32898 objects misplaced (8.265%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:33:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:43 smithi027 ceph-mon[98970]: pgmap v1214: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 688 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3880/32898 objects degraded (11.794%); 2719/32898 objects misplaced (8.265%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:33:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:44.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:44.083 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:44.112 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:44.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:44.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:44.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:44.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:44.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:44.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:43 smithi171 ceph-mon[90031]: pgmap v1214: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 688 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 3880/32898 objects degraded (11.794%); 2719/32898 objects misplaced (8.265%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:33:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:33:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:44.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:44.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:44.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:44.474 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:44.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:44.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:44.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:33:44.612 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:33:44.613 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:33:44.613 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:33:44.613 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:33:44.627 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:44.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:44.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:44.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:44.754 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:44.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:44.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:44.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:44.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:44.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:44.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:45.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:45.135 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:45.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:45.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:45.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:45.396 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:45.419 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:45.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:45.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:45.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:45.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:45.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:45.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:45.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:45.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:45.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:46.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:46.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:46.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:45 smithi027 ceph-mon[98970]: pgmap v1215: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 547 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4008/26103 objects degraded (15.355%); 1911/26103 objects misplaced (7.321%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:33:46.123 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3555345912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:46.124 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:45 smithi027 ceph-mon[93921]: pgmap v1215: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 547 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4008/26103 objects degraded (15.355%); 1911/26103 objects misplaced (7.321%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:33:46.124 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3555345912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:46.181 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:33:46.181 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:33:46.181 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:33:46.181 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:33:46.184 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:33:46.184 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:33:46.184 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:33:46.184 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:33:46.215 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:33:46.215 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:33:46.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:46.255 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:33:46.255 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:33:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:45 smithi171 ceph-mon[90031]: pgmap v1215: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 547 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4008/26103 objects degraded (15.355%); 1911/26103 objects misplaced (7.321%); 7.0 MiB/s, 112 objects/s recovering 2024-08-29T15:33:46.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3555345912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:46.268 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:46.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:46.298 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:33:46.298 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:33:46.342 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:33:46.343 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:33:46.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:46.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:46.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:46.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:46.497 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:46.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:46.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:46.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:46.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:46.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:46.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:46.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:46.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:33:46.779 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:33:46.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-08-29T15:33:46.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:46.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:46.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:46.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:46.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:46.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:47.022 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:47.024 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:33:47.047 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:47.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:47.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:47.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:47.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:47.204 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:47.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/395553387' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:47.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:46 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:47.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:47.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/395553387' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:47.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:46 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/395553387' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:47.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:46 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:33:47.352 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:47.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:47.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:47.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:47.503 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:47.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:47.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:47.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:47.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:47.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:47.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:47.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:47.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:47.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:47.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:47.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:48.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:48.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:48.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:48.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:48.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:48.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:48.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:47 smithi171 ceph-mon[90031]: pgmap v1216: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 495 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 4008/23643 objects degraded (16.952%); 1911/23643 objects misplaced (8.083%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:33:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:47 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:33:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 4008/26103 objects degraded (15.355%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:47 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:33:48.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:47 smithi171 ceph-mon[90031]: osdmap e335: 8 total, 8 up, 8 in 2024-08-29T15:33:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:48 smithi027 ceph-mon[98970]: pgmap v1216: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 495 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 4008/23643 objects degraded (16.952%); 1911/23643 objects misplaced (8.083%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:33:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:48 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:33:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:48 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 4008/26103 objects degraded (15.355%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:48 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:33:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:48 smithi027 ceph-mon[98970]: osdmap e335: 8 total, 8 up, 8 in 2024-08-29T15:33:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:47 smithi027 ceph-mon[93921]: pgmap v1216: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 495 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 4008/23643 objects degraded (16.952%); 1911/23643 objects misplaced (8.083%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:33:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:47 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:33:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 4008/26103 objects degraded (15.355%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:47 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:33:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:47 smithi027 ceph-mon[93921]: osdmap e335: 8 total, 8 up, 8 in 2024-08-29T15:33:48.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:48.301 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:48.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:48.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:48.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:48.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:48.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:48.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:48.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:48.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:48.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:48.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:48.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:48.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:48.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:48.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:48.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:48.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:48.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:49.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:49.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:49.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:49.045 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:49.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:49.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:49.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:49.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:49.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:49.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:49.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:49.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:33:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:33:49] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-29T15:33:49.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:49.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:49.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:49.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:49.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:49.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:49.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:33:49.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:49.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:49.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:49.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:49.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:49.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:49.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:49.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:50.020 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:50.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:50.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:50.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:50.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:49 smithi171 ceph-mon[90031]: pgmap v1218: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 434 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 4158/20709 objects degraded (20.078%); 1524/20709 objects misplaced (7.359%); 5.7 MiB/s, 91 objects/s recovering 2024-08-29T15:33:50.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:50.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:33:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:50 smithi027 ceph-mon[98970]: pgmap v1218: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 434 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 4158/20709 objects degraded (20.078%); 1524/20709 objects misplaced (7.359%); 5.7 MiB/s, 91 objects/s recovering 2024-08-29T15:33:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:50 smithi027 ceph-mon[93921]: pgmap v1218: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 434 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 4158/20709 objects degraded (20.078%); 1524/20709 objects misplaced (7.359%); 5.7 MiB/s, 91 objects/s recovering 2024-08-29T15:33:50.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:50.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:50.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:50.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:50.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:50.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:50.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:50.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:50.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:50.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.827 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:50.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:50.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:50.973 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:50.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:51.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:51.128 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:51.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:51.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:51.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:51.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:51.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:51.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:51.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:51.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:51.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:51.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:51.547 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:51.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:51.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:51.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:51.673 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:51.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:51.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:51.799 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:51.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:51.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:51.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:51.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:51.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:51.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:52.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:52.099 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:52.102 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:52.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:52.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:52.128 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:52 smithi027 ceph-mon[93921]: pgmap v1219: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 350 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 4158/16659 objects degraded (24.959%); 1172/16659 objects misplaced (7.035%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:33:52.133 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:52 smithi027 ceph-mon[98970]: pgmap v1219: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 350 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 4158/16659 objects degraded (24.959%); 1172/16659 objects misplaced (7.035%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:33:52.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:52.229 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:52.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:52.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:52 smithi171 ceph-mon[90031]: pgmap v1219: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 350 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 4158/16659 objects degraded (24.959%); 1172/16659 objects misplaced (7.035%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:33:52.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:52.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:52.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:52.442 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106255 7.5 7:a92a03f2:::benchmark_data_smithi027_271924_object106254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.566708+0000", 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.87278018399999, 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17617830000000001, 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106255 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.566708+0000", 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.566708+0000", 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.566710+0000", 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.491e-06 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.566788+0000", 2024-08-29T15:33:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8052000000000004e-05 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.566801+0000", 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3298999999999999e-05 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.566808+0000", 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5390000000000002e-06 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.567328+0000", 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00051930800000000001 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.567450+0000", 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121792 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.567575+0000", 2024-08-29T15:33:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012492100000000001 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.572069+0000", 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044944290000000003 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573191+0000", 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001122428 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.742675+0000", 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.169483155 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.742847+0000", 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000172122 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.742886+0000", 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9764000000000001e-05 2024-08-29T15:33:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106258 7.5 7:a9a075f7:::benchmark_data_smithi027_271924_object106257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.573091+0000", 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.86639697300001, 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17006568799999999, 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106258 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573091+0000", 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573091+0000", 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573093+0000", 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.296e-06 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573116+0000", 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3289999999999999e-05 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573126+0000", 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8740000000000007e-06 2024-08-29T15:33:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573132+0000", 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5089999999999999e-06 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573224+0000", 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1321000000000002e-05 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573352+0000", 2024-08-29T15:33:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128305 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.573487+0000", 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013555800000000001 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577428+0000", 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039406170000000004 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.577445+0000", 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6495999999999999e-05 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743073+0000", 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.165628624 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743123+0000", 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9870000000000002e-05 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743157+0000", 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3927999999999998e-05 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106261 7.9 7:9bd2556c:::benchmark_data_smithi027_271924_object106260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.574186+0000", 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.865302383, 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.167734418, 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106261 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574186+0000", 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574186+0000", 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574187+0000", 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2299999999999999e-07 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574249+0000", 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1728000000000003e-05 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574266+0000", 2024-08-29T15:33:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7467999999999999e-05 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574273+0000", 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9600000000000003e-06 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574336+0000", 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3213999999999995e-05 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574453+0000", 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116427 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.574579+0000", 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012620599999999999 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581148+0000", 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065688919999999998 2024-08-29T15:33:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.582364+0000", 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012167600000000001 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741722+0000", 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15935739800000001 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741872+0000", 2024-08-29T15:33:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015015499999999999 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741920+0000", 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8287e-05 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:106262 7.9 7:939d7ac7:::benchmark_data_smithi027_271924_object106261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:29.575241+0000", 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.86424696500001, 2024-08-29T15:33:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16818393700000001, 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106262 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575241+0000", 2024-08-29T15:33:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575241+0000", 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575242+0000", 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0699999999999999e-06 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575265+0000", 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2498999999999999e-05 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575275+0000", 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0356999999999999e-05 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.575282+0000", 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6900000000000003e-06 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578197+0000", 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002915141 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578348+0000", 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000150822 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.578504+0000", 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015640700000000001 2024-08-29T15:33:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.581661+0000", 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031568989999999999 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.582370+0000", 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070890000000000005 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.741959+0000", 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15958888299999999 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743380+0000", 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014212070000000001 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:29.743425+0000", 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5062000000000003e-05 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107330 7.5 7:afe6ca22:::benchmark_data_smithi027_271924_object107329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.030085+0000", 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.409403771, 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18882079299999999, 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107330 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030085+0000", 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030085+0000", 2024-08-29T15:33:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030085+0000", 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7000000000000003e-07 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030111+0000", 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5040000000000001e-05 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030122+0000", 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1202999999999999e-05 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030129+0000", 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3950000000000003e-06 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030209+0000", 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0060000000000003e-05 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030340+0000", 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013098600000000001 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030475+0000", 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013435899999999999 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.040457+0000", 2024-08-29T15:33:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099824749999999993 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.043916+0000", 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003459036 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.218767+0000", 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17485062300000001 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.218833+0000", 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5906000000000002e-05 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.218905+0000", 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2739999999999998e-05 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107334 7.5 7:add079f7:::benchmark_data_smithi027_271924_object107333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.030407+0000", 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.40908162700001, 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.194597518, 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107334 2024-08-29T15:33:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030407+0000", 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030407+0000", 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030407+0000", 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2399999999999998e-07 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030436+0000", 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8883000000000001e-05 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030446+0000", 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3449999999999995e-06 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.030452+0000", 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.708e-06 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.036612+0000", 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061602719999999996 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.036751+0000", 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013884800000000001 2024-08-29T15:33:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.036892+0000", 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014108100000000001 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.043969+0000", 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070766880000000002 2024-08-29T15:33:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.044045+0000", 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5603000000000001e-05 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.224906+0000", 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.180861153 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.224959+0000", 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3134999999999999e-05 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.225004+0000", 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5178e-05 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107336 7.5 7:ac11cbb4:::benchmark_data_smithi027_271924_object107335:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.031114+0000", 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.40837411300001, 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19405344399999999, 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107336 2024-08-29T15:33:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031114+0000", 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031114+0000", 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031115+0000", 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7199999999999997e-07 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031136+0000", 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0948999999999999e-05 2024-08-29T15:33:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031147+0000", 2024-08-29T15:33:52.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0917000000000001e-05 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.031154+0000", 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1029999999999996e-06 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.040492+0000", 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093378639999999995 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.040708+0000", 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021579600000000001 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.040858+0000", 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015010499999999999 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.044030+0000", 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031721890000000002 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.047248+0000", 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032182930000000001 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.225063+0000", 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.177815111 2024-08-29T15:33:52.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.225130+0000", 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6883000000000003e-05 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.225168+0000", 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7261999999999997e-05 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:107338 7.5 7:ad52cad6:::benchmark_data_smithi027_271924_object107337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:31.035446+0000", 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 201.40404210700001, 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18985079499999999, 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 107338 2024-08-29T15:33:52.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035446+0000", 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035446+0000", 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035448+0000", 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.491e-06 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035473+0000", 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5477e-05 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035490+0000", 2024-08-29T15:33:52.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7311000000000001e-05 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.035498+0000", 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2139999999999999e-06 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.044075+0000", 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085775069999999998 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.044211+0000", 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135914 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.044351+0000", 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139586 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.047373+0000", 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030226620000000002 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.051127+0000", 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037535649999999999 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.225205+0000", 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17407847900000001 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.225263+0000", 2024-08-29T15:33:52.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7352999999999998e-05 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:31.225297+0000", 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4236000000000002e-05 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110056 7.9 7:939c17ba:::benchmark_data_smithi027_271924_object110055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.607461+0000", 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 197.832027479, 2024-08-29T15:33:52.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.267845159, 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110056 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607461+0000", 2024-08-29T15:33:52.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607461+0000", 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607462+0000", 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4500000000000001e-06 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607488+0000", 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5347999999999999e-05 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607498+0000", 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0738e-05 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607506+0000", 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1600000000000001e-06 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607605+0000", 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9621e-05 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607746+0000", 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000140975 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.607908+0000", 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016179399999999999 2024-08-29T15:33:52.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613268+0000", 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053602010000000002 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618098+0000", 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004830048 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.875109+0000", 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25701096600000001 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.875257+0000", 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014796500000000001 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.875306+0000", 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8893000000000001e-05 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.505 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110063 7.9 7:9f80d71b:::benchmark_data_smithi027_271924_object110062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.613302+0000", 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 197.826186423, 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.262878838, 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.506 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110063 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613302+0000", 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613302+0000", 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613303+0000", 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.432e-06 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613327+0000", 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4114e-05 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613343+0000", 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5277999999999998e-05 2024-08-29T15:33:52.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613351+0000", 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8590000000000005e-06 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613441+0000", 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0384999999999996e-05 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613566+0000", 2024-08-29T15:33:52.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125458 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.613692+0000", 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012515200000000001 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618197+0000", 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045052800000000004 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618610+0000", 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041330000000000002 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.876083+0000", 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25747285199999997 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.876146+0000", 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2626999999999995e-05 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.876181+0000", 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5101000000000002e-05 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110072 7.5 7:aad7cccc:::benchmark_data_smithi027_271924_object110071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.621071+0000", 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 197.818417625, 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34367331099999998, 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.512 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110072 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621071+0000", 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621071+0000", 2024-08-29T15:33:52.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621072+0000", 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4419999999999999e-06 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621096+0000", 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3625000000000002e-05 2024-08-29T15:33:52.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621110+0000", 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4277000000000001e-05 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621117+0000", 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7229999999999998e-06 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621181+0000", 2024-08-29T15:33:52.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3983e-05 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621324+0000", 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000143196 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.621460+0000", 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013589599999999999 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626512+0000", 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050519069999999996 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626538+0000", 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6420000000000001e-05 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.964518+0000", 2024-08-29T15:33:52.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33797954600000002 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.964700+0000", 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018241399999999999 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.964744+0000", 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3881999999999998e-05 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148243 7.6 7:68483b54:::benchmark_data_smithi027_271924_object31902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:25.315460+0000", 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 147.12402870599999, 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.304888842, 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148243 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315460+0000", 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315460+0000", 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315462+0000", 2024-08-29T15:33:52.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0229999999999999e-06 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315469+0000", 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.396e-06 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315489+0000", 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9533000000000001e-05 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315499+0000", 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0545e-05 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315569+0000", 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9507999999999999e-05 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.315580+0000", 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0915e-05 2024-08-29T15:33:52.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.196201+0000", 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88062173799999999 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.196211+0000", 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4080000000000004e-06 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.458265+0000", 2024-08-29T15:33:52.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2620544849999999 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.458394+0000", 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012898099999999999 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,7", 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.458530+0000", 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013598199999999999 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.464126+0000", 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055962929999999996 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.464179+0000", 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3090999999999997e-05 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.466793+0000", 2024-08-29T15:33:52.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026134959999999999 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.620144+0000", 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.153351442 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.620233+0000", 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8417000000000004e-05 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.620348+0000", 2024-08-29T15:33:52.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011558899999999999 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148195 7.5 7:a56df394:::benchmark_data_smithi027_271924_object31854:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e311)", 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:26.186993+0000", 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 146.252495372, 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.288557521, 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.530 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148195 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.186993+0000", 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.186993+0000", 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.186995+0000", 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.84e-06 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187001+0000", 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7939999999999999e-06 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187013+0000", 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2495e-05 2024-08-29T15:33:52.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187022+0000", 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6789999999999994e-06 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187361+0000", 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033887300000000001 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:33:52.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187369+0000", 2024-08-29T15:33:52.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6139999999999992e-06 2024-08-29T15:33:52.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.475461+0000", 2024-08-29T15:33:52.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.288091879 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.475550+0000", 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9346999999999997e-05 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148242 7.5 7:a401fbac:::benchmark_data_smithi027_271924_object31901:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e311)", 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:26.187038+0000", 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 146.25245015499999, 2024-08-29T15:33:52.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2973670589999999, 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148242 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187038+0000", 2024-08-29T15:33:52.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187038+0000", 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187039+0000", 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.44e-07 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187042+0000", 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1209999999999998e-06 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187054+0000", 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2296999999999999e-05 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187061+0000", 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6100000000000002e-06 2024-08-29T15:33:52.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187395+0000", 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033415300000000001 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.187402+0000", 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4590000000000001e-06 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.475784+0000", 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.288381709 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.475912+0000", 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127668 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2,7", 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.476042+0000", 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000130512 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.480957+0000", 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004914486 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.484140+0000", 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003182973 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.484197+0000", 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7382e-05 2024-08-29T15:33:52.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.484220+0000", 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2884999999999999e-05 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.484276+0000", 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5841000000000003e-05 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:27.484405+0000", 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129419 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34116.0:64 2.1 2:88c1567c:::rbd_trash_purge_schedule:head [omap-get-vals] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e313)", 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:28.838362+0000", 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 143.601126123, 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.46107675599999998, 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.544 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34116", 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1227943899", 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 64 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838362+0000", 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838362+0000", 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838363+0000", 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.22e-06 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838374+0000", 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0353999999999999e-05 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838387+0000", 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3585e-05 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838394+0000", 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5529999999999998e-06 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838514+0000", 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120254 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:28.838523+0000", 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7180000000000002e-06 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:29.298977+0000", 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.46045381099999999 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:29.299439+0000", 2024-08-29T15:33:52.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000462261 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:166124 7.9 7:95f42971:::benchmark_data_smithi027_271924_object49783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:01.171171+0000", 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 111.268316867, 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.198513881, 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166124 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171171+0000", 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171171+0000", 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171173+0000", 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9319999999999999e-06 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171180+0000", 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0269999999999998e-06 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171202+0000", 2024-08-29T15:33:52.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1784999999999999e-05 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171219+0000", 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6387000000000001e-05 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171288+0000", 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9522000000000003e-05 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171402+0000", 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113923 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.171512+0000", 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010959 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.176712+0000", 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052008460000000003 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.176740+0000", 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7997999999999999e-05 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.369536+0000", 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.192796048 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.369639+0000", 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102024 2024-08-29T15:33:52.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.369685+0000", 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6798999999999997e-05 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:166126 7.9 7:94f90bdb:::benchmark_data_smithi027_271924_object49785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:01.177276+0000", 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 111.262212432, 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.213224317, 2024-08-29T15:33:52.555 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166126 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177276+0000", 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177276+0000", 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177277+0000", 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.446e-06 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177282+0000", 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8269999999999997e-06 2024-08-29T15:33:52.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177307+0000", 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4522999999999999e-05 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177315+0000", 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8660000000000006e-06 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177388+0000", 2024-08-29T15:33:52.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3751999999999995e-05 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177513+0000", 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012487300000000001 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177638+0000", 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125189 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.181640+0000", 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040015750000000003 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.181960+0000", 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031994800000000002 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.390215+0000", 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20825553599999999 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.390435+0000", 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021940999999999999 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.390500+0000", 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5371999999999997e-05 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:166134 7.5 7:acf3be3e:::benchmark_data_smithi027_271924_object49793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-08-29T15:33:52.561 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:01.191036+0000", 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 111.24845231499999, 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18241047799999999, 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166134 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191036+0000", 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191036+0000", 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191038+0000", 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3879999999999998e-06 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191046+0000", 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7500000000000003e-06 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191067+0000", 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0744999999999999e-05 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191077+0000", 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7049999999999994e-06 2024-08-29T15:33:52.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191154+0000", 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7693000000000003e-05 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191306+0000", 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015128299999999999 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2,7", 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.191459+0000", 2024-08-29T15:33:52.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015345699999999999 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.196606+0000", 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051473150000000004 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.196751+0000", 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144518 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.196918+0000", 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016757 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.373264+0000", 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17634562300000001 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.373325+0000", 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1427999999999995e-05 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.373446+0000", 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012100299999999999 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:182043 7.6 7:6c450977:::benchmark_data_smithi027_271924_object65702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:32.353279+0000", 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 80.086209311000005, 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17037012900000001, 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.568 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 182043 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.353279+0000", 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.353279+0000", 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.353280+0000", 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.018e-06 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.353285+0000", 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1510000000000003e-06 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.353297+0000", 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1831000000000001e-05 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.353304+0000", 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.272e-06 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.355716+0000", 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024113889999999999 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.355850+0000", 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134141 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,7", 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.355929+0000", 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8885000000000001e-05 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.359552+0000", 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036233559999999999 2024-08-29T15:33:52.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.472110+0000", 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11255762699999999 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.472333+0000", 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000223456 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.523381+0000", 2024-08-29T15:33:52.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.051048299999999998 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.523523+0000", 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014146699999999999 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:32.523649+0000", 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126236 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.574 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:203451 7.9 7:914cde87:::benchmark_data_smithi027_271924_object87110:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e332)", 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:33:09.073935+0000", 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 43.365553095000003, 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17029997199999999, 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 203451 2024-08-29T15:33:52.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.073935+0000", 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.073935+0000", 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.073936+0000", 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2399999999999999e-07 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.073937+0000", 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6789999999999999e-06 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.073943+0000", 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3689999999999998e-06 2024-08-29T15:33:52.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.073947+0000", 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2350000000000001e-06 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.079886+0000", 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059393509999999998 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.080001+0000", 2024-08-29T15:33:52.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114827 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.218766+0000", 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13876486099999999 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.238657+0000", 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019890781 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.238696+0000", 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9878e-05 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.244051+0000", 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053549319999999997 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.244185+0000", 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134022 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:33:52.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:33:09.244235+0000", 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9713000000000003e-05 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:33:52.581 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:33:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:52.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:52.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:52.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:52.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:52.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:52.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:52.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:52.848 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:52.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:52.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:52.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:52.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:52.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:53.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:53.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:53.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:53.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:53.249 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:53.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:53.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:53.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:53.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:53 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 4158/16659 objects degraded (24.959%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:53.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:53 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 4158/16659 objects degraded (24.959%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:53.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:53.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:53.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:53 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 4158/16659 objects degraded (24.959%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:53.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:53.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:53.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:53.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:53.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:53.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:53.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:53.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:53.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:53.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:53.846 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:33:53.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:53.952 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:33:53.953 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:33:53.954 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:33:53.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:53.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:54.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:54.129 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:54.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:54.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:54.139 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:54 smithi027 ceph-mon[98970]: pgmap v1220: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 299 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 4158/14217 objects degraded (29.247%); 1172/14217 objects misplaced (8.244%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:33:54.140 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:54 smithi027 ceph-mon[93921]: pgmap v1220: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 299 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 4158/14217 objects degraded (29.247%); 1172/14217 objects misplaced (8.244%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:33:54.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:54.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:54.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:54.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:54.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:54.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:54.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:54 smithi171 ceph-mon[90031]: pgmap v1220: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 299 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 4158/14217 objects degraded (29.247%); 1172/14217 objects misplaced (8.244%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:33:54.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:54.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:54.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:54.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:54.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:54.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:54.880 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:54.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:54.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:54.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:55.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:55.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:55.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:55.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:55.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:55.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:55.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:55.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:55.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:55.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:55.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:55.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:55.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:55.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:55.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:55.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:55.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:55.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:55.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:55.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:55.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:55.888 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:55.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:55.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:55.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:56.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:56.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:56.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:56.162 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:56.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:56.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:56.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:56.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:56 smithi027 ceph-mon[98970]: pgmap v1221: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 153 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 4313/7230 objects degraded (59.654%); 401/7230 objects misplaced (5.546%); 5.2 MiB/s, 88 objects/s recovering 2024-08-29T15:33:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:56 smithi027 ceph-mon[93921]: pgmap v1221: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 153 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 4313/7230 objects degraded (59.654%); 401/7230 objects misplaced (5.546%); 5.2 MiB/s, 88 objects/s recovering 2024-08-29T15:33:56.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:56.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:56 smithi171 ceph-mon[90031]: pgmap v1221: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 153 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 4313/7230 objects degraded (59.654%); 401/7230 objects misplaced (5.546%); 5.2 MiB/s, 88 objects/s recovering 2024-08-29T15:33:56.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:56.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:56.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:56.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:56.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:56.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.859 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:56.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:56.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:56.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:56.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:57.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:57.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:57.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:57.202 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Removed 116338 objects 2024-08-29T15:33:57.203 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Clean up completed and total clean up time :194.702 2024-08-29T15:33:57.219 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:57.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.244 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:57.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 4313/7230 objects degraded (59.654%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 4313/7230 objects degraded (59.654%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:57.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:57.371 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:57.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:57.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:57.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:57.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 4313/7230 objects degraded (59.654%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:33:57.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:33:57.617 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:57.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:33:57.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:57.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:57.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:57.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:57.869 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:57.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:57.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:57.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:57.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:58.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:58.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:58.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:58.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:58.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:58 smithi027 ceph-mon[93921]: pgmap v1222: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 105 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 4313/4893 objects degraded (88.146%); 401/4893 objects misplaced (8.195%); 5.2 MiB/s, 88 objects/s recovering 2024-08-29T15:33:58.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:58 smithi027 ceph-mon[98970]: pgmap v1222: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 105 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 4313/4893 objects degraded (88.146%); 401/4893 objects misplaced (8.195%); 5.2 MiB/s, 88 objects/s recovering 2024-08-29T15:33:58.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:58.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:58.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:58.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:58.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:58 smithi171 ceph-mon[90031]: pgmap v1222: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 105 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 4313/4893 objects degraded (88.146%); 401/4893 objects misplaced (8.195%); 5.2 MiB/s, 88 objects/s recovering 2024-08-29T15:33:58.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.628 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:58.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:58.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:33:58.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:58.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:33:58.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:58.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:58.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:58.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:33:58.998 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:58.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:58.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:33:59.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:33:59.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:59.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:33:59.163 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:59.164 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:33:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3606296949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:59.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:59.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:33:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3606296949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:59.164 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:33:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:33:59] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-08-29T15:33:59.206 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-08-29T15:33:59.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-08-29T15:33:59.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:59.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:59.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:59.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:33:59.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:33:59.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:59.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:33:59.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:33:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:33:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:33:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3606296949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:33:59.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:59.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:33:59.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:59.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:59.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:59.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:33:59.776 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:59.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:59.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:33:59.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:33:59.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:33:59.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:33:59.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:33:59.954 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:33:59.955 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:33:59.956 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:33:59.956 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:33:59.956 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:33:59.956 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:33:59.956 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:33:59.956 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:34:00.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:00.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:00.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:00.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:00.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:00.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:00.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:00.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:00.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:00.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:00 smithi027 ceph-mon[98970]: pgmap v1223: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 61 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4448/2784 objects degraded (159.770%); 99/2784 objects misplaced (3.556%); 5.3 MiB/s, 109 objects/s recovering 2024-08-29T15:34:00.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:00 smithi027 ceph-mon[93921]: pgmap v1223: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 61 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4448/2784 objects degraded (159.770%); 99/2784 objects misplaced (3.556%); 5.3 MiB/s, 109 objects/s recovering 2024-08-29T15:34:00.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:00.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:00.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:00.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:00 smithi171 ceph-mon[90031]: pgmap v1223: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 61 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4448/2784 objects degraded (159.770%); 99/2784 objects misplaced (3.556%); 5.3 MiB/s, 109 objects/s recovering 2024-08-29T15:34:00.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:00.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:00.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.654 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:00.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:00.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:00.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:00.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:00.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:00.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:00.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:01.002 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:01.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:01.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:01.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:01.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:01.275 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:01.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:01.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:01.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:01.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:01.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:01.627 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:01.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:01.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:01.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:01.773 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:01.875 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:01.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:01.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:01.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:02.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:02.127 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:02.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:02.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:02.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:02 smithi027 ceph-mon[93921]: pgmap v1224: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 13 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 4448/510 objects degraded (872.157%); 4.0 MiB/s, 123 objects/s recovering 2024-08-29T15:34:02.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 4448/510 objects degraded (872.157%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:02 smithi027 ceph-mon[98970]: pgmap v1224: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 13 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 4448/510 objects degraded (872.157%); 4.0 MiB/s, 123 objects/s recovering 2024-08-29T15:34:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 4448/510 objects degraded (872.157%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:02.374 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:02.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:02.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:02 smithi171 ceph-mon[90031]: pgmap v1224: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 13 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 4448/510 objects degraded (872.157%); 4.0 MiB/s, 123 objects/s recovering 2024-08-29T15:34:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 4448/510 objects degraded (872.157%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:02.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:02.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:02.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:02.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:02.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:02.874 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:02.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:02.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:02.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:03.002 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:03.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:03.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:03.148 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:03.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:03.249 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:03.271 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:03.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:03.373 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:03.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:03.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:03.497 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:03.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:03.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:03.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:03.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:03.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:03.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:03.899 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:03.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:03.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:04.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:04.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:04.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:04.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:04.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:04.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:04.282 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:04.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:04 smithi027 ceph-mon[98970]: pgmap v1225: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 4448/84 objects degraded (5295.238%); 3.0 MiB/s, 108 objects/s recovering 2024-08-29T15:34:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:04 smithi027 ceph-mon[93921]: pgmap v1225: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 4448/84 objects degraded (5295.238%); 3.0 MiB/s, 108 objects/s recovering 2024-08-29T15:34:04.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:04.406 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:04.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:04.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:04 smithi171 ceph-mon[90031]: pgmap v1225: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 4448/84 objects degraded (5295.238%); 3.0 MiB/s, 108 objects/s recovering 2024-08-29T15:34:04.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:04.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:04.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:04.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:04.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:04.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:04.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:04.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:04.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:04.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:04.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:04.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:05.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:05.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:05.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:05.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:05.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:05.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:05.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:05.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:05.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:05.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:05.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:05.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:05.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:05.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:05.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:05.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:05.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:05.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:05.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:06.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:06.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:06.130 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:06.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:06.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:06.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:06.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:06 smithi027 ceph-mon[93921]: pgmap v1226: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 4448/84 objects degraded (5295.238%); 3.0 MiB/s, 253 objects/s recovering 2024-08-29T15:34:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:06 smithi027 ceph-mon[98970]: pgmap v1226: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 4448/84 objects degraded (5295.238%); 3.0 MiB/s, 253 objects/s recovering 2024-08-29T15:34:06.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:06.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:06.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:06 smithi171 ceph-mon[90031]: pgmap v1226: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 4448/84 objects degraded (5295.238%); 3.0 MiB/s, 253 objects/s recovering 2024-08-29T15:34:06.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:06.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:06.658 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:06.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:06.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:06.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:06.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:06.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:06.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:06.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:07.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:07.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:07.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:07.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:07.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:07.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:07.262 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:07.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:07.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 4448/84 objects degraded (5295.238%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:07.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 4448/84 objects degraded (5295.238%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:07.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:07.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:07.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 4448/84 objects degraded (5295.238%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:07.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:07.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:07.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:07.658 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:07.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:07.761 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:07.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:07.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:07.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:07.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:07.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:07.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:08.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:08.033 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:08.135 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:08.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:08.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:08.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:08 smithi027 ceph-mon[93921]: pgmap v1227: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 4448/84 objects degraded (5295.238%); 890 KiB/s, 213 objects/s recovering 2024-08-29T15:34:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:08 smithi027 ceph-mon[98970]: pgmap v1227: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 4448/84 objects degraded (5295.238%); 890 KiB/s, 213 objects/s recovering 2024-08-29T15:34:08.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:08.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:08.418 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:08 smithi171 ceph-mon[90031]: pgmap v1227: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 4448/84 objects degraded (5295.238%); 890 KiB/s, 213 objects/s recovering 2024-08-29T15:34:08.508 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:08.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:08.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:08.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:08.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:08.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:08.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:08.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:08.904 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:08.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:08.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:09.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:09.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:09.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:09.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:09.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:09.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:09.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:09.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:34:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:34:09] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.51.0" 2024-08-29T15:34:09.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:09.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:09.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:09.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:09.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:09.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:09.658 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:09.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:09.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:09.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:09.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:09.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:09.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:10.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:10.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:10.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:10.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:10.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:10.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:10.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:10.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:10.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:10 smithi027 ceph-mon[93921]: pgmap v1228: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 4448/84 objects degraded (5295.238%); 890 KiB/s, 304 objects/s recovering 2024-08-29T15:34:10.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:10 smithi027 ceph-mon[98970]: pgmap v1228: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 4448/84 objects degraded (5295.238%); 890 KiB/s, 304 objects/s recovering 2024-08-29T15:34:10.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:10.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:10.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:10 smithi171 ceph-mon[90031]: pgmap v1228: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 4448/84 objects degraded (5295.238%); 890 KiB/s, 304 objects/s recovering 2024-08-29T15:34:10.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:10.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:10.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:10.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:10.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:10.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:10.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:10.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:10.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:10.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:11.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:11.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:11.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:11.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:11.166 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:11.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:11.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:11.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:11.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:11.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:11.548 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:11.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:11.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:11.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:11.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:11.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:11.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:11.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:12.028 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:12.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:12.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:12.178 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:12.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:12.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:12.356 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:12 smithi027 ceph-mon[93921]: pgmap v1229: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 4448/84 objects degraded (5295.238%); 0 B/s, 344 objects/s recovering 2024-08-29T15:34:12.357 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:12 smithi027 ceph-mon[98970]: pgmap v1229: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 4448/84 objects degraded (5295.238%); 0 B/s, 344 objects/s recovering 2024-08-29T15:34:12.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:12.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:12 smithi171 ceph-mon[90031]: pgmap v1229: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 4448/84 objects degraded (5295.238%); 0 B/s, 344 objects/s recovering 2024-08-29T15:34:12.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:12.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:12.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:12.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:12.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:12.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:12.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:12.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:12.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:12.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:12.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:12.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:12.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:13.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:13.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:13.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:13.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:13.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:13.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:13.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:13.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:13.432 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:13.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:13.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:13.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:13.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:13.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:13.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:13.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:13.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:13.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:13.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:13.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:14.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:14.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:14.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:14.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:14.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:14.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:14.306 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:14.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:14.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:14.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:14 smithi171 ceph-mon[90031]: pgmap v1230: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 4448/84 objects degraded (5295.238%); 0 B/s, 308 objects/s recovering 2024-08-29T15:34:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:34:14.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:14.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:14 smithi027 ceph-mon[98970]: pgmap v1230: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 4448/84 objects degraded (5295.238%); 0 B/s, 308 objects/s recovering 2024-08-29T15:34:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:34:14.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:14 smithi027 ceph-mon[93921]: pgmap v1230: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 4448/84 objects degraded (5295.238%); 0 B/s, 308 objects/s recovering 2024-08-29T15:34:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:34:14.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:14.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:14.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:14.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:14.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:14.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:14.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:15.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:15.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:15.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:15.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:15.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:15.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:15.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:15.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:15.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:15 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:15 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:15.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:15.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:15.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:15.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:15 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:15.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:15 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:15.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:15 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:15.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:15 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:15.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:15.682 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:15.784 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:15.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:15.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:15.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:15.931 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:15.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:16.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:16.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:16.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:16.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:16.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:16 smithi171 ceph-mon[90031]: pgmap v1231: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4448/84 objects degraded (5295.238%); 0 B/s, 486 objects/s recovering 2024-08-29T15:34:16.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:16.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:16.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:16.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:16.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:16.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.556 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:16.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:16 smithi027 ceph-mon[98970]: pgmap v1231: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4448/84 objects degraded (5295.238%); 0 B/s, 486 objects/s recovering 2024-08-29T15:34:16.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:16 smithi027 ceph-mon[93921]: pgmap v1231: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4448/84 objects degraded (5295.238%); 0 B/s, 486 objects/s recovering 2024-08-29T15:34:16.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:16.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:16.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:16.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:16.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:16.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:16.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:17.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:17.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:17.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:17.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:17.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:17.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:17.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:17.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:17.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:17.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:17.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:17.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:17.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:17.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:17.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:17.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:17.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:17.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:17.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:17.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:17.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:18.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:18.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:18.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:18.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:18.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:18.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:18.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:18.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:18.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:18.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:18.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:18.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:18 smithi171 ceph-mon[90031]: pgmap v1232: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4448/84 objects degraded (5295.238%); 0 B/s, 342 objects/s recovering 2024-08-29T15:34:18.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:18.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:18 smithi027 ceph-mon[93921]: pgmap v1232: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4448/84 objects degraded (5295.238%); 0 B/s, 342 objects/s recovering 2024-08-29T15:34:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:18 smithi027 ceph-mon[98970]: pgmap v1232: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4448/84 objects degraded (5295.238%); 0 B/s, 342 objects/s recovering 2024-08-29T15:34:18.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:18.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:18.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:18.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:18.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:18.930 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:18.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:19.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:19.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:19.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:19.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:19.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:19.267 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:34:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:34:19] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.51.0" 2024-08-29T15:34:19.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:19.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:19.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:19.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:19.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:19.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:19.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:19.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:19 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3516/84 objects degraded (4185.714%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:19.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:19 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3516/84 objects degraded (4185.714%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:19.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:19.673 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:19.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:19 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3516/84 objects degraded (4185.714%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:34:19.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:19.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:19.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:19.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:19.920 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:19.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:19.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:20.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:20.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:20.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:20.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:20.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:20.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:20.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:20.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:20.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:20.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:20.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:20.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:20.559 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:20 smithi171 ceph-mon[90031]: pgmap v1233: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 459 objects/s recovering 2024-08-29T15:34:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:20 smithi027 ceph-mon[93921]: pgmap v1233: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 459 objects/s recovering 2024-08-29T15:34:20.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:20 smithi027 ceph-mon[98970]: pgmap v1233: 89 pgs: 1 active+recovering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 459 objects/s recovering 2024-08-29T15:34:20.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:20.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:20.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:20.791 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:20.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:20.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:20.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:21.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:21.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:21.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:21.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:21.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:21.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:21.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:21.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:21.416 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:21.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:21.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:21.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:21.641 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:21.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:21.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:21.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:21.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:21.912 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:21.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:22.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:22.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:22.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:22.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:22.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:22.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:22.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:22.286 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:22.388 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:22.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:22.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:22.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:22 smithi027 ceph-mon[98970]: pgmap v1234: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 436 objects/s recovering 2024-08-29T15:34:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:22.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:22 smithi027 ceph-mon[93921]: pgmap v1234: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 436 objects/s recovering 2024-08-29T15:34:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:22.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:22.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:22.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:22.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:22.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:22 smithi171 ceph-mon[90031]: pgmap v1234: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 436 objects/s recovering 2024-08-29T15:34:22.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:22.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:22.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:22.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:22.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:22.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:22.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:23.010 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:23.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:23.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:23.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:23.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:23.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:23.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:23.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:23.410 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:23.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:34:23.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:23.533 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:23.584 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:34:23.585 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:34:23.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-08-29T15:34:23.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:23.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:23.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:23.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:23.783 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:23.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:23.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:23.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:24.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:23 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:23 smithi171 ceph-mon[90031]: pgmap v1235: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 363 objects/s recovering 2024-08-29T15:34:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:23 smithi171 ceph-mon[90031]: osdmap e336: 8 total, 8 up, 8 in 2024-08-29T15:34:24.030 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:24.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[93921]: pgmap v1235: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 363 objects/s recovering 2024-08-29T15:34:24.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[93921]: osdmap e336: 8 total, 8 up, 8 in 2024-08-29T15:34:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[98970]: pgmap v1235: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3516/84 objects degraded (4185.714%); 0 B/s, 363 objects/s recovering 2024-08-29T15:34:24.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:23 smithi027 ceph-mon[98970]: osdmap e336: 8 total, 8 up, 8 in 2024-08-29T15:34:24.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:24.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:24.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:24.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:24.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:24.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:24.407 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:24.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:24.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:24.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:24.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:24.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:24.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:24.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:24.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:24.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:24.823 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:24.824 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:24.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:24.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:24.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:24.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:25.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:25.040 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:25.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:25.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:25.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:25.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:25.285 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:34:25.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:25.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:25.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:25.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:25.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:25 smithi027 ceph-mon[98970]: osdmap e337: 8 total, 8 up, 8 in 2024-08-29T15:34:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3402108955' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:34:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:25 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:34:25.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:25 smithi027 ceph-mon[93921]: osdmap e337: 8 total, 8 up, 8 in 2024-08-29T15:34:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3402108955' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:34:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:25 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:34:25.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:25.673 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:25.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:25 smithi171 ceph-mon[90031]: osdmap e337: 8 total, 8 up, 8 in 2024-08-29T15:34:25.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3402108955' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:34:25.713 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:25 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:34:25.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:25.797 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:25.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:25.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:25.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:25.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:26.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:26.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:26.150 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:26.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:26.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:26.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:26.296 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:26.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:34:26.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:26.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:26.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:26.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[93921]: pgmap v1238: 89 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1733/84 objects degraded (2063.095%); 0 B/s, 500 objects/s recovering 2024-08-29T15:34:26.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 1733/84 objects degraded (2063.095%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:34:26.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:34:26.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:34:26.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[93921]: osdmap e338: 8 total, 8 up, 8 in 2024-08-29T15:34:26.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[98970]: pgmap v1238: 89 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1733/84 objects degraded (2063.095%); 0 B/s, 500 objects/s recovering 2024-08-29T15:34:26.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 1733/84 objects degraded (2063.095%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:34:26.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:34:26.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:34:26.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:26 smithi027 ceph-mon[98970]: osdmap e338: 8 total, 8 up, 8 in 2024-08-29T15:34:26.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:26.646 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:26.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:26.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:26.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:26 smithi171 ceph-mon[90031]: pgmap v1238: 89 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1733/84 objects degraded (2063.095%); 0 B/s, 500 objects/s recovering 2024-08-29T15:34:26.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:26 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 1733/84 objects degraded (2063.095%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:34:26.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:26 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:34:26.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:26 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:34:26.740 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:26 smithi171 ceph-mon[90031]: osdmap e338: 8 total, 8 up, 8 in 2024-08-29T15:34:26.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:26.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:26.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:26.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:26.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:27.020 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:27.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:27.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:27.168 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:27.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:27.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:27.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:27.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:27.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:27.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:27.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:27.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.673 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:27.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:27.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:27.799 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:27.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:27.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:27.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:27.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:27.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:28.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:28.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:28.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:28.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:28.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:28.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:28.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:28 smithi027 ceph-mon[98970]: pgmap v1240: 89 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1733/84 objects degraded (2063.095%); 0 B/s, 297 objects/s recovering 2024-08-29T15:34:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:28 smithi027 ceph-mon[93921]: pgmap v1240: 89 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1733/84 objects degraded (2063.095%); 0 B/s, 297 objects/s recovering 2024-08-29T15:34:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:28.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:34:28.383 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:28.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:28.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:28 smithi171 ceph-mon[90031]: pgmap v1240: 89 pgs: 1 active+remapped+backfill_wait, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1733/84 objects degraded (2063.095%); 0 B/s, 297 objects/s recovering 2024-08-29T15:34:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:28.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:28.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:28.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:28.682 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:28.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:28.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:28.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:28.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:28.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:28.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:29.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:29.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:29.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:29.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:29.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:29.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:34:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:34:29] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.51.0" 2024-08-29T15:34:29.321 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:29.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:29.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:29.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:29.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:29.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:29.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:29.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:29.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:29.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:29.807 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:29.831 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:29.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:29.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:29.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:29.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:30.061 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:30.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:30.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:30.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:30.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:30.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:30.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:30.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:30.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:30 smithi171 ceph-mon[90031]: pgmap v1241: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 585 objects/s recovering 2024-08-29T15:34:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:30 smithi171 ceph-mon[90031]: osdmap e339: 8 total, 8 up, 8 in 2024-08-29T15:34:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]} : dispatch 2024-08-29T15:34:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-08-29T15:34:30.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:30 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]} : dispatch 2024-08-29T15:34:30.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[98970]: pgmap v1241: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 585 objects/s recovering 2024-08-29T15:34:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[98970]: osdmap e339: 8 total, 8 up, 8 in 2024-08-29T15:34:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]} : dispatch 2024-08-29T15:34:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-08-29T15:34:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]} : dispatch 2024-08-29T15:34:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[93921]: pgmap v1241: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 87 active+clean; 4.6 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 585 objects/s recovering 2024-08-29T15:34:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[93921]: osdmap e339: 8 total, 8 up, 8 in 2024-08-29T15:34:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]} : dispatch 2024-08-29T15:34:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]} : dispatch 2024-08-29T15:34:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:30 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]} : dispatch 2024-08-29T15:34:30.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:30.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:30.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:30.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:30.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:30.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:30.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:30.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:30.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:31.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:31.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:31.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:31.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:31.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:31.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:31.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:31.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:31.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:31.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:31.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:31.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-08-29T15:34:31.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-08-29T15:34:31.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2024-08-29T15:34:31.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[98970]: osdmap e340: 8 total, 8 up, 8 in 2024-08-29T15:34:31.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-08-29T15:34:31.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-08-29T15:34:31.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2024-08-29T15:34:31.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:31 smithi027 ceph-mon[93921]: osdmap e340: 8 total, 8 up, 8 in 2024-08-29T15:34:31.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:31.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:31.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:31.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:31.723 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 0]}]': finished 2024-08-29T15:34:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 0]}]': finished 2024-08-29T15:34:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2024-08-29T15:34:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:31 smithi171 ceph-mon[90031]: osdmap e340: 8 total, 8 up, 8 in 2024-08-29T15:34:31.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:31.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:31.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:31.952 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:31.977 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:31.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:31.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:32.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:32.103 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:32.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:32.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.231 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:32.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:32.295 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:32.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:32.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:32.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:32.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:32 smithi027 ceph-mon[93921]: pgmap v1244: 89 pgs: 1 peering, 1 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 288 objects/s recovering 2024-08-29T15:34:32.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:32 smithi027 ceph-mon[93921]: osdmap e341: 8 total, 8 up, 8 in 2024-08-29T15:34:32.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-29T15:34:32.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:32 smithi027 ceph-mon[98970]: pgmap v1244: 89 pgs: 1 peering, 1 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 288 objects/s recovering 2024-08-29T15:34:32.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:32 smithi027 ceph-mon[98970]: osdmap e341: 8 total, 8 up, 8 in 2024-08-29T15:34:32.377 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-29T15:34:32.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:32.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:32.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:32.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:32.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:32.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:32.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:32 smithi171 ceph-mon[90031]: pgmap v1244: 89 pgs: 1 peering, 1 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 288 objects/s recovering 2024-08-29T15:34:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:32 smithi171 ceph-mon[90031]: osdmap e341: 8 total, 8 up, 8 in 2024-08-29T15:34:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-29T15:34:32.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:32.861 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:32.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:32.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:32.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:33.093 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:33.119 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:33.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:33.220 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:33.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:33.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:33.291 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_2' removed 2024-08-29T15:34:33.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:33.385 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:33.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:33.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:33.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:33.511 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:33.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3790489931' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:34:33.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3790489931' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:34:33.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:33.634 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:33.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:33.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:33.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:33.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:33.758 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:33.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3790489931' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:34:33.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:33.882 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:33.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:33.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:33.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:34.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:34.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:34.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:34.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:34.231 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:34.252 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:34.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:34.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:34.376 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:34:34.376 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:34:34.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:34.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:34.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:34 smithi027 ceph-mon[98970]: pgmap v1246: 89 pgs: 1 peering, 1 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 289 objects/s recovering 2024-08-29T15:34:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3790489931' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:34:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:34 smithi027 ceph-mon[98970]: osdmap e342: 8 total, 8 up, 8 in 2024-08-29T15:34:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:34 smithi027 ceph-mon[93921]: pgmap v1246: 89 pgs: 1 peering, 1 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 289 objects/s recovering 2024-08-29T15:34:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3790489931' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:34:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:34 smithi027 ceph-mon[93921]: osdmap e342: 8 total, 8 up, 8 in 2024-08-29T15:34:34.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:34.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:34.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:34.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:34.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:34 smithi171 ceph-mon[90031]: pgmap v1246: 89 pgs: 1 peering, 1 active+undersized+remapped, 87 active+clean; 4.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 289 objects/s recovering 2024-08-29T15:34:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3790489931' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:34:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:34 smithi171 ceph-mon[90031]: osdmap e342: 8 total, 8 up, 8 in 2024-08-29T15:34:34.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:34.855 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:34:34.855 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:34:34.855 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:34:34.855 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:34:34.855 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:34:34.855 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:34:34.855 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:34:34.856 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:34:34.856 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:34:34.856 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:34:34.856 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:34:34.856 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:34:34.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:34.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:34.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:34.999 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:35.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:35.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:35.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.249 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:35.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:35.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:35.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:35.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:35.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:35 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-29T15:34:35.462 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:35 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-29T15:34:35.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:35.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:34:35.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:35.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:35.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:35.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:35.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:35.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:35.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:35 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-29T15:34:35.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:35.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:35.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:35.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:35.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:35.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:36.095 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-29T15:34:36.096 INFO:tasks.radosbench:Beginning radosbench... 2024-08-29T15:34:36.096 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-08-29T15:34:36.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-08-29T15:34:36.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:36.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:36.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:36.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:36.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:36.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:36.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:36.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:36.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:36.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:36.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:36 smithi171 ceph-mon[90031]: pgmap v1248: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:36.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:34:36.597 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:34:36.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-08-29T15:34:36.607 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:36.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:36.619 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:36.630 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:36 smithi027 ceph-mon[98970]: pgmap v1248: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:36.630 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:36 smithi027 ceph-mon[93921]: pgmap v1248: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:36.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:36.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:36.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:36.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:36.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:36.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:36.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:36.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:37.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:37.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:37.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:37.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:37.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:37.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:37.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:37.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:37.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:37.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:37.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:37.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:37.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:37.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:37.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:37.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:37.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.857 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:37.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:37.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:37.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:37.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:37.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:38.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:38.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.105 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:38.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:38.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:38.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:38.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:38.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:38.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:38.419 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:38 smithi027 ceph-mon[93921]: pgmap v1249: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:38.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:38 smithi027 ceph-mon[98970]: pgmap v1249: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:38.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:38.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:38.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.604 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:38.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:38.669 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:34:38.669 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:34:38.669 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:34:38.669 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:34:38.670 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:34:38.671 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:34:38.671 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:34:38.671 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:34:38.671 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:34:38.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:38.728 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:38.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:38.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:38 smithi171 ceph-mon[90031]: pgmap v1249: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:38.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:38.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:38.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:38.894 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:38.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:38.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:38.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:39.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:39.103 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:39.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:39.227 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:39.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:39.274 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:34:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:34:39] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.51.0" 2024-08-29T15:34:39.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:39.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:39.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:39.451 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:39.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:39.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:39.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:39 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/901358638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:34:39.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:39 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/901358638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:34:39.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:39.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.599 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:39.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:39.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.723 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:39.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:39 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/901358638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:34:39.824 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:39.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:39.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:39.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:39.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:39.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:40.076 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:40.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:40.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:40.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:40.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:40.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:40.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:40.364 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:34:40.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:40.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.478 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:40.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:40.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:40.604 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:40.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:40.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:40.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:40 smithi171 ceph-mon[90031]: pgmap v1250: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/936284676' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:34:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:40 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:34:40.800 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:40 smithi027 ceph-mon[93921]: pgmap v1250: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:40.800 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/936284676' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:34:40.800 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:40 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:34:40.800 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:40 smithi027 ceph-mon[98970]: pgmap v1250: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:40.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/936284676' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:34:40.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:40 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:34:40.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:40.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:40.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:40.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:40.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:41.085 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:41.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:41.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:41.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:41.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:41.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:41.465 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:41.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:41.591 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:41.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:41.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:34:41.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:34:41.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:34:41.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[93921]: osdmap e343: 8 total, 8 up, 8 in 2024-08-29T15:34:41.640 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:34:41.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:34:41.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:34:41.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:41 smithi027 ceph-mon[98970]: osdmap e343: 8 total, 8 up, 8 in 2024-08-29T15:34:41.716 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:41.740 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:41 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:34:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:41 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:34:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:41 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:34:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:41 smithi171 ceph-mon[90031]: osdmap e343: 8 total, 8 up, 8 in 2024-08-29T15:34:41.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:41.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:41.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:41.992 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:41.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:42.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:42.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:42.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:42.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:42.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:42.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:42.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:42.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:42.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:42.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:42.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:42.631 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:42 smithi027 ceph-mon[98970]: pgmap v1252: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:42.631 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:42 smithi027 ceph-mon[93921]: pgmap v1252: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:42.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:42.747 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:42 smithi171 ceph-mon[90031]: pgmap v1252: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:42.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:42.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:42.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:42.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:42.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:43.034 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:34:43.034 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:34:43.034 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:34:43.035 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:34:43.035 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:34:43.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:43.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:43.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:43.224 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:43.248 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:43.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:43.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:43.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:43.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:43.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:43.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:43.497 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:43.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:43.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:43.625 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:43.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:43.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:34:43.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:34:43.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:43.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:43.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:43.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:43.879 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:43.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:43.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:44.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:44.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:44.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:44.236 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:44.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:44.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:44.358 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:44.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:44.388 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:44.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:44.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:44.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:44.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:44.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:34:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:44 smithi027 ceph-mon[93921]: pgmap v1253: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:44 smithi027 ceph-mon[98970]: pgmap v1253: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:44.614 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:44.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:44.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:44.767 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:44 smithi171 ceph-mon[90031]: pgmap v1253: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:44.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:44.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:44.895 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:44.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:44.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:45.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:45.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:45.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:45.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:45.250 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:45.276 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:45.378 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:45.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:45.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:45.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:45.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:45.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:45.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:45.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:45.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:45.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.784 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:45.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:45.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:45.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:45.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:45.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:46.010 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:46.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:46.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:46.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:46.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:46.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:46.289 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:46.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:46.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:46.414 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:46.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:34:46.495 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:34:46.496 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:34:46.496 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:34:46.500 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:46.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:46.541 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:46.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:46.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:46.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:46.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:46 smithi171 ceph-mon[90031]: pgmap v1254: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:46.793 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:46.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:46.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:46 smithi027 ceph-mon[98970]: pgmap v1254: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:46.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:46 smithi027 ceph-mon[93921]: pgmap v1254: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:46.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:46.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:46.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:46.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:47.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:47.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:47.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:47.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:47.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:47.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:47.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:47.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:47.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:47.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:47.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:47.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:47.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:47.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:47.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:47.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:47.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:47.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:47.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:47.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:47.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:47.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:48.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:48.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:48.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:48.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:48.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:48.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:48.399 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_3' created 2024-08-29T15:34:48.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:48.414 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:34:48.414 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:34:48.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-08-29T15:34:48.438 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:48.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:48.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:48.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:48.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:48.665 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:48.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:48.753 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[98970]: pgmap v1255: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/515559138' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[93921]: pgmap v1255: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/515559138' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:48.754 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:48 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:48 smithi171 ceph-mon[90031]: pgmap v1255: 73 pgs: 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/515559138' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-29T15:34:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:48 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-29T15:34:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:48 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:48.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:48 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:48.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:48.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:48.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:48.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:48.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:48.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:49.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:49.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:49.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:49.110 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:49.111 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:49.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:49.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:49.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:49.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:34:49] "GET /metrics HTTP/1.1" 200 34196 "" "Prometheus/2.51.0" 2024-08-29T15:34:49.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:49.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:49.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:49.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:49.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:49.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:49.669 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-29T15:34:49.669 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[98970]: osdmap e344: 8 total, 8 up, 8 in 2024-08-29T15:34:49.669 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:34:49.669 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:34:49.670 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:49.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-29T15:34:49.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[93921]: osdmap e344: 8 total, 8 up, 8 in 2024-08-29T15:34:49.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:34:49.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:34:49.670 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:49.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:49 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-29T15:34:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:49 smithi171 ceph-mon[90031]: osdmap e344: 8 total, 8 up, 8 in 2024-08-29T15:34:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:34:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:34:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:49.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:49.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:49.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:49.912 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:49.936 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:49.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:50.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:50.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:50.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:50.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:50.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:50.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-08-29T15:34:50.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:50.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:50.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:50.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:50.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:50.474 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:50 smithi027 ceph-mon[98970]: pgmap v1257: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:50.474 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:50 smithi027 ceph-mon[98970]: osdmap e345: 8 total, 8 up, 8 in 2024-08-29T15:34:50.474 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:50 smithi027 ceph-mon[98970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:34:50.475 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:50 smithi027 ceph-mon[93921]: pgmap v1257: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:50.475 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:50 smithi027 ceph-mon[93921]: osdmap e345: 8 total, 8 up, 8 in 2024-08-29T15:34:50.475 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:50 smithi027 ceph-mon[93921]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:34:50.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:50.555 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:50.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:50.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:50.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:50 smithi171 ceph-mon[90031]: pgmap v1257: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:50 smithi171 ceph-mon[90031]: osdmap e345: 8 total, 8 up, 8 in 2024-08-29T15:34:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:50 smithi171 ceph-mon[90031]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:34:50.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:50.804 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:50.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:50.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:50.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:51.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:51.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:51.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:51.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:51.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:51.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:51.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:51.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:51.300 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:51.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:51.354 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:34:51.354 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:34:51.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:34:51.354 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:34:51.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.355 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:110025 7.2 e303/294 7:4fa9aa4d:::benchmark_data_smithi027_271924_object110024:head v 303'6778, mlcod=303'6774)", 2024-08-29T15:34:51.355 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.603177+0000", 2024-08-29T15:34:51.355 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.747794677, 2024-08-29T15:34:51.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26727316899999998, 2024-08-29T15:34:51.356 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.356 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.356 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.356 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.356 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603177+0000", 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603177+0000", 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603178+0000", 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.071e-06 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603203+0000", 2024-08-29T15:34:51.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5550000000000001e-05 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603217+0000", 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.34e-05 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603225+0000", 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.038e-06 2024-08-29T15:34:51.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603286+0000", 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1625999999999996e-05 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.603362+0000", 2024-08-29T15:34:51.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5788e-05 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.612711+0000", 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093491259999999993 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.870450+0000", 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25773857 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.36273.0:110056 7.9 e303/294 7:939c17ba:::benchmark_data_smithi027_271924_object110055:head v 303'6744, mlcod=303'6742)", 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.608739+0000", 2024-08-29T15:34:51.362 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.74223204700002, 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26618966500000002, 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608739+0000", 2024-08-29T15:34:51.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608739+0000", 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608741+0000", 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.319e-06 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608763+0000", 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2427000000000001e-05 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608775+0000", 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1592e-05 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608782+0000", 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1790000000000002e-06 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608859+0000", 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7601000000000005e-05 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.608878+0000", 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8439000000000001e-05 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.874879+0000", 2024-08-29T15:34:51.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26600132999999998 2024-08-29T15:34:51.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.368 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.874929+0000", 2024-08-29T15:34:51.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9777999999999997e-05 2024-08-29T15:34:51.368 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.369 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.369 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.369 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148195 7.5 7.29cfb6a5 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:34:51.370 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:34:51.370 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 206.15933355799999, 2024-08-29T15:34:51.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.001068407, 2024-08-29T15:34:51.370 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.370 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.370 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:34:51.371 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.371 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.371 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.371 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148195 2024-08-29T15:34:51.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.372 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:34:51.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:34:51.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191640+0000", 2024-08-29T15:34:51.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2989999999999999e-06 2024-08-29T15:34:51.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191646+0000", 2024-08-29T15:34:51.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8690000000000001e-06 2024-08-29T15:34:51.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191664+0000", 2024-08-29T15:34:51.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8189000000000002e-05 2024-08-29T15:34:51.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191673+0000", 2024-08-29T15:34:51.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7229999999999993e-06 2024-08-29T15:34:51.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.198665+0000", 2024-08-29T15:34:51.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069919730000000003 2024-08-29T15:34:51.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.198675+0000", 2024-08-29T15:34:51.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8740000000000007e-06 2024-08-29T15:34:51.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192690+0000", 2024-08-29T15:34:51.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99401576199999997 2024-08-29T15:34:51.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192706+0000", 2024-08-29T15:34:51.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5718000000000001e-05 2024-08-29T15:34:51.380 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.381 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.381 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.381 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148242 7.5 7.35df8025 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:34:51.381 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:25.313183+0000", 2024-08-29T15:34:51.382 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 206.03778786199999, 2024-08-29T15:34:51.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87956382, 2024-08-29T15:34:51.382 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.382 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.382 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:34:51.383 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.383 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.383 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.383 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148242 2024-08-29T15:34:51.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.383 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313183+0000", 2024-08-29T15:34:51.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313183+0000", 2024-08-29T15:34:51.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313185+0000", 2024-08-29T15:34:51.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.186e-06 2024-08-29T15:34:51.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313189+0000", 2024-08-29T15:34:51.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2339999999999996e-06 2024-08-29T15:34:51.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313201+0000", 2024-08-29T15:34:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1674e-05 2024-08-29T15:34:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313208+0000", 2024-08-29T15:34:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3390000000000004e-06 2024-08-29T15:34:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313262+0000", 2024-08-29T15:34:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4545e-05 2024-08-29T15:34:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313268+0000", 2024-08-29T15:34:51.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6069999999999998e-06 2024-08-29T15:34:51.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192738+0000", 2024-08-29T15:34:51.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87947040700000001 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192747+0000", 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8279999999999992e-06 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.392 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.393 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191681 7.5 7:a9d21a90:::benchmark_data_smithi027_271924_object75340:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.393 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:34:51.393 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.404523686, 2024-08-29T15:34:51.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.12813253, 2024-08-29T15:34:51.394 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.394 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.394 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:34:51.394 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.394 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.395 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.395 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191681 2024-08-29T15:34:51.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.395 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:34:51.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:34:51.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946449+0000", 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6509999999999999e-06 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946455+0000", 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8769999999999998e-06 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946472+0000", 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7169000000000001e-05 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946481+0000", 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3289999999999994e-06 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946564+0000", 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3615000000000005e-05 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op must wait for map", 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946571+0000", 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0849999999999999e-06 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.952704+0000", 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061325340000000002 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.952713+0000", 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2709999999999998e-06 2024-08-29T15:34:51.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042516+0000", 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.089802597 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042641+0000", 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012506600000000001 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074432+0000", 2024-08-29T15:34:51.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.031791257000000003 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074580+0000", 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014807899999999999 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191685 7.5 7:a0ab1ff6:::benchmark_data_smithi027_271924_object75344:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:34:51.401 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.40446921100001, 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1510580690000001, 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191685 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:34:51.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946503+0000", 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4199999999999996e-07 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946505+0000", 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2589999999999999e-06 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946512+0000", 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9999999999999999e-06 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:51.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946517+0000", 2024-08-29T15:34:51.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.079e-06 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946614+0000", 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.692e-05 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting_for_map not empty", 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946619+0000", 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1200000000000001e-06 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.953271+0000", 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066521210000000004 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.953278+0000", 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.204e-06 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042965+0000", 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.089686486 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043078+0000", 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113163 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097459+0000", 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.054381391000000001 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097560+0000", 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100784 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191701 7.5 7:a40d5d11:::benchmark_data_smithi027_271924_object75360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.369426475, 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.099438114, 2024-08-29T15:34:51.410 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191701 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981546+0000", 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.291e-06 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981551+0000", 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9779999999999996e-06 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981563+0000", 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1787000000000001e-05 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981570+0000", 2024-08-29T15:34:51.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4240000000000004e-06 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981641+0000", 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1018000000000003e-05 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981649+0000", 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9379999999999992e-06 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043134+0000", 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0614847059999999 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043265+0000", 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013072100000000001 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043851+0000", 2024-08-29T15:34:51.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058628399999999996 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.067960+0000", 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024109189 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.069771+0000", 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018104740000000001 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080779+0000", 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011008132 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080848+0000", 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9249000000000001e-05 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080983+0000", 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013492300000000001 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.417 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191702 7.5 7:a63127fd:::benchmark_data_smithi027_271924_object75361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.36938410099999, 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.099668844, 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.419 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191702 2024-08-29T15:34:51.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.419 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:34:51.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:34:51.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981588+0000", 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7399999999999998e-07 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981590+0000", 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.266e-06 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981597+0000", 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.579e-06 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981602+0000", 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3329999999999996e-06 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981671+0000", 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9581999999999999e-05 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981677+0000", 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9000000000000003e-06 2024-08-29T15:34:51.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065439+0000", 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0837620489999999 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065523+0000", 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3906000000000006e-05 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065611+0000", 2024-08-29T15:34:51.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7347000000000002e-05 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072205+0000", 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065941050000000003 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072219+0000", 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3817e-05 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081102+0000", 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088836209999999995 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081154+0000", 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1907000000000003e-05 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081256+0000", 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101958 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.426 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191704 7.5 7:adaa8151:::benchmark_data_smithi027_271924_object75363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.36401854499999, 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0945452710000001, 2024-08-29T15:34:51.427 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191704 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.428 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986955+0000", 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.781e-06 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986961+0000", 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2349999999999996e-06 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986976+0000", 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5676e-05 2024-08-29T15:34:51.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986984+0000", 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8369999999999997e-06 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.001951+0000", 2024-08-29T15:34:51.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014966225 2024-08-29T15:34:51.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.433 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.433 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.001960+0000", 2024-08-29T15:34:51.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8379999999999997e-06 2024-08-29T15:34:51.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.069808+0000", 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.067847722 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.070348+0000", 2024-08-29T15:34:51.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00053970699999999995 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.070512+0000", 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016436399999999999 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074579+0000", 2024-08-29T15:34:51.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004066315 2024-08-29T15:34:51.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080650+0000", 2024-08-29T15:34:51.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060713909999999998 2024-08-29T15:34:51.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081349+0000", 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00069884000000000005 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081398+0000", 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9524999999999998e-05 2024-08-29T15:34:51.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081498+0000", 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9815000000000006e-05 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191709 7.5 7:aeaa9bf4:::benchmark_data_smithi027_271924_object75368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.362515618, 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0993208800000001, 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.439 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191709 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:34:51.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:34:51.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988457+0000", 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4270000000000001e-06 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988462+0000", 2024-08-29T15:34:51.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0159999999999999e-06 2024-08-29T15:34:51.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988477+0000", 2024-08-29T15:34:51.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5116999999999999e-05 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988484+0000", 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1799999999999999e-06 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.003492+0000", 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015007802000000001 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.003518+0000", 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5970999999999999e-05 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072246+0000", 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.068727446 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072383+0000", 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013752099999999999 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072536+0000", 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015290200000000001 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077323+0000", 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047870919999999997 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081054+0000", 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037308939999999998 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087498+0000", 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006444015 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087668+0000", 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016991900000000001 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087777+0000", 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108578 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191717 7.5 7:a9f87d2e:::benchmark_data_smithi027_271924_object75376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.344182866, 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0898794949999999, 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191717 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.449 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006790+0000", 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9970000000000001e-06 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006797+0000", 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4570000000000002e-06 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006830+0000", 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2867999999999998e-05 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006840+0000", 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.01e-05 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006951+0000", 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011073600000000001 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006963+0000", 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1977999999999999e-05 2024-08-29T15:34:51.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074609+0000", 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0676467890000001 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074705+0000", 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5200999999999999e-05 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074792+0000", 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6955999999999999e-05 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080664+0000", 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058721310000000001 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081300+0000", 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00063660400000000003 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096470+0000", 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015169472 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096534+0000", 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3911999999999994e-05 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096668+0000", 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134294 2024-08-29T15:34:51.455 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191718 7.5 7:a50fea8b:::benchmark_data_smithi027_271924_object75377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.34410341100001, 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.090114451, 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191718 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006869+0000", 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2099999999999996e-07 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006872+0000", 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2729999999999998e-06 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006882+0000", 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8779999999999993e-06 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006887+0000", 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6409999999999999e-06 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006985+0000", 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7449999999999997e-05 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006994+0000", 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5799999999999998e-06 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077356+0000", 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0703613439999999 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077468+0000", 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112322 2024-08-29T15:34:51.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077596+0000", 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012779999999999999 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087397+0000", 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098012389999999998 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087443+0000", 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6204e-05 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096811+0000", 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093673639999999996 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096888+0000", 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7793000000000005e-05 2024-08-29T15:34:51.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096982+0000", 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3942000000000003e-05 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191761 7.5 7:a52c750c:::benchmark_data_smithi027_271924_object75420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.262733352, 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0210746239999999, 2024-08-29T15:34:51.463 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191761 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088240+0000", 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6300000000000001e-06 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088247+0000", 2024-08-29T15:34:51.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9990000000000002e-06 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088261+0000", 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4416000000000001e-05 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088269+0000", 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1629999999999996e-06 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088333+0000", 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.347e-05 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088342+0000", 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.363e-06 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081533+0000", 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99319060000000003 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081648+0000", 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115339 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081776+0000", 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012817300000000001 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087457+0000", 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056813039999999999 2024-08-29T15:34:51.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087842+0000", 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038465299999999999 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109101+0000", 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021259324 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109209+0000", 2024-08-29T15:34:51.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107187 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109313+0000", 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104003 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191763 7.5 7:a9d168ed:::benchmark_data_smithi027_271924_object75422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.259427605, 2024-08-29T15:34:51.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.018041102, 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191763 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091547+0000", 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8590000000000001e-06 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091556+0000", 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4599999999999992e-06 2024-08-29T15:34:51.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091580+0000", 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4314000000000001e-05 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091592+0000", 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1768e-05 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091686+0000", 2024-08-29T15:34:51.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4029000000000004e-05 2024-08-29T15:34:51.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091696+0000", 2024-08-29T15:34:51.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7410000000000004e-06 2024-08-29T15:34:51.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084389+0000", 2024-08-29T15:34:51.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99269340699999997 2024-08-29T15:34:51.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084526+0000", 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136444 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084664+0000", 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013856400000000001 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087792+0000", 2024-08-29T15:34:51.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003127286 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087905+0000", 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113612 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109403+0000", 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021497480999999999 2024-08-29T15:34:51.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109491+0000", 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7936000000000006e-05 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109585+0000", 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4201000000000002e-05 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191826 7.5 7:a4f0d2f2:::benchmark_data_smithi027_271924_object75485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.224887+0000", 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.12608452000001, 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89589589999999997, 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191826 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224887+0000", 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224887+0000", 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224888+0000", 2024-08-29T15:34:51.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.468e-06 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224894+0000", 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9989999999999997e-06 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224907+0000", 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3117e-05 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224916+0000", 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9409999999999996e-06 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224984+0000", 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7309000000000002e-05 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224996+0000", 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2308e-05 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087963+0000", 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86296706599999995 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.088088+0000", 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012504700000000001 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.088213+0000", 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012473599999999999 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096738+0000", 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085248749999999995 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097073+0000", 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033500400000000002 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120619+0000", 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023546629999999999 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120679+0000", 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9697000000000001e-05 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120783+0000", 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010370300000000001 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.485 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191832 7.5 7:abd490dc:::benchmark_data_smithi027_271924_object75491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.253103+0000", 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.097868139, 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86786427600000005, 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191832 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253103+0000", 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253103+0000", 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253105+0000", 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1739999999999999e-06 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253113+0000", 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.464e-06 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253131+0000", 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8467000000000001e-05 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253142+0000", 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0973e-05 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253227+0000", 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4938999999999994e-05 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253242+0000", 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4722999999999999e-05 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.090751+0000", 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83750952899999997 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.090874+0000", 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000122059 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.091012+0000", 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013845799999999999 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097056+0000", 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060442359999999997 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097136+0000", 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9408999999999998e-05 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120833+0000", 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023696993 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120879+0000", 2024-08-29T15:34:51.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5948000000000001e-05 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120967+0000", 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8904000000000001e-05 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191850 7.5 7:a96a40a5:::benchmark_data_smithi027_271924_object75509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.290683+0000", 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.060288492, 2024-08-29T15:34:51.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83046312, 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191850 2024-08-29T15:34:51.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.495 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290683+0000", 2024-08-29T15:34:51.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290683+0000", 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290685+0000", 2024-08-29T15:34:51.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7630000000000001e-06 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290692+0000", 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5789999999999998e-06 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290710+0000", 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7948000000000002e-05 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290718+0000", 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8080000000000005e-06 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290781+0000", 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2927000000000002e-05 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290791+0000", 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.006e-05 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.093701+0000", 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80291038400000003 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.093810+0000", 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108713 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.093953+0000", 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014307600000000001 2024-08-29T15:34:51.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097143+0000", 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031897739999999998 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097599+0000", 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045595800000000001 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.121007+0000", 2024-08-29T15:34:51.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023407661999999999 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.121054+0000", 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7503999999999998e-05 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.121146+0000", 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1964000000000002e-05 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191855 7.5 7:aa7d08ea:::benchmark_data_smithi027_271924_object75514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.302772+0000", 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.048199341, 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.82153611599999998, 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191855 2024-08-29T15:34:51.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302772+0000", 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302772+0000", 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302774+0000", 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0420000000000001e-06 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302781+0000", 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9319999999999999e-06 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302801+0000", 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9596e-05 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302810+0000", 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8900000000000002e-06 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302895+0000", 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4055999999999996e-05 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302908+0000", 2024-08-29T15:34:51.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2989999999999999e-05 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097630+0000", 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79472256399999996 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097730+0000", 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9567000000000004e-05 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097857+0000", 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127324 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.106251+0000", 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083942419999999997 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109062+0000", 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028109110000000001 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124144+0000", 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015081695000000001 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124201+0000", 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7362999999999999e-05 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124308+0000", 2024-08-29T15:34:51.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106944 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191856 7.5 7:af2f8f1a:::benchmark_data_smithi027_271924_object75515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.304572+0000", 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 122.046398985, 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.81997056599999996, 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.509 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191856 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304572+0000", 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304572+0000", 2024-08-29T15:34:51.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304574+0000", 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5400000000000001e-06 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304585+0000", 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0951999999999999e-05 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304598+0000", 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3587999999999999e-05 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304606+0000", 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9899999999999997e-06 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304669+0000", 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2188999999999997e-05 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304679+0000", 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0149999999999999e-05 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.100714+0000", 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79603570099999998 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.100834+0000", 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119759 2024-08-29T15:34:51.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.100967+0000", 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013277100000000001 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109323+0000", 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083562849999999998 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109365+0000", 2024-08-29T15:34:51.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1319999999999997e-05 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124347+0000", 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014982812 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124444+0000", 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6793000000000007e-05 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124543+0000", 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8715999999999995e-05 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191865 7.5 7:a304caa0:::benchmark_data_smithi027_271924_object75524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.374259+0000", 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 121.976712642, 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.75050312799999996, 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:34:51.516 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191865 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374259+0000", 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:34:51.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374259+0000", 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374261+0000", 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2680000000000001e-06 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374269+0000", 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4139999999999994e-06 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374290+0000", 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0143e-05 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374300+0000", 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0037000000000001e-05 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374379+0000", 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9752000000000005e-05 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374392+0000", 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2704999999999999e-05 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.103523+0000", 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72913066900000001 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.103653+0000", 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000130576 2024-08-29T15:34:51.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.103799+0000", 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014593900000000001 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109328+0000", 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055292700000000002 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.113609+0000", 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042803959999999997 2024-08-29T15:34:51.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124621+0000", 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011012088999999999 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124673+0000", 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1977000000000001e-05 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124762+0000", 2024-08-29T15:34:51.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8893000000000004e-05 2024-08-29T15:34:51.523 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.523 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.523 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.523 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:34:51.523 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:34:51.523 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:34:51.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:51.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:51.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:51.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:51 smithi027 ceph-mon[98970]: osdmap e346: 8 total, 8 up, 8 in 2024-08-29T15:34:51.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:51 smithi027 ceph-mon[93921]: osdmap e346: 8 total, 8 up, 8 in 2024-08-29T15:34:51.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:51.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:51.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:51 smithi171 ceph-mon[90031]: osdmap e346: 8 total, 8 up, 8 in 2024-08-29T15:34:51.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:51.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:51.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:51.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:51.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:51.920 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:51.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:51.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:52.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:52.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:52.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:52.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:52.171 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:52.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:52.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:52.393 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:52.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:52.423 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:52.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:52.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:52 smithi027 ceph-mon[93921]: pgmap v1260: 89 pgs: 1 creating+activating, 6 creating+peering, 8 unknown, 74 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:52.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:52 smithi027 ceph-mon[98970]: pgmap v1260: 89 pgs: 1 creating+activating, 6 creating+peering, 8 unknown, 74 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:52.526 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:52.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:52.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:52.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:52.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:52 smithi171 ceph-mon[90031]: pgmap v1260: 89 pgs: 1 creating+activating, 6 creating+peering, 8 unknown, 74 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:52.778 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:52.804 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:52.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:52.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:52.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:52.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:53.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:53.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:53.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:53.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:53.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:53.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:53.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:53.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:53.435 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:53.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:53.537 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:53.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:53.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:53.665 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:53.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:53.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:53.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:53.816 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:53.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:53.944 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:53.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:53.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:34:54.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:54.068 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:54.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:54.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:54.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:54.297 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:54.323 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:54.424 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:54.450 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:54.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:54.454 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:54 smithi027 ceph-mon[98970]: pgmap v1261: 89 pgs: 1 creating+activating, 6 creating+peering, 82 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:54.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:54 smithi027 ceph-mon[93921]: pgmap v1261: 89 pgs: 1 creating+activating, 6 creating+peering, 82 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:54.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:54.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:54.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:54.656 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:54.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:54.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:54 smithi171 ceph-mon[90031]: pgmap v1261: 89 pgs: 1 creating+activating, 6 creating+peering, 82 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:54.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:54.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:54.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:54.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:54.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:54.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:54.965 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:34:54.965 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:34:54.965 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:34:54.965 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:34:54.966 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:34:54.966 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:34:54.975 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:34:54.975 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:34:54.996 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:34:54.997 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:34:55.014 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:34:55.015 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:34:55.032 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:34:55.032 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:34:55.048 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:34:55.049 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:34:55.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:55.081 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:55.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:55.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:55.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:55.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:55.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:55.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:55.422 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:34:55.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:55.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:55.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:55.591 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:55.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:55.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:55.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:55.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2557772350' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:34:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/98297859' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:34:55.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2557772350' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:34:55.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/98297859' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:34:55.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2557772350' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:34:55.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/98297859' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:34:55.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:55.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:55.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:55.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:56.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:56.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:56.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:56.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:56.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:56.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:56.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:56.346 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:56.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:56.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:56.474 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:56.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[98970]: pgmap v1262: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:56.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:34:56.489 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2557772350' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:34:56.489 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[98970]: osdmap e347: 8 total, 8 up, 8 in 2024-08-29T15:34:56.489 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[93921]: pgmap v1262: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:56.489 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:34:56.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2557772350' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:34:56.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:56 smithi027 ceph-mon[93921]: osdmap e347: 8 total, 8 up, 8 in 2024-08-29T15:34:56.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:56.601 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:56.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:56.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.728 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:56.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:56.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:56 smithi171 ceph-mon[90031]: pgmap v1262: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:56.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:56 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:34:56.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2557772350' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:34:56.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:56 smithi171 ceph-mon[90031]: osdmap e347: 8 total, 8 up, 8 in 2024-08-29T15:34:56.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:56.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:56.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:56.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:56.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:56.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:56.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:57.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:57.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:57.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:57.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:57.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:57.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:57.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:57.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:57.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:57.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:57.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:57.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:57.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:57.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:57.595 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:57.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:57.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:57.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:57.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:57.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:57.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:57.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:57.851 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:57.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:57.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:57.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:57.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:58.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:58.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:58.130 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:58.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:58.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:58.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:58.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:58.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:34:58.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:58.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:34:58.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:58.634 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:58.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:58 smithi027 ceph-mon[98970]: pgmap v1264: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:58.706 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:58 smithi027 ceph-mon[93921]: pgmap v1264: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:58.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:58.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:58 smithi171 ceph-mon[90031]: pgmap v1264: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:58.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:58.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:58.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:58.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:58.988 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:34:59.014 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:59.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:34:59.025 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:34:59.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:59.140 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:59.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:34:59.264 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:34:59.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:34:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:34:59] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.51.0" 2024-08-29T15:34:59.366 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:34:59.392 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:59.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:34:59.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:34:59.548 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:59.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:34:59.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:34:59.673 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:59.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:34:59.690 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:34:59.690 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:34:59.690 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:34:59.690 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:34:59.691 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:34:59.691 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:34:59.691 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:34:59.691 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:34:59.691 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:34:59.691 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:34:59.692 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:34:59.692 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:34:59.717 INFO:teuthology.orchestra.run.smithi027.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-08-29T15:34:59.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:34:59.800 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:34:59.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:34:59.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:34:59.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:34:59.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:34:59.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:34:59.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:59.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:59.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[93921]: pgmap v1265: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:59.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/903721064' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:34:59.997 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:34:59.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:34:59.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[98970]: pgmap v1265: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:34:59.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:34:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/903721064' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:35:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:35:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:59 smithi171 ceph-mon[90031]: pgmap v1265: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:35:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:34:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/903721064' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:35:00.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:00.053 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:00.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.181 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:00.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:00.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:00.300 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:35:00.301 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:35:00.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-08-29T15:35:00.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:00.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:00.432 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:00.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:00.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:00.558 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:00.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:00.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:00.684 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:00.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:00.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:00.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:00.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:00.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:00.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:00.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/903721064' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:35:00.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:00 smithi027 ceph-mon[98970]: osdmap e348: 8 total, 8 up, 8 in 2024-08-29T15:35:00.962 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/903721064' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:35:00.962 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:00 smithi027 ceph-mon[93921]: osdmap e348: 8 total, 8 up, 8 in 2024-08-29T15:35:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/903721064' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:35:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:00 smithi171 ceph-mon[90031]: osdmap e348: 8 total, 8 up, 8 in 2024-08-29T15:35:01.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:01.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:01.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:01.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:01.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:01.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:01.316 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:01.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:01.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:01.444 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:01.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:35:01.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:01.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:01.672 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:01.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:01.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:01.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:01 smithi027 ceph-mon[98970]: pgmap v1267: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:35:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:01 smithi027 ceph-mon[93921]: pgmap v1267: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:35:01.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:01.921 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:01.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:01.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:01.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:01 smithi171 ceph-mon[90031]: pgmap v1267: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:35:02.021 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:02.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:02.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:02.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:02.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:02.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:02.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:02.321 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:02.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:02.423 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:02.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:02.540 DEBUG:teuthology.orchestra.run.smithi027:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-08-29T15:35:02.543 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-29T15:35:02.543 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-29T15:35:02.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:02.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:02.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:02.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:02.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:02.729 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:hints = 1 2024-08-29T15:35:02.729 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-29T15:35:02.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object prefix: benchmark_data_smithi027_306999 2024-08-29T15:35:02.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:35:02.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 0 0 0 0 0 0 - 0 2024-08-29T15:35:02.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:02.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:02.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:02.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:02.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:02.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:02 smithi171 ceph-mon[90031]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:35:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:02 smithi027 ceph-mon[93921]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:35:03.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:02 smithi027 ceph-mon[98970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:35:03.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:03.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:03.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:03.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:03.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:03.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:03.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:03.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:03.391 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:03.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:03.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:03.534 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:03.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:03.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:03.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:03.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:03.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:03.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 1 16 3887 3871 241.896 241.938 0.00414739 0.00411933 2024-08-29T15:35:03.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:03.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:03.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:03.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:03.925 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:03 smithi027 ceph-mon[93921]: pgmap v1268: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:35:03.926 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:03 smithi027 ceph-mon[98970]: pgmap v1268: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:35:03.966 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:03.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:03.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:03.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:04.009 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:35:04.009 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:35:04.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:03 smithi171 ceph-mon[90031]: pgmap v1268: 89 pgs: 89 active+clean; 4.6 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-29T15:35:04.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:04.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:04.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:04.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:04.239 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:04.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:04.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:04.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:04.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:04.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:04.519 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:04.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:04.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:04.627 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:35:04.628 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:35:04.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:04.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:04.673 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:35:04.673 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:35:04.711 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:35:04.711 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:35:04.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 2 16 7218 7202 225.033 208.188 0.00762209 0.00443114 2024-08-29T15:35:04.752 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:35:04.752 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:35:04.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:04.787 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:35:04.787 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:35:04.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:04.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:04.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:04.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2253760951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:04.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2253760951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:04.825 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:35:04.825 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:35:04.859 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:35:04.859 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:35:04.895 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:35:04.895 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:35:04.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:04.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:04.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:05.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2253760951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:05.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:05.102 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:05.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:05.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:05.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:05.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:05.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:05.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:05.376 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:05.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:05.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:05.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:05.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:05.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:05.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:05.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:35:05.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:05.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:05.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:05.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:05.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 3 16 10495 10479 218.287 204.812 0.00241156 0.00457286 2024-08-29T15:35:05.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:05.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:05.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:05.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:05.940 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:05.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:06.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:05 smithi171 ceph-mon[90031]: pgmap v1269: 89 pgs: 89 active+clean; 173 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2024-08-29T15:35:06.041 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:05 smithi027 ceph-mon[98970]: pgmap v1269: 89 pgs: 89 active+clean; 173 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2024-08-29T15:35:06.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:05 smithi027 ceph-mon[93921]: pgmap v1269: 89 pgs: 89 active+clean; 173 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2024-08-29T15:35:06.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:06.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:06.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:06.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:06.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:06.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:06.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:06.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:06.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:06.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:06.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:06.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:06.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:06.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:06.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:06.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:06.627 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:06.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:06.688 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:06.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:06.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 4 16 13615 13599 212.454 195 0.00585283 0.00469903 2024-08-29T15:35:06.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:06.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:06.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:06.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:06.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:07.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:07.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:07.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:07.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:07.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:07.181 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:07.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:07.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:07.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:07.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:07.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:07.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:07.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:07.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:07.601 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:07.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:07.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:07.730 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 5 16 16651 16635 207.906 189.75 0.00531923 0.00480373 2024-08-29T15:35:07.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:07.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:07.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:07.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:07.885 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:07.917 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:07.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:07.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:35:08.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:08.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:08.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:08.135 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:07 smithi027 ceph-mon[98970]: pgmap v1270: 89 pgs: 89 active+clean; 556 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2024-08-29T15:35:08.135 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:07 smithi027 ceph-mon[93921]: pgmap v1270: 89 pgs: 89 active+clean; 556 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2024-08-29T15:35:08.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:08.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:08.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:08.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:07 smithi171 ceph-mon[90031]: pgmap v1270: 89 pgs: 89 active+clean; 556 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2024-08-29T15:35:08.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:08.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:08.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:08.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:08.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:08.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:08.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:08.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:08.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:08.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:08.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:08.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 6 16 19278 19262 200.619 164.188 0.00268162 0.00497887 2024-08-29T15:35:08.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:08.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:08.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:08.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:08.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:08.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:09.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:09.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:09.169 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:09.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:09.201 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:35:09.201 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:35:09.201 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:35:09.201 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:35:09.201 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:35:09.201 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:35:09.201 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:35:09.202 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:35:09.202 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:35:09.202 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:35:09.202 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:35:09.202 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:35:09.237 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:35:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:35:09] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.51.0" 2024-08-29T15:35:09.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:09.309 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:09.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:09.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:09.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:09.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:09.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:09.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:09.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:09.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 7 16 21799 21783 194.461 157.562 0.00988282 0.00513617 2024-08-29T15:35:09.811 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:09.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:09.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:09.982 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:09.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:10.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:09 smithi027 ceph-mon[93921]: pgmap v1271: 89 pgs: 89 active+clean; 735 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-29T15:35:10.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:09 smithi027 ceph-mon[98970]: pgmap v1271: 89 pgs: 89 active+clean; 735 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-29T15:35:10.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:10.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:10.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:10.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:10.266 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:10.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:10.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:09 smithi171 ceph-mon[90031]: pgmap v1271: 89 pgs: 89 active+clean; 735 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-29T15:35:10.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:10.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:10.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:10.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:10.567 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:10.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:10.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:10.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:10.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:10.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:10.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 8 16 24398 24382 190.456 162.438 0.00452154 0.00524478 2024-08-29T15:35:10.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:10.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:10.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:10.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:10.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:10.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:11.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:11.131 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:11.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:11.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:11.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:11.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:11.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:11.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:11.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:11.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:11.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:11.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:11.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:11.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:11.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:11.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:11.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:11.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:11.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 9 16 26632 26616 184.806 139.625 0.00155779 0.00537236 2024-08-29T15:35:11.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:11.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:11.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:11.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:11.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:11.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:11.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:11.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:12.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:12.102 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:12.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:12.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:12.233 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:11 smithi027 ceph-mon[93921]: pgmap v1272: 89 pgs: 89 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-29T15:35:12.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:11 smithi027 ceph-mon[98970]: pgmap v1272: 89 pgs: 89 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-29T15:35:12.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:12.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:12.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:11 smithi171 ceph-mon[90031]: pgmap v1272: 89 pgs: 89 active+clean; 1.0 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-29T15:35:12.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:12.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:12.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:12.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:12.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:12.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:12.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:12.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:12.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:12.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:12.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 10 15 28936 28921 180.731 144.062 0.00661823 0.00552801 2024-08-29T15:35:12.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:12.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:12.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:12.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:12.873 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:35:12.874 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:35:12.874 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-08-29T15:35:12.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:12.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:12.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:12.933 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:13.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:13.053 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:13.066 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:35:13.094 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/580287913' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:13.094 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:12 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:13.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/580287913' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:13.096 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:13.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:13.202 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:13.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/580287913' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:12 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:13.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:13.325 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:13.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:13.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:13.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:13.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:13.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:13.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:13.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:13.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 11 16 30188 30172 171.408 78.1875 0.0100206 0.00582817 2024-08-29T15:35:13.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:13.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:13.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:13.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:13.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:14.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:14.130 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:14.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:14.213 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[98970]: pgmap v1273: 89 pgs: 89 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:35:14.213 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:35:14.213 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:35:14.213 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[98970]: osdmap e349: 8 total, 8 up, 8 in 2024-08-29T15:35:14.213 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:35:14.213 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:14.214 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[93921]: pgmap v1273: 89 pgs: 89 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:35:14.214 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:35:14.214 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:35:14.214 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[93921]: osdmap e349: 8 total, 8 up, 8 in 2024-08-29T15:35:14.214 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:35:14.214 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:14.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:14.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:14.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:14.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:13 smithi171 ceph-mon[90031]: pgmap v1273: 89 pgs: 89 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:35:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:13 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:35:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:35:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:13 smithi171 ceph-mon[90031]: osdmap e349: 8 total, 8 up, 8 in 2024-08-29T15:35:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:35:14.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:14.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:14.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:14.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:14.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:14.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:14.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:14.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:14.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:14.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:14.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 12 16 32042 32026 166.78 115.875 0.00385381 0.0059899 2024-08-29T15:35:14.768 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:14.804 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:14.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:14.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:14.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:14.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:15.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:15.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:15.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:15.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:15.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:15.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:15.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:15.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:15.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:15.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:15.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:15.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:15.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:15.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:15.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:15.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:15.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:15.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:15.731 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 13 16 34039 34023 163.551 124.812 0.00776171 0.00610838 2024-08-29T15:35:15.751 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:15.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:15.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:15.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:15.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:15.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:15.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:15.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:16.029 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:16.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:16.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:16.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:16.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:15 smithi171 ceph-mon[90031]: pgmap v1275: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-08-29T15:35:16.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:16.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:16.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:15 smithi027 ceph-mon[93921]: pgmap v1275: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-08-29T15:35:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:15 smithi027 ceph-mon[98970]: pgmap v1275: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 161 MiB/s wr, 2.58k op/s 2024-08-29T15:35:16.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:16.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:16.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:16.572 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:16.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:16.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:16.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 14 16 36198 36182 161.506 134.938 0.00589701 0.00618653 2024-08-29T15:35:16.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:16.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:16.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:16.988 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:16.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:17.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:17.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:17.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:17.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:17.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:17.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:17.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:17.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:17.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:17.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:17.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:17.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:17.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:17.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:17.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:17.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:17.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:17.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:17.665 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:17.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:17.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 15 16 38143 38127 158.842 121.562 0.00722301 0.00629043 2024-08-29T15:35:17.766 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:17.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:17.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:17.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:17.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:17.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:17.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:18.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:18.073 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:18.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:18.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:18.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:18.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:17 smithi171 ceph-mon[90031]: pgmap v1276: 89 pgs: 89 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2024-08-29T15:35:18.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:18.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:17 smithi027 ceph-mon[98970]: pgmap v1276: 89 pgs: 89 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2024-08-29T15:35:18.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:17 smithi027 ceph-mon[93921]: pgmap v1276: 89 pgs: 89 active+clean; 2.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2024-08-29T15:35:18.354 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:18.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:18.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:18.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:18.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:18.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:18.614 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:18.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:18.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:18.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:18.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 16 16 40117 40101 156.625 123.375 0.00681211 0.00637932 2024-08-29T15:35:18.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:18.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:18.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:18.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:18.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:18.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:18.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:19.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:19.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:19.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:19.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:19.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:19.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:19.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:35:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:35:19] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-08-29T15:35:19.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:19.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.442 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:19.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:19.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:19.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:19.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:19.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:19.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 17 16 41498 41482 152.488 86.3125 0.0019894 0.00646136 2024-08-29T15:35:19.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:19.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:19.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:19.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:19.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:19.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:20.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:20.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:20.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:20.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:20.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:20.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:20.237 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:19 smithi027 ceph-mon[98970]: pgmap v1277: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-08-29T15:35:20.237 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:19 smithi027 ceph-mon[93921]: pgmap v1277: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-08-29T15:35:20.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:19 smithi171 ceph-mon[90031]: pgmap v1277: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-08-29T15:35:20.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:20.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:20.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:20.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:20.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:20.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:20.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:20.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:20.649 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:20.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:20.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:20.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 18 16 43384 43368 150.565 117.875 0.00748355 0.00663671 2024-08-29T15:35:20.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:20.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:20.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:20.886 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:20.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:20.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:21.025 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:21.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:21.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:21.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:21.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:21.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:21.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:21.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:21.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:21.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:21.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:21.432 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:21.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:21.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:21.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:21.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:21.579 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:21.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:21.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:21.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:21.734 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 19 16 45282 45266 148.883 118.625 0.00697804 0.00671107 2024-08-29T15:35:21.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:21.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:21.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:21.908 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:21.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:22.010 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:22.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:22.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:22.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:22.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:21 smithi027 ceph-mon[93921]: pgmap v1278: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2024-08-29T15:35:22.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:21 smithi027 ceph-mon[98970]: pgmap v1278: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2024-08-29T15:35:22.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:22.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:22.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:21 smithi171 ceph-mon[90031]: pgmap v1278: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2024-08-29T15:35:22.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:22.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:22.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:22.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:22.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:22.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:22.474 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:22.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:22.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:22.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:22.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:22.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:22.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:22.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:35:22.732428+0000 min lat: 0.00117312 max lat: 0.294735 avg lat: 0.00678078 2024-08-29T15:35:22.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:35:22.732 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 20 16 47154 47138 147.289 117 0.00590357 0.00678078 2024-08-29T15:35:22.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:22.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:22.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:22.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:22.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:22.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:22.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:23.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.020 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:23.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:23.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:23.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:23.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:23.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:23.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:23.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:23.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:23.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:23.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:23.531 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:35:23.531 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:35:23.531 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:35:23.531 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:35:23.531 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:35:23.560 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:23.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:23.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:23.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:23.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 21 16 48985 48969 145.724 114.438 0.00700917 0.00685743 2024-08-29T15:35:23.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:23.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:23.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:23.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:23.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:23.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:23.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:24.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:24.107 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:24.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:24.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:35:24.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:24.213 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:23 smithi027 ceph-mon[93921]: pgmap v1279: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-08-29T15:35:24.213 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:23 smithi027 ceph-mon[98970]: pgmap v1279: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-08-29T15:35:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:24.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:24.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:23 smithi171 ceph-mon[90031]: pgmap v1279: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-08-29T15:35:24.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:24.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:24.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:24.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:24.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:24.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:24.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:24.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:24.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:24.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:24.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:24.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 22 16 50822 50806 144.318 114.812 0.0044676 0.00692444 2024-08-29T15:35:24.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:24.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:24.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:24.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:24.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:24.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:24.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:25.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:25.101 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:25.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:25.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:25.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:25.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:25.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:25.329 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:25.364 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:25.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:25.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:25.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:25.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:25.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:25.642 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:25.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:25.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:25.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 23 16 52500 52484 142.603 104.875 0.0123073 0.00700701 2024-08-29T15:35:25.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:25.756 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:25.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:35:25.793 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:25.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:25.895 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:25.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:25.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:26.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:25 smithi027 ceph-mon[93921]: pgmap v1280: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-08-29T15:35:26.028 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:25 smithi027 ceph-mon[98970]: pgmap v1280: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-08-29T15:35:26.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:26.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:26.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:26.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:26.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:26.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:26.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:25 smithi171 ceph-mon[90031]: pgmap v1280: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-08-29T15:35:26.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:26.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:26.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:26.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:26.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:26.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:26.576 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2024-08-29T15:35:26.576 INFO:teuthology.orchestra.run.smithi027.stderr:0 2024-08-29T15:35:26.576 INFO:teuthology.orchestra.run.smithi027.stderr:1 2024-08-29T15:35:26.576 INFO:teuthology.orchestra.run.smithi027.stderr:2 2024-08-29T15:35:26.576 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2024-08-29T15:35:26.576 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2024-08-29T15:35:26.576 INFO:teuthology.orchestra.run.smithi027.stderr:bench [] [] [] [] 2024-08-29T15:35:26.577 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs debug_inject_read_zeros 2024-08-29T15:35:26.577 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs files list 2024-08-29T15:35:26.577 INFO:teuthology.orchestra.run.smithi027.stderr:bluefs stats 2024-08-29T15:35:26.577 INFO:teuthology.orchestra.run.smithi027.stderr:bluestore allocator dump block 2024-08-29T15:35:26.577 INFO:teuthology.orchestra.run.smithi027.stderr:Error EINVAL: invalid command 2024-08-29T15:35:26.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:26.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:26.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:26.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:26.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 24 16 53949 53933 140.434 90.5625 0.0113639 0.00711596 2024-08-29T15:35:26.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:26.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:26.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:26.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:26.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:26.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:26.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:26.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:27.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:27.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:27.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:27.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:27.287 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:27.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:27.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:27.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:27.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:27.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:27.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:27.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:27.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 25 16 55633 55617 139.026 105.25 0.00732772 0.00718794 2024-08-29T15:35:27.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:27.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:27.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:27.948 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:35:27.949 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:35:27.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:27.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:27.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:27.995 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:35:27.995 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:35:27.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:27 smithi027 ceph-mon[98970]: pgmap v1281: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:35:27.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1349132660' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:27 smithi027 ceph-mon[93921]: pgmap v1281: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:35:27.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1349132660' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:28.016 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:35:28.016 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:35:28.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:28.091 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:35:28.091 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:35:28.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:28.131 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:35:28.131 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:35:28.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:28.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:28.162 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:35:28.162 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:35:28.195 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:35:28.195 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:35:28.228 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:35:28.229 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:35:28.233 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:28.264 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:28.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:27 smithi171 ceph-mon[90031]: pgmap v1281: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-08-29T15:35:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1349132660' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:28.367 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:28.388 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:28.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:28.489 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:28.518 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:28.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:28.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:28.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:28.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:28.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:28.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 26 16 56978 56962 136.912 84.0625 0.0105183 0.00729866 2024-08-29T15:35:28.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:28.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:28.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:28.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:28.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:28.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:28.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:29.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:29.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:29.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:29.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:29.225 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:29.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:29.275 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:29.275 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:29.275 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:35:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:35:29] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.51.0" 2024-08-29T15:35:29.327 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:29.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:29.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:29.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:29.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:29.495 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:29.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:29.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:29.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:29.636 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:29.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:29.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 27 16 58464 58448 135.281 92.875 0.0146225 0.00738752 2024-08-29T15:35:29.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:29.773 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:29.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:29.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:29.875 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:29.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:29.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:30.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:30.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:30.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:30.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:30.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:29 smithi171 ceph-mon[90031]: pgmap v1282: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:35:30.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:29 smithi027 ceph-mon[98970]: pgmap v1282: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:35:30.271 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:29 smithi027 ceph-mon[93921]: pgmap v1282: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:35:30.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:30.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:30.413 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:30.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:30.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:30.581 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:30.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:30.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:30.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 28 16 60005 59989 133.889 96.3125 0.00797344 0.00746355 2024-08-29T15:35:30.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:30.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.845 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:30.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:30.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:30.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:30.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:31.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:31.128 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:31.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:31.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:31.268 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:31.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:31.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:31.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:31.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:31.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:31.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:31.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:31.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:31.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:31.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:31.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:31.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:31.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 29 16 61561 61545 132.625 97.25 0.00693166 0.00753512 2024-08-29T15:35:31.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:31.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:31.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:31.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:31.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:31.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:31.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:31.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:32.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:31 smithi027 ceph-mon[93921]: pgmap v1283: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-29T15:35:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:31 smithi027 ceph-mon[98970]: pgmap v1283: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-29T15:35:32.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:32.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:32.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:32.223 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:32.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:32.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:31 smithi171 ceph-mon[90031]: pgmap v1283: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-08-29T15:35:32.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:32.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:32.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:32.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:32.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:32.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:32.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:32.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:32.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:32.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:32.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:32.733 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 30 16 62742 62726 130.665 73.8125 0.00762854 0.00764805 2024-08-29T15:35:32.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:32.751 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:32.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:32.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:32.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:32.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:32.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:32.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:33.011 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:33.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:33.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:33.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:33.287 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:33.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:33.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:33.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:33.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:33.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:33.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:33.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:33.735 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 31 16 64163 64147 129.309 88.8125 0.014081 0.00772839 2024-08-29T15:35:33.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:33.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:33.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:33.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:33.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:34.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:34.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:34.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:34.183 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:33 smithi027 ceph-mon[93921]: pgmap v1284: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:35:34.183 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:33 smithi027 ceph-mon[98970]: pgmap v1284: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:35:34.184 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-29T15:35:34.184 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-29T15:35:34.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:34.266 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:34.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:34.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:34.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:33 smithi171 ceph-mon[90031]: pgmap v1284: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:35:34.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:34.415 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:34.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:34.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:34.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:34.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:34.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:34.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:34.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:34.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:34.726 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:34.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 32 16 65644 65628 128.156 92.5625 0.0167402 0.00779677 2024-08-29T15:35:34.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:34.828 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:34.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:34.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:34.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:34.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:34.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:35.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:35.099 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:35.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:35.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:35.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:35.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:35.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:35.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:35.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:35:35.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:35.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:35.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:35.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:35.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:35.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:35.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:35.629 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:35.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:35.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:35.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 33 16 67062 67046 126.958 88.625 0.0084096 0.00787163 2024-08-29T15:35:35.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:35.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:35.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:35.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:35.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:35.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:35.984 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:35:36.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:36.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:36.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:36.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:36.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:36.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:36.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:35 smithi171 ceph-mon[90031]: pgmap v1285: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-08-29T15:35:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/915362977' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:35:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:35 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:35:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:35 smithi027 ceph-mon[93921]: pgmap v1285: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-08-29T15:35:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/915362977' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:35:36.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:35 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:35:36.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:35 smithi027 ceph-mon[98970]: pgmap v1285: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-08-29T15:35:36.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/915362977' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:35:36.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:35 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:35:36.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:36.377 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:36.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:36.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:36.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:36.512 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:36.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:36.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:36.651 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:36.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:36.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 34 16 68576 68560 126.007 94.625 0.00964383 0.00793099 2024-08-29T15:35:36.754 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:36.787 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:36.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:36.889 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:36.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:36.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:36.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:36 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:35:36.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:36 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:35:36.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:36 smithi027 ceph-mon[98970]: osdmap e350: 8 total, 8 up, 8 in 2024-08-29T15:35:36.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:36 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:35:36.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:36 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:35:36.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:36 smithi027 ceph-mon[93921]: osdmap e350: 8 total, 8 up, 8 in 2024-08-29T15:35:37.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:37.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:37.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:37.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:37.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:37.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:37.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:37.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:36 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:35:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:36 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:35:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:36 smithi171 ceph-mon[90031]: osdmap e350: 8 total, 8 up, 8 in 2024-08-29T15:35:37.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:37.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:37.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:37.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:37.431 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:37.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:37.463 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:35:37.463 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:35:37.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:37.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:37.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:37.625 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:37.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:37.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:37.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:37.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 35 16 69989 69973 124.93 88.3125 0.00736906 0.00799956 2024-08-29T15:35:37.759 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:37.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:37.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:37.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:37.896 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:37.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:37.998 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:38.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:38.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:38.130 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:38.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:38.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:38.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:37 smithi171 ceph-mon[90031]: pgmap v1287: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-29T15:35:38.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:38.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:37 smithi027 ceph-mon[98970]: pgmap v1287: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-29T15:35:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:37 smithi027 ceph-mon[93921]: pgmap v1287: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-08-29T15:35:38.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:38.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:38.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:38.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:38.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:38.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:38.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:38.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 36 16 71306 71290 123.746 82.3125 0.0128814 0.00807636 2024-08-29T15:35:38.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:35:38.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:38.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:38.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:38.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:38.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:39.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:39.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:35:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:35:39] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.51.0" 2024-08-29T15:35:39.102 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:39.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:39.185 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:35:39.185 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:35:39.185 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-08-29T15:35:39.185 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:35:39.185 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:35:39.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-08-29T15:35:39.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:39.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:39.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:39.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:39.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:39.394 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:39.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:39.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:39.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:39.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:39.559 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:35:39.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:39.671 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:39.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:39.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 37 16 72642 72626 122.658 83.5 0.0149343 0.00814762 2024-08-29T15:35:39.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:39.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:39.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:39.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:39.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:39.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:40.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:40.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:40.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:40.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:40.189 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:40.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:40.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:39 smithi171 ceph-mon[90031]: pgmap v1288: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-29T15:35:40.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:40.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:39 smithi027 ceph-mon[98970]: pgmap v1288: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-29T15:35:40.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:39 smithi027 ceph-mon[93921]: pgmap v1288: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-29T15:35:40.328 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:40.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:40.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:40.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:40.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:40.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:40.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:40.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:40.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:40.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:40.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 38 16 73657 73641 121.1 63.4375 0.00357357 0.00825247 2024-08-29T15:35:40.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:40.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:40.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:40.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:40.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:40.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:40.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:40.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:41.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:41.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:41.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:41.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:41.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:41.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:41.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:41.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:41.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:41.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:41.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:41.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:41.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:41.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:41.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:41.661 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:35:41.662 INFO:teuthology.orchestra.run.smithi171.stdout: "success": "" 2024-08-29T15:35:41.662 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:35:41.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:41.723 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:41.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:41.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:41.736 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 39 16 75033 75017 120.2 86 0.0108825 0.0083144 2024-08-29T15:35:41.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:41.861 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:41.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:41.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:35:41.929 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:41.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:42.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:42.003 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:42.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:42.042 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:41 smithi027 ceph-mon[93921]: pgmap v1289: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-29T15:35:42.042 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:41 smithi027 ceph-mon[98970]: pgmap v1289: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-29T15:35:42.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:42.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:42.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:42.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:41 smithi171 ceph-mon[90031]: pgmap v1289: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-29T15:35:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:42.303 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:42.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:42.404 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:42.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:42.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:42.530 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:35:42.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:42.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:42.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:42.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:42.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:35:42.736892+0000 min lat: 0.00117312 max lat: 0.294735 avg lat: 0.00839275 2024-08-29T15:35:42.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:35:42.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 40 16 76242 76226 119.083 75.5625 0.0129744 0.00839275 2024-08-29T15:35:42.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:42.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:42.780 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:35:42.780 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:35:42.818 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:35:42.819 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:35:42.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:35:42.850 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:35:42.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:42.887 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:35:42.887 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:35:42.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:42.902 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:42.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:42.921 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:35:42.921 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:35:42.958 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:35:42.958 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:35:42.998 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:35:42.998 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:35:43.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:43.032 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:43.033 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:35:43.033 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:35:43.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:43.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:43.135 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:43.181 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:43.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/909639951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:43.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:43.316 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:43.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:43.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/909639951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:43.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/909639951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:43.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:43.456 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:43.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:43.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:43.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:43.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:43.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:43.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:43.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 41 16 77429 77413 117.988 74.1875 0.016921 0.00847077 2024-08-29T15:35:43.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:43.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:43.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:43.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:43.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:43.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:43.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:44.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:44.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:44.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:44.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:44.134 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:44 smithi171 ceph-mon[90031]: pgmap v1290: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-29T15:35:44.134 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:44.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:44.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:44.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:44.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:44.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:44 smithi027 ceph-mon[98970]: pgmap v1290: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-29T15:35:44.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:44.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:44 smithi027 ceph-mon[93921]: pgmap v1290: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-29T15:35:44.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:44.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:44.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:44.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:44.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:78444 8.5 8:a192ff75:::benchmark_data_smithi027_306999_object78443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:35:44.546822+0000", 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.013273425, 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01330657, 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:35:44.563 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 78444 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.546822+0000", 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.546822+0000", 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:35:44.564 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.546822+0000", 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.546882+0000", 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0268e-05 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:35:44.565 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.546889+0000", 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9500000000000004e-06 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.546894+0000", 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9899999999999997e-06 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.547034+0000", 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013946999999999999 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:35:44.566 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.547179+0000", 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014571899999999999 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.547302+0000", 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012301099999999999 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553514+0000", 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0062114409999999998 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.567 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.558416+0000", 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049026369999999996 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.560056+0000", 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016398059999999999 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.560139+0000", 2024-08-29T15:35:44.568 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2863999999999998e-05 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:78450 8.2 8:4692de27:::benchmark_data_smithi027_306999_object78449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:35:44.551368+0000", 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0087275489999999994, 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0089037030000000007, 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:35:44.569 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 78450 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.551368+0000", 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.551368+0000", 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:35:44.570 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.551366+0000", 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.551427+0000", 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0466e-05 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.551441+0000", 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4182000000000001e-05 2024-08-29T15:35:44.571 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.551449+0000", 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3480000000000004e-06 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553147+0000", 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001697617 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553361+0000", 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021417699999999999 2024-08-29T15:35:44.572 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,5", 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553521+0000", 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000160111 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.557674+0000", 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041531490000000001 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.558563+0000", 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00088847199999999996 2024-08-29T15:35:44.573 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:78453 8.5 8:adf6b119:::benchmark_data_smithi027_306999_object78452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e350)", 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:35:44.553644+0000", 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0064518350000000004, 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067276590000000004, 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:35:44.574 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 78453 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553644+0000", 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553644+0000", 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553643+0000", 2024-08-29T15:35:44.575 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553725+0000", 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2566999999999997e-05 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553735+0000", 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9110000000000005e-06 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:35:44.576 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.553741+0000", 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5380000000000002e-06 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.554094+0000", 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00035374799999999997 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.554227+0000", 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013254400000000001 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:35:44.577 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.554346+0000", 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011927 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.558578+0000", 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042319649999999999 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:35:44.560324+0000", 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017454599999999999 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:35:44.578 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:35:44.579 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:35:44.579 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:35:44.579 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 3 2024-08-29T15:35:44.579 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:35:44.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:44.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:44.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:44.724 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:44.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:44.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 42 16 78684 78668 117.046 78.4375 0.0115364 0.00853903 2024-08-29T15:35:44.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:44.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:44.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:44.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:44.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:45.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:45.105 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:45.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:45.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:45.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:45.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:45.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:45.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:45.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:45.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:45.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:45.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:45.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:45.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:45.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 43 15 79855 79840 116.027 73.25 0.00817804 0.00861342 2024-08-29T15:35:45.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:45.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:45.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:45.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:45.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:45.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:35:46.035 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:35:46.072 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:46.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:46.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:46.196 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:46 smithi171 ceph-mon[90031]: pgmap v1291: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:35:46.196 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:46 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2412665316' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:46.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:46.236 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:35:46.249 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:46.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:46 smithi027 ceph-mon[98970]: pgmap v1291: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:35:46.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:46 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2412665316' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:46 smithi027 ceph-mon[93921]: pgmap v1291: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:35:46.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:46 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2412665316' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:35:46.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:46.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:46.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:46.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:46.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:46.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:46.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:46.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:46.606 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:35:46.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:46.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:46.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:46.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 44 16 81113 81097 115.176 78.5625 0.0205827 0.00867719 2024-08-29T15:35:46.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:46.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:46.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:46.917 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:46.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:46.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:46.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:46.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2412665316' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[93921]: osdmap e351: 8 total, 8 up, 8 in 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2412665316' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:35:47.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:47 smithi027 ceph-mon[98970]: osdmap e351: 8 total, 8 up, 8 in 2024-08-29T15:35:47.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:47.115 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:47.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:47.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:47.217 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:47.254 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:47.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:47.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:47.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:47.367 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:35:47.367 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:35:47.367 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2024-08-29T15:35:47.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:35:47.394 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:47.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:47.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:47.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:47 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:35:47.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:47 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:35:47.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2412665316' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:35:47.521 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:47 smithi171 ceph-mon[90031]: osdmap e351: 8 total, 8 up, 8 in 2024-08-29T15:35:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:47.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:47.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:47.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:47.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:47.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:47.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 45 16 82352 82336 114.337 77.4375 0.00525348 0.00874096 2024-08-29T15:35:47.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:47.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:47.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:47.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:47.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:47.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:47.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:48.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:48.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:48.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:48.139 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:48 smithi171 ceph-mon[90031]: pgmap v1293: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:35:48.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:48.235 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:48.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:48.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:48.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:48 smithi027 ceph-mon[98970]: pgmap v1293: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:35:48.298 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:48 smithi027 ceph-mon[93921]: pgmap v1293: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:35:48.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:48.376 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:48.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:48.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:48.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:48.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:48.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:48.619 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:48.622 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:35:48.623 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:35:48.623 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:35:48.623 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:35:48.623 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:35:48.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:48.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:48.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:48.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 46 16 83592 83576 113.536 77.5 0.0115908 0.00880216 2024-08-29T15:35:48.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:48.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:35:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:48.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:48.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:48.996 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:48.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:49.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:49.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:49.161 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:49.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:49.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:49.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:49.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:49.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:35:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:35:49] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-08-29T15:35:49.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:35:49.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:35:49.322 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:35:49.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:35:49.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:49.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:49.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:49.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:49.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:49.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:49.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:49.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:49.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:49.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:49.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:49.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 47 16 84588 84572 112.445 62.25 0.0150645 0.00888773 2024-08-29T15:35:49.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:49.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:49.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:49.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:49.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:35:49.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:35:49.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:35:49.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:35:49.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:35:49.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:35:49.945 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:35:49.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:50.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:50.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:50.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:50.150 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:50.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:50 smithi171 ceph-mon[90031]: pgmap v1294: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:35:50.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2054431115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:50.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:50.291 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:50 smithi027 ceph-mon[93921]: pgmap v1294: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:35:50.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2054431115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:50.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:50 smithi027 ceph-mon[98970]: pgmap v1294: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:35:50.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2054431115' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:35:50.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:50.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.418 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:50.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:50.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:50.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:50.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:50.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:50.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:50.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 48 16 85777 85761 111.65 74.3125 0.00861406 0.00895136 2024-08-29T15:35:50.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:50.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:50.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:50.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:50.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:50.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:50.991 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:51.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:51.125 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:51.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:51.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:51.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:51.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:51.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:51.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:51.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:51.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:51.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:51.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:51.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:51.560 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:51.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:51.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:51.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:51.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:51.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:51.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 49 16 86935 86919 110.849 72.375 0.00962507 0.0090165 2024-08-29T15:35:51.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:51.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:51.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:51.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:51.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:51.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:51.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:51.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:52.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:52.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:52.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:52.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:52.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:52.249 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:52.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:52.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:52 smithi027 ceph-mon[93921]: pgmap v1295: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-29T15:35:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:52 smithi027 ceph-mon[98970]: pgmap v1295: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-29T15:35:52.352 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:52.402 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:52.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:52.457 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:35:52.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:52 smithi171 ceph-mon[90031]: pgmap v1295: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-08-29T15:35:52.549 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:52.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:52.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:52.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:52.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:52.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:52.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:52.737 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 50 16 88114 88098 110.105 73.6875 0.0150195 0.00907695 2024-08-29T15:35:52.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:52.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:52.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:52.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:52.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:35:52.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:52.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:52.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:53.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:53.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:53.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:53.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:53.263 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:53.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:53.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:53.365 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:53.394 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:53.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:53.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:53.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:53.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:53.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:53.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:53.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:53.738 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 51 16 88917 88901 108.93 50.1875 0.00391 0.00916703 2024-08-29T15:35:53.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:53.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:53.778 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:53.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:53.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:53.921 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:53.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:54.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:54.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:54.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:54.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:54.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:54.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:54.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:54.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:54.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:54.330 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:54 smithi171 ceph-mon[90031]: pgmap v1296: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-29T15:35:54.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:54.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:54.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:54.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:54 smithi027 ceph-mon[93921]: pgmap v1296: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-29T15:35:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:54 smithi027 ceph-mon[98970]: pgmap v1296: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-29T15:35:54.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:54.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:54.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:54.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:54.738 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 52 15 89974 89959 108.107 66.125 0.0193658 0.0092448 2024-08-29T15:35:54.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:54.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:54.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:54.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:54.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:54.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:55.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:55.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:55.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:55.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:55.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:55.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:55.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:55.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:55.425 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:55.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:55.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:55.668 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:55.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:55.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:55.738 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 53 16 91098 91082 107.392 70.1875 0.0235254 0.00930665 2024-08-29T15:35:55.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.848 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:55.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:55.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:55.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:55.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:56.024 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:35:56.024 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:35:56.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:56.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:56.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:56.208 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:56.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:56.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:56.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:56.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:56 smithi171 ceph-mon[90031]: pgmap v1297: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-29T15:35:56.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:56.379 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:56.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:56.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:56.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:56.518 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:56.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:56 smithi027 ceph-mon[93921]: pgmap v1297: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-29T15:35:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:56 smithi027 ceph-mon[98970]: pgmap v1297: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-29T15:35:56.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:56.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:56.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:56.738 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 54 16 92029 92013 106.48 58.1875 0.0152233 0.00938661 2024-08-29T15:35:56.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:56.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:35:56.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:56.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:56.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:57.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:57.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:57.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:57.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:57.144 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:57.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:57.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:57.282 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:57.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:57.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:57.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:57.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:57.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:57.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:57.527 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:35:57.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:57.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:57.653 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:57.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:57.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:57.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:57.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 55 16 93163 93147 105.832 70.875 0.0179292 0.00944335 2024-08-29T15:35:57.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:57.836 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:35:57.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:57.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:57.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:57.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:35:58.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:58.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:35:58.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:35:58.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:58.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:35:58.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:58 smithi171 ceph-mon[90031]: pgmap v1298: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:35:58.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:58.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:58.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:58.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:58.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:58.413 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:58 smithi027 ceph-mon[93921]: pgmap v1298: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:35:58.413 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:58 smithi027 ceph-mon[98970]: pgmap v1298: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:35:58.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:58.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:58.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:58.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:58.640 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:35:58.642 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:35:58.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:58.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:58.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 56 16 94085 94069 104.971 57.625 0.0154804 0.00952112 2024-08-29T15:35:58.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:58.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:58.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:58.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:58.923 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:58.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:59.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:35:59.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:59.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:59.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:35:59.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:59.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:59.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:59.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/792467746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:35:59.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:35:59.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:35:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:59.301 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:35:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/792467746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:35:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:35:59.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:35:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/792467746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:35:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:35:59.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:35:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:35:59.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:35:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:35:59] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.51.0" 2024-08-29T15:35:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:59.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:59.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:35:59.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:35:59.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:59.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:35:59.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:35:59.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:59.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:59.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:35:59.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:35:59.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:35:59.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:59.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:35:59.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 57 16 94718 94702 103.824 39.5625 0.0105369 0.00962664 2024-08-29T15:35:59.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:35:59.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:35:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:35:59.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:35:59.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:00.005 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:00.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:00.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:00.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:00.120 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:36:00.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[93921]: from='client.56269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:36:00.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[93921]: pgmap v1299: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:00.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:00.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/792467746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:36:00.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[93921]: osdmap e352: 8 total, 8 up, 8 in 2024-08-29T15:36:00.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[98970]: from='client.56269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:36:00.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[98970]: pgmap v1299: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:00.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:00.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/792467746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:36:00.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:00 smithi027 ceph-mon[98970]: osdmap e352: 8 total, 8 up, 8 in 2024-08-29T15:36:00.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:00.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:00.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:00 smithi171 ceph-mon[90031]: from='client.56269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:36:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:00 smithi171 ceph-mon[90031]: pgmap v1299: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:00 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/792467746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:36:00.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:00 smithi171 ceph-mon[90031]: osdmap e352: 8 total, 8 up, 8 in 2024-08-29T15:36:00.287 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:00.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:00.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:00.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:00.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:00.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:00.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:00.596 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:00.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:00.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:00.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:00.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:00.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:00.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 58 16 95799 95783 103.199 67.5625 0.0136376 0.00968365 2024-08-29T15:36:00.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:00.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:00.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:00.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:01.020 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:01.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:01.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:01.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:01.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:01.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:01.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:01.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:01.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:01.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:01.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:01.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:01.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:01.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:01.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:01.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:01.664 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1298,"stamp":"2024-08-29T15:35:56.863296+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5936519141,"num_objects":90539,"num_object_clones":0,"num_object_copies":271617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90539,"num_whiteouts":0,"num_read":6365,"num_read_kb":13822,"num_write":91634,"num_write_kb":5807745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":10166863,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91693,"ondisk_log_size":91693,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21165588,"kb_used_data":17272724,"kb_used_omap":43,"kb_used_meta":3892756,"kb_avail":728795628,"statfs":{"total":767960285184,"available":746286723072,"internally_reserved":0,"allocated":17687269376,"data_stored":17670013222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":3986182636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":932773888,"num_objects":14233,"num_object_clones":0,"num_object_copies":42699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14233,"num_write_kb":910912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.827538"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":591,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.048623+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:35:46.048623+0000","last_peered":"2024-08-29T15:35:46.048623+0000","last_clean":"2024-08-29T15:35:46.048623+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:35:46.048623+0000","last_undegraded":"2024-08-29T15:35:46.048623+0000","last_fullsized":"2024-08-29T15:35:46.048623+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":141,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987293+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:35:35.987293+0000","last_peered":"2024-08-29T15:35:35.987293+0000","last_clean":"2024-08-29T15:35:35.987293+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:35:35.987293+0000","last_undegraded":"2024-08-29T15:35:35.987293+0000","last_fullsized":"2024-08-29T15:35:35.987293+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":223,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.982720+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:35:35.982720+0000","last_peered":"2024-08-29T15:35:35.982720+0000","last_clean":"2024-08-29T15:35:35.982720+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:35:35.982720+0000","last_undegraded":"2024-08-29T15:35:35.982720+0000","last_fullsized":"2024-08-29T15:35:35.982720+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":565,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.048776+0000","last_change":"2024-08-29T15:31:26.199672+0000","last_active":"2024-08-29T15:35:46.048776+0000","last_peered":"2024-08-29T15:35:46.048776+0000","last_clean":"2024-08-29T15:35:46.048776+0000","last_became_active":"2024-08-29T15:31:26.199057+0000","last_became_peered":"2024-08-29T15:31:26.199057+0000","last_unstale":"2024-08-29T15:35:46.048776+0000","last_undegraded":"2024-08-29T15:35:46.048776+0000","last_fullsized":"2024-08-29T15:35:46.048776+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:21:31.006988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":557,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047669+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:35:46.047669+0000","last_peered":"2024-08-29T15:35:46.047669+0000","last_clean":"2024-08-29T15:35:46.047669+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:35:46.047669+0000","last_undegraded":"2024-08-29T15:35:46.047669+0000","last_fullsized":"2024-08-29T15:35:46.047669+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":134,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985424+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:35:35.985424+0000","last_peered":"2024-08-29T15:35:35.985424+0000","last_clean":"2024-08-29T15:35:35.985424+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:35:35.985424+0000","last_undegraded":"2024-08-29T15:35:35.985424+0000","last_fullsized":"2024-08-29T15:35:35.985424+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":376,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.064191+0000","last_change":"2024-08-29T15:31:26.189733+0000","last_active":"2024-08-29T15:35:46.064191+0000","last_peered":"2024-08-29T15:35:46.064191+0000","last_clean":"2024-08-29T15:35:46.064191+0000","last_became_active":"2024-08-29T15:31:26.189414+0000","last_became_peered":"2024-08-29T15:31:26.189414+0000","last_unstale":"2024-08-29T15:35:46.064191+0000","last_undegraded":"2024-08-29T15:35:46.064191+0000","last_fullsized":"2024-08-29T15:35:46.064191+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:12:46.739048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":195,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.983096+0000","last_change":"2024-08-29T15:32:26.143498+0000","last_active":"2024-08-29T15:35:35.983096+0000","last_peered":"2024-08-29T15:35:35.983096+0000","last_clean":"2024-08-29T15:35:35.983096+0000","last_became_active":"2024-08-29T15:32:26.143154+0000","last_became_peered":"2024-08-29T15:32:26.143154+0000","last_unstale":"2024-08-29T15:35:35.983096+0000","last_undegraded":"2024-08-29T15:35:35.983096+0000","last_fullsized":"2024-08-29T15:35:35.983096+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:44.284293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.4","version":"351'5699","reported_seq":5715,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.569748+0000","last_change":"2024-08-29T15:34:49.663635+0000","last_active":"2024-08-29T15:35:56.569748+0000","last_peered":"2024-08-29T15:35:56.569748+0000","last_clean":"2024-08-29T15:35:56.569748+0000","last_became_active":"2024-08-29T15:34:49.663325+0000","last_became_peered":"2024-08-29T15:34:49.663325+0000","last_unstale":"2024-08-29T15:35:56.569748+0000","last_undegraded":"2024-08-29T15:35:56.569748+0000","last_fullsized":"2024-08-29T15:35:56.569748+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:01:09.917949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373489664,"num_objects":5699,"num_object_clones":0,"num_object_copies":17097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5699,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":152,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:39.363362+0000","last_change":"2024-08-29T15:31:26.215108+0000","last_active":"2024-08-29T15:35:39.363362+0000","last_peered":"2024-08-29T15:35:39.363362+0000","last_clean":"2024-08-29T15:35:39.363362+0000","last_became_active":"2024-08-29T15:31:26.197355+0000","last_became_peered":"2024-08-29T15:31:26.197355+0000","last_unstale":"2024-08-29T15:35:39.363362+0000","last_undegraded":"2024-08-29T15:35:39.363362+0000","last_fullsized":"2024-08-29T15:35:39.363362+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:10:37.515092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":242,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.983005+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:35:35.983005+0000","last_peered":"2024-08-29T15:35:35.983005+0000","last_clean":"2024-08-29T15:35:35.983005+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:35:35.983005+0000","last_undegraded":"2024-08-29T15:35:35.983005+0000","last_fullsized":"2024-08-29T15:35:35.983005+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"351'5601","reported_seq":5616,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:54.359941+0000","last_change":"2024-08-29T15:34:49.878392+0000","last_active":"2024-08-29T15:35:54.359941+0000","last_peered":"2024-08-29T15:35:54.359941+0000","last_clean":"2024-08-29T15:35:54.359941+0000","last_became_active":"2024-08-29T15:34:49.878081+0000","last_became_peered":"2024-08-29T15:34:49.878081+0000","last_unstale":"2024-08-29T15:35:54.359941+0000","last_undegraded":"2024-08-29T15:35:54.359941+0000","last_fullsized":"2024-08-29T15:35:54.359941+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5601,"log_dups_size":0,"ondisk_log_size":5601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:39:43.629402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367067136,"num_objects":5601,"num_object_clones":0,"num_object_copies":16803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5601,"num_write_kb":358464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":139,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987669+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:35:35.987669+0000","last_peered":"2024-08-29T15:35:35.987669+0000","last_clean":"2024-08-29T15:35:35.987669+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:35:35.987669+0000","last_undegraded":"2024-08-29T15:35:35.987669+0000","last_fullsized":"2024-08-29T15:35:35.987669+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":144,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985204+0000","last_change":"2024-08-29T15:31:26.195631+0000","last_active":"2024-08-29T15:35:35.985204+0000","last_peered":"2024-08-29T15:35:35.985204+0000","last_clean":"2024-08-29T15:35:35.985204+0000","last_became_active":"2024-08-29T15:31:26.195293+0000","last_became_peered":"2024-08-29T15:31:26.195293+0000","last_unstale":"2024-08-29T15:35:35.985204+0000","last_undegraded":"2024-08-29T15:35:35.985204+0000","last_fullsized":"2024-08-29T15:35:35.985204+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:52:28.638575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"351'5670","reported_seq":5694,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:54.847410+0000","last_change":"2024-08-29T15:34:49.877713+0000","last_active":"2024-08-29T15:35:54.847410+0000","last_peered":"2024-08-29T15:35:54.847410+0000","last_clean":"2024-08-29T15:35:54.847410+0000","last_became_active":"2024-08-29T15:34:49.877477+0000","last_became_peered":"2024-08-29T15:34:49.877477+0000","last_unstale":"2024-08-29T15:35:54.847410+0000","last_undegraded":"2024-08-29T15:35:54.847410+0000","last_fullsized":"2024-08-29T15:35:54.847410+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:52.346157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371589120,"num_objects":5670,"num_object_clones":0,"num_object_copies":17010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5670,"num_write_kb":362880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":374,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047117+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:35:46.047117+0000","last_peered":"2024-08-29T15:35:46.047117+0000","last_clean":"2024-08-29T15:35:46.047117+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:35:46.047117+0000","last_undegraded":"2024-08-29T15:35:46.047117+0000","last_fullsized":"2024-08-29T15:35:46.047117+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":193,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056037+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:35:46.056037+0000","last_peered":"2024-08-29T15:35:46.056037+0000","last_clean":"2024-08-29T15:35:46.056037+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:35:46.056037+0000","last_undegraded":"2024-08-29T15:35:46.056037+0000","last_fullsized":"2024-08-29T15:35:46.056037+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"351'5654","reported_seq":5679,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.267727+0000","last_change":"2024-08-29T15:34:49.403476+0000","last_active":"2024-08-29T15:35:56.267727+0000","last_peered":"2024-08-29T15:35:56.267727+0000","last_clean":"2024-08-29T15:35:56.267727+0000","last_became_active":"2024-08-29T15:34:49.403150+0000","last_became_peered":"2024-08-29T15:34:49.403150+0000","last_unstale":"2024-08-29T15:35:56.267727+0000","last_undegraded":"2024-08-29T15:35:56.267727+0000","last_fullsized":"2024-08-29T15:35:56.267727+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5654,"log_dups_size":0,"ondisk_log_size":5654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:53:04.999655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370540544,"num_objects":5654,"num_object_clones":0,"num_object_copies":16962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5654,"num_write_kb":361856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:36.519970+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:35:36.519970+0000","last_peered":"2024-08-29T15:35:36.519970+0000","last_clean":"2024-08-29T15:35:36.519970+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:35:36.519970+0000","last_undegraded":"2024-08-29T15:35:36.519970+0000","last_fullsized":"2024-08-29T15:35:36.519970+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":572,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.046751+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:35:46.046751+0000","last_peered":"2024-08-29T15:35:46.046751+0000","last_clean":"2024-08-29T15:35:46.046751+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:35:46.046751+0000","last_undegraded":"2024-08-29T15:35:46.046751+0000","last_fullsized":"2024-08-29T15:35:46.046751+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"351'5553","reported_seq":5578,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:53.398773+0000","last_change":"2024-08-29T15:34:49.400123+0000","last_active":"2024-08-29T15:35:53.398773+0000","last_peered":"2024-08-29T15:35:53.398773+0000","last_clean":"2024-08-29T15:35:53.398773+0000","last_became_active":"2024-08-29T15:34:49.399790+0000","last_became_peered":"2024-08-29T15:34:49.399790+0000","last_unstale":"2024-08-29T15:35:53.398773+0000","last_undegraded":"2024-08-29T15:35:53.398773+0000","last_fullsized":"2024-08-29T15:35:53.398773+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:39:43.941483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363921408,"num_objects":5553,"num_object_clones":0,"num_object_copies":16659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5553,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":577,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047799+0000","last_change":"2024-08-29T15:31:26.213123+0000","last_active":"2024-08-29T15:35:46.047799+0000","last_peered":"2024-08-29T15:35:46.047799+0000","last_clean":"2024-08-29T15:35:46.047799+0000","last_became_active":"2024-08-29T15:31:26.196584+0000","last_became_peered":"2024-08-29T15:31:26.196584+0000","last_unstale":"2024-08-29T15:35:46.047799+0000","last_undegraded":"2024-08-29T15:35:46.047799+0000","last_fullsized":"2024-08-29T15:35:46.047799+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:51:23.571768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":135,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985474+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:35:35.985474+0000","last_peered":"2024-08-29T15:35:35.985474+0000","last_clean":"2024-08-29T15:35:35.985474+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:35:35.985474+0000","last_undegraded":"2024-08-29T15:35:35.985474+0000","last_fullsized":"2024-08-29T15:35:35.985474+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"351'5733","reported_seq":5758,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.289350+0000","last_change":"2024-08-29T15:34:49.664231+0000","last_active":"2024-08-29T15:35:56.289350+0000","last_peered":"2024-08-29T15:35:56.289350+0000","last_clean":"2024-08-29T15:35:56.289350+0000","last_became_active":"2024-08-29T15:34:49.663889+0000","last_became_peered":"2024-08-29T15:34:49.663889+0000","last_unstale":"2024-08-29T15:35:56.289350+0000","last_undegraded":"2024-08-29T15:35:56.289350+0000","last_fullsized":"2024-08-29T15:35:56.289350+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:19:39.280091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375717888,"num_objects":5733,"num_object_clones":0,"num_object_copies":17199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5733,"num_write_kb":366912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.986135+0000","last_change":"2024-08-29T15:32:26.140568+0000","last_active":"2024-08-29T15:35:35.986135+0000","last_peered":"2024-08-29T15:35:35.986135+0000","last_clean":"2024-08-29T15:35:35.986135+0000","last_became_active":"2024-08-29T15:32:26.140257+0000","last_became_peered":"2024-08-29T15:32:26.140257+0000","last_unstale":"2024-08-29T15:35:35.986135+0000","last_undegraded":"2024-08-29T15:35:35.986135+0000","last_fullsized":"2024-08-29T15:35:35.986135+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:04:33.833909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":135,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987018+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:35:35.987018+0000","last_peered":"2024-08-29T15:35:35.987018+0000","last_clean":"2024-08-29T15:35:35.987018+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:35:35.987018+0000","last_undegraded":"2024-08-29T15:35:35.987018+0000","last_fullsized":"2024-08-29T15:35:35.987018+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"351'5656","reported_seq":5672,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:54.360806+0000","last_change":"2024-08-29T15:34:49.398857+0000","last_active":"2024-08-29T15:35:54.360806+0000","last_peered":"2024-08-29T15:35:54.360806+0000","last_clean":"2024-08-29T15:35:54.360806+0000","last_became_active":"2024-08-29T15:34:49.398263+0000","last_became_peered":"2024-08-29T15:34:49.398263+0000","last_unstale":"2024-08-29T15:35:54.360806+0000","last_undegraded":"2024-08-29T15:35:54.360806+0000","last_fullsized":"2024-08-29T15:35:54.360806+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:23.024563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370671616,"num_objects":5656,"num_object_clones":0,"num_object_copies":16968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5656,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":135,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985277+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:35:35.985277+0000","last_peered":"2024-08-29T15:35:35.985277+0000","last_clean":"2024-08-29T15:35:35.985277+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:35:35.985277+0000","last_undegraded":"2024-08-29T15:35:35.985277+0000","last_fullsized":"2024-08-29T15:35:35.985277+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":340,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987745+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:35:35.987745+0000","last_peered":"2024-08-29T15:35:35.987745+0000","last_clean":"2024-08-29T15:35:35.987745+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:35:35.987745+0000","last_undegraded":"2024-08-29T15:35:35.987745+0000","last_fullsized":"2024-08-29T15:35:35.987745+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"351'5659","reported_seq":5684,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.277429+0000","last_change":"2024-08-29T15:34:49.399082+0000","last_active":"2024-08-29T15:35:56.277429+0000","last_peered":"2024-08-29T15:35:56.277429+0000","last_clean":"2024-08-29T15:35:56.277429+0000","last_became_active":"2024-08-29T15:34:49.398776+0000","last_became_peered":"2024-08-29T15:34:49.398776+0000","last_unstale":"2024-08-29T15:35:56.277429+0000","last_undegraded":"2024-08-29T15:35:56.277429+0000","last_fullsized":"2024-08-29T15:35:56.277429+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5659,"log_dups_size":0,"ondisk_log_size":5659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:05:00.791394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370868224,"num_objects":5659,"num_object_clones":0,"num_object_copies":16977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5659,"num_write_kb":362176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":161,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056304+0000","last_change":"2024-08-29T15:31:26.192931+0000","last_active":"2024-08-29T15:35:46.056304+0000","last_peered":"2024-08-29T15:35:46.056304+0000","last_clean":"2024-08-29T15:35:46.056304+0000","last_became_active":"2024-08-29T15:31:26.191104+0000","last_became_peered":"2024-08-29T15:31:26.191104+0000","last_unstale":"2024-08-29T15:35:46.056304+0000","last_undegraded":"2024-08-29T15:35:46.056304+0000","last_fullsized":"2024-08-29T15:35:46.056304+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:06:43.150576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":562,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.064266+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:35:46.064266+0000","last_peered":"2024-08-29T15:35:46.064266+0000","last_clean":"2024-08-29T15:35:46.064266+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:35:46.064266+0000","last_undegraded":"2024-08-29T15:35:46.064266+0000","last_fullsized":"2024-08-29T15:35:46.064266+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"351'5750","reported_seq":5765,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.579694+0000","last_change":"2024-08-29T15:34:49.398582+0000","last_active":"2024-08-29T15:35:56.579694+0000","last_peered":"2024-08-29T15:35:56.579694+0000","last_clean":"2024-08-29T15:35:56.579694+0000","last_became_active":"2024-08-29T15:34:49.398256+0000","last_became_peered":"2024-08-29T15:34:49.398256+0000","last_unstale":"2024-08-29T15:35:56.579694+0000","last_undegraded":"2024-08-29T15:35:56.579694+0000","last_fullsized":"2024-08-29T15:35:56.579694+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:35:41.421188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376832000,"num_objects":5750,"num_object_clones":0,"num_object_copies":17250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5750,"num_write_kb":368000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":663,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047279+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:35:46.047279+0000","last_peered":"2024-08-29T15:35:46.047279+0000","last_clean":"2024-08-29T15:35:46.047279+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:35:46.047279+0000","last_undegraded":"2024-08-29T15:35:46.047279+0000","last_fullsized":"2024-08-29T15:35:46.047279+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:19:19.804801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":826,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.982539+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:35:35.982539+0000","last_peered":"2024-08-29T15:35:35.982539+0000","last_clean":"2024-08-29T15:35:35.982539+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:35:35.982539+0000","last_undegraded":"2024-08-29T15:35:35.982539+0000","last_fullsized":"2024-08-29T15:35:35.982539+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":343,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.984305+0000","last_change":"2024-08-29T15:31:26.189487+0000","last_active":"2024-08-29T15:35:35.984305+0000","last_peered":"2024-08-29T15:35:35.984305+0000","last_clean":"2024-08-29T15:35:35.984305+0000","last_became_active":"2024-08-29T15:31:26.188972+0000","last_became_peered":"2024-08-29T15:31:26.188972+0000","last_unstale":"2024-08-29T15:35:35.984305+0000","last_undegraded":"2024-08-29T15:35:35.984305+0000","last_fullsized":"2024-08-29T15:35:35.984305+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:46:31.878072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":101,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.986883+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:35:35.986883+0000","last_peered":"2024-08-29T15:35:35.986883+0000","last_clean":"2024-08-29T15:35:35.986883+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:35:35.986883+0000","last_undegraded":"2024-08-29T15:35:35.986883+0000","last_fullsized":"2024-08-29T15:35:35.986883+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"351'5709","reported_seq":5734,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.264269+0000","last_change":"2024-08-29T15:34:49.665429+0000","last_active":"2024-08-29T15:35:56.264269+0000","last_peered":"2024-08-29T15:35:56.264269+0000","last_clean":"2024-08-29T15:35:56.264269+0000","last_became_active":"2024-08-29T15:34:49.664936+0000","last_became_peered":"2024-08-29T15:34:49.664936+0000","last_unstale":"2024-08-29T15:35:56.264269+0000","last_undegraded":"2024-08-29T15:35:56.264269+0000","last_fullsized":"2024-08-29T15:35:56.264269+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:28:06.384268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374145024,"num_objects":5709,"num_object_clones":0,"num_object_copies":17127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5709,"num_write_kb":365376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1035,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.055669+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:35:46.055669+0000","last_peered":"2024-08-29T15:35:46.055669+0000","last_clean":"2024-08-29T15:35:46.055669+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:35:46.055669+0000","last_undegraded":"2024-08-29T15:35:46.055669+0000","last_fullsized":"2024-08-29T15:35:46.055669+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":580,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.046299+0000","last_change":"2024-08-29T15:31:26.193377+0000","last_active":"2024-08-29T15:35:46.046299+0000","last_peered":"2024-08-29T15:35:46.046299+0000","last_clean":"2024-08-29T15:35:46.046299+0000","last_became_active":"2024-08-29T15:31:26.193034+0000","last_became_peered":"2024-08-29T15:31:26.193034+0000","last_unstale":"2024-08-29T15:35:46.046299+0000","last_undegraded":"2024-08-29T15:35:46.046299+0000","last_fullsized":"2024-08-29T15:35:46.046299+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:51.414695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":392,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.055943+0000","last_change":"2024-08-29T15:31:26.189633+0000","last_active":"2024-08-29T15:35:46.055943+0000","last_peered":"2024-08-29T15:35:46.055943+0000","last_clean":"2024-08-29T15:35:46.055943+0000","last_became_active":"2024-08-29T15:31:26.189312+0000","last_became_peered":"2024-08-29T15:31:26.189312+0000","last_unstale":"2024-08-29T15:35:46.055943+0000","last_undegraded":"2024-08-29T15:35:46.055943+0000","last_fullsized":"2024-08-29T15:35:46.055943+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:53:49.593868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"351'5588","reported_seq":5604,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:54.357455+0000","last_change":"2024-08-29T15:34:49.665017+0000","last_active":"2024-08-29T15:35:54.357455+0000","last_peered":"2024-08-29T15:35:54.357455+0000","last_clean":"2024-08-29T15:35:54.357455+0000","last_became_active":"2024-08-29T15:34:49.664771+0000","last_became_peered":"2024-08-29T15:34:49.664771+0000","last_unstale":"2024-08-29T15:35:54.357455+0000","last_undegraded":"2024-08-29T15:35:54.357455+0000","last_fullsized":"2024-08-29T15:35:54.357455+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5588,"log_dups_size":0,"ondisk_log_size":5588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:12:10.349198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366215168,"num_objects":5588,"num_object_clones":0,"num_object_copies":16764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5588,"num_write_kb":357632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":599,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.046477+0000","last_change":"2024-08-29T15:31:29.300646+0000","last_active":"2024-08-29T15:35:46.046477+0000","last_peered":"2024-08-29T15:35:46.046477+0000","last_clean":"2024-08-29T15:35:46.046477+0000","last_became_active":"2024-08-29T15:31:29.296994+0000","last_became_peered":"2024-08-29T15:31:29.296994+0000","last_unstale":"2024-08-29T15:35:46.046477+0000","last_undegraded":"2024-08-29T15:35:46.046477+0000","last_fullsized":"2024-08-29T15:35:46.046477+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:04:42.548079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":86,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985080+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:35:35.985080+0000","last_peered":"2024-08-29T15:35:35.985080+0000","last_clean":"2024-08-29T15:35:35.985080+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:35:35.985080+0000","last_undegraded":"2024-08-29T15:35:35.985080+0000","last_fullsized":"2024-08-29T15:35:35.985080+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":607,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.064067+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:35:46.064067+0000","last_peered":"2024-08-29T15:35:46.064067+0000","last_clean":"2024-08-29T15:35:46.064067+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:35:46.064067+0000","last_undegraded":"2024-08-29T15:35:46.064067+0000","last_fullsized":"2024-08-29T15:35:46.064067+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"351'5666","reported_seq":5680,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.578656+0000","last_change":"2024-08-29T15:34:49.878086+0000","last_active":"2024-08-29T15:35:56.578656+0000","last_peered":"2024-08-29T15:35:56.578656+0000","last_clean":"2024-08-29T15:35:56.578656+0000","last_became_active":"2024-08-29T15:34:49.877786+0000","last_became_peered":"2024-08-29T15:34:49.877786+0000","last_unstale":"2024-08-29T15:35:56.578656+0000","last_undegraded":"2024-08-29T15:35:56.578656+0000","last_fullsized":"2024-08-29T15:35:56.578656+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:35:28.352102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371326976,"num_objects":5666,"num_object_clones":0,"num_object_copies":16998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5666,"num_write_kb":362624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":294,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.984076+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:35:35.984076+0000","last_peered":"2024-08-29T15:35:35.984076+0000","last_clean":"2024-08-29T15:35:35.984076+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:35:35.984076+0000","last_undegraded":"2024-08-29T15:35:35.984076+0000","last_fullsized":"2024-08-29T15:35:35.984076+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.988018+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:35:35.988018+0000","last_peered":"2024-08-29T15:35:35.988018+0000","last_clean":"2024-08-29T15:35:35.988018+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:35:35.988018+0000","last_undegraded":"2024-08-29T15:35:35.988018+0000","last_fullsized":"2024-08-29T15:35:35.988018+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":201,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.983221+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:35:35.983221+0000","last_peered":"2024-08-29T15:35:35.983221+0000","last_clean":"2024-08-29T15:35:35.983221+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:35:35.983221+0000","last_undegraded":"2024-08-29T15:35:35.983221+0000","last_fullsized":"2024-08-29T15:35:35.983221+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"351'5621","reported_seq":5646,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:54.800628+0000","last_change":"2024-08-29T15:34:49.663668+0000","last_active":"2024-08-29T15:35:54.800628+0000","last_peered":"2024-08-29T15:35:54.800628+0000","last_clean":"2024-08-29T15:35:54.800628+0000","last_became_active":"2024-08-29T15:34:49.663353+0000","last_became_peered":"2024-08-29T15:34:49.663353+0000","last_unstale":"2024-08-29T15:35:54.800628+0000","last_undegraded":"2024-08-29T15:35:54.800628+0000","last_fullsized":"2024-08-29T15:35:54.800628+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5621,"log_dups_size":0,"ondisk_log_size":5621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:14:05.928215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368377856,"num_objects":5621,"num_object_clones":0,"num_object_copies":16863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5621,"num_write_kb":359744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":226,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.982645+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:35:35.982645+0000","last_peered":"2024-08-29T15:35:35.982645+0000","last_clean":"2024-08-29T15:35:35.982645+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:35:35.982645+0000","last_undegraded":"2024-08-29T15:35:35.982645+0000","last_fullsized":"2024-08-29T15:35:35.982645+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1000,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056512+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:35:46.056512+0000","last_peered":"2024-08-29T15:35:46.056512+0000","last_clean":"2024-08-29T15:35:46.056512+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:35:46.056512+0000","last_undegraded":"2024-08-29T15:35:46.056512+0000","last_fullsized":"2024-08-29T15:35:46.056512+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":577,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056351+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:35:46.056351+0000","last_peered":"2024-08-29T15:35:46.056351+0000","last_clean":"2024-08-29T15:35:46.056351+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:35:46.056351+0000","last_undegraded":"2024-08-29T15:35:46.056351+0000","last_fullsized":"2024-08-29T15:35:46.056351+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"351'5545","reported_seq":5570,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:53.379721+0000","last_change":"2024-08-29T15:34:49.400720+0000","last_active":"2024-08-29T15:35:53.379721+0000","last_peered":"2024-08-29T15:35:53.379721+0000","last_clean":"2024-08-29T15:35:53.379721+0000","last_became_active":"2024-08-29T15:34:49.400373+0000","last_became_peered":"2024-08-29T15:34:49.400373+0000","last_unstale":"2024-08-29T15:35:53.379721+0000","last_undegraded":"2024-08-29T15:35:53.379721+0000","last_fullsized":"2024-08-29T15:35:53.379721+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:24:43.709510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363397120,"num_objects":5545,"num_object_clones":0,"num_object_copies":16635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5545,"num_write_kb":354880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":582,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056758+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:35:46.056758+0000","last_peered":"2024-08-29T15:35:46.056758+0000","last_clean":"2024-08-29T15:35:46.056758+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:35:46.056758+0000","last_undegraded":"2024-08-29T15:35:46.056758+0000","last_fullsized":"2024-08-29T15:35:46.056758+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":806,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987170+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:35:35.987170+0000","last_peered":"2024-08-29T15:35:35.987170+0000","last_clean":"2024-08-29T15:35:35.987170+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:35:35.987170+0000","last_undegraded":"2024-08-29T15:35:35.987170+0000","last_fullsized":"2024-08-29T15:35:35.987170+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":573,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.046657+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:35:46.046657+0000","last_peered":"2024-08-29T15:35:46.046657+0000","last_clean":"2024-08-29T15:35:46.046657+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:35:46.046657+0000","last_undegraded":"2024-08-29T15:35:46.046657+0000","last_fullsized":"2024-08-29T15:35:46.046657+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"351'5703","reported_seq":5728,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:53.352469+0000","last_change":"2024-08-29T15:34:49.878565+0000","last_active":"2024-08-29T15:35:53.352469+0000","last_peered":"2024-08-29T15:35:53.352469+0000","last_clean":"2024-08-29T15:35:53.352469+0000","last_became_active":"2024-08-29T15:34:49.878202+0000","last_became_peered":"2024-08-29T15:34:49.878202+0000","last_unstale":"2024-08-29T15:35:53.352469+0000","last_undegraded":"2024-08-29T15:35:53.352469+0000","last_fullsized":"2024-08-29T15:35:53.352469+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:14:20.836128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373751808,"num_objects":5703,"num_object_clones":0,"num_object_copies":17109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5703,"num_write_kb":364992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":566,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.051286+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:35:46.051286+0000","last_peered":"2024-08-29T15:35:46.051286+0000","last_clean":"2024-08-29T15:35:46.051286+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:35:46.051286+0000","last_undegraded":"2024-08-29T15:35:46.051286+0000","last_fullsized":"2024-08-29T15:35:46.051286+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1218,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056206+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:35:46.056206+0000","last_peered":"2024-08-29T15:35:46.056206+0000","last_clean":"2024-08-29T15:35:46.056206+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:35:46.056206+0000","last_undegraded":"2024-08-29T15:35:46.056206+0000","last_fullsized":"2024-08-29T15:35:46.056206+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":94,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.055451+0000","last_change":"2024-08-29T15:31:26.189321+0000","last_active":"2024-08-29T15:35:46.055451+0000","last_peered":"2024-08-29T15:35:46.055451+0000","last_clean":"2024-08-29T15:35:46.055451+0000","last_became_active":"2024-08-29T15:31:26.188778+0000","last_became_peered":"2024-08-29T15:31:26.188778+0000","last_unstale":"2024-08-29T15:35:46.055451+0000","last_undegraded":"2024-08-29T15:35:46.055451+0000","last_fullsized":"2024-08-29T15:35:46.055451+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:11:57.644178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.d","version":"351'5704","reported_seq":5720,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:56.568058+0000","last_change":"2024-08-29T15:34:49.878253+0000","last_active":"2024-08-29T15:35:56.568058+0000","last_peered":"2024-08-29T15:35:56.568058+0000","last_clean":"2024-08-29T15:35:56.568058+0000","last_became_active":"2024-08-29T15:34:49.877999+0000","last_became_peered":"2024-08-29T15:34:49.877999+0000","last_unstale":"2024-08-29T15:35:56.568058+0000","last_undegraded":"2024-08-29T15:35:56.568058+0000","last_fullsized":"2024-08-29T15:35:56.568058+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:49:48.499809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373817344,"num_objects":5704,"num_object_clones":0,"num_object_copies":17112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5704,"num_write_kb":365056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":120,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056045+0000","last_change":"2024-08-29T15:31:26.211099+0000","last_active":"2024-08-29T15:35:46.056045+0000","last_peered":"2024-08-29T15:35:46.056045+0000","last_clean":"2024-08-29T15:35:46.056045+0000","last_became_active":"2024-08-29T15:31:26.196998+0000","last_became_peered":"2024-08-29T15:31:26.196998+0000","last_unstale":"2024-08-29T15:35:46.056045+0000","last_undegraded":"2024-08-29T15:35:46.056045+0000","last_fullsized":"2024-08-29T15:35:46.056045+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:07:19.391161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":207,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987306+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:35:35.987306+0000","last_peered":"2024-08-29T15:35:35.987306+0000","last_clean":"2024-08-29T15:35:35.987306+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:35:35.987306+0000","last_undegraded":"2024-08-29T15:35:35.987306+0000","last_fullsized":"2024-08-29T15:35:35.987306+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":135,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985349+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:35:35.985349+0000","last_peered":"2024-08-29T15:35:35.985349+0000","last_clean":"2024-08-29T15:35:35.985349+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:35:35.985349+0000","last_undegraded":"2024-08-29T15:35:35.985349+0000","last_fullsized":"2024-08-29T15:35:35.985349+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987371+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:35:35.987371+0000","last_peered":"2024-08-29T15:35:35.987371+0000","last_clean":"2024-08-29T15:35:35.987371+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:35:35.987371+0000","last_undegraded":"2024-08-29T15:35:35.987371+0000","last_fullsized":"2024-08-29T15:35:35.987371+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":144,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985518+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:35:35.985518+0000","last_peered":"2024-08-29T15:35:35.985518+0000","last_clean":"2024-08-29T15:35:35.985518+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:35:35.985518+0000","last_undegraded":"2024-08-29T15:35:35.985518+0000","last_fullsized":"2024-08-29T15:35:35.985518+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":571,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056242+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:35:46.056242+0000","last_peered":"2024-08-29T15:35:46.056242+0000","last_clean":"2024-08-29T15:35:46.056242+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:35:46.056242+0000","last_undegraded":"2024-08-29T15:35:46.056242+0000","last_fullsized":"2024-08-29T15:35:46.056242+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":329,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047791+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:35:46.047791+0000","last_peered":"2024-08-29T15:35:46.047791+0000","last_clean":"2024-08-29T15:35:46.047791+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:35:46.047791+0000","last_undegraded":"2024-08-29T15:35:46.047791+0000","last_fullsized":"2024-08-29T15:35:46.047791+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":223,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.982644+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:35:35.982644+0000","last_peered":"2024-08-29T15:35:35.982644+0000","last_clean":"2024-08-29T15:35:35.982644+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:35:35.982644+0000","last_undegraded":"2024-08-29T15:35:35.982644+0000","last_fullsized":"2024-08-29T15:35:35.982644+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":529,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047385+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:35:46.047385+0000","last_peered":"2024-08-29T15:35:46.047385+0000","last_clean":"2024-08-29T15:35:46.047385+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:35:46.047385+0000","last_undegraded":"2024-08-29T15:35:46.047385+0000","last_fullsized":"2024-08-29T15:35:46.047385+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":583,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047309+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:35:46.047309+0000","last_peered":"2024-08-29T15:35:46.047309+0000","last_clean":"2024-08-29T15:35:46.047309+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:35:46.047309+0000","last_undegraded":"2024-08-29T15:35:46.047309+0000","last_fullsized":"2024-08-29T15:35:46.047309+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":581,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.049234+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:35:46.049234+0000","last_peered":"2024-08-29T15:35:46.049234+0000","last_clean":"2024-08-29T15:35:46.049234+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:35:46.049234+0000","last_undegraded":"2024-08-29T15:35:46.049234+0000","last_fullsized":"2024-08-29T15:35:46.049234+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.982779+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:35:35.982779+0000","last_peered":"2024-08-29T15:35:35.982779+0000","last_clean":"2024-08-29T15:35:35.982779+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:35:35.982779+0000","last_undegraded":"2024-08-29T15:35:35.982779+0000","last_fullsized":"2024-08-29T15:35:35.982779+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":93,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.049138+0000","last_change":"2024-08-29T15:31:26.195212+0000","last_active":"2024-08-29T15:35:46.049138+0000","last_peered":"2024-08-29T15:35:46.049138+0000","last_clean":"2024-08-29T15:35:46.049138+0000","last_became_active":"2024-08-29T15:31:26.194866+0000","last_became_peered":"2024-08-29T15:31:26.194866+0000","last_unstale":"2024-08-29T15:35:46.049138+0000","last_undegraded":"2024-08-29T15:35:46.049138+0000","last_fullsized":"2024-08-29T15:35:46.049138+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:50:45.933437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":574,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047757+0000","last_change":"2024-08-29T15:34:31.566349+0000","last_active":"2024-08-29T15:35:46.047757+0000","last_peered":"2024-08-29T15:35:46.047757+0000","last_clean":"2024-08-29T15:35:46.047757+0000","last_became_active":"2024-08-29T15:34:31.566045+0000","last_became_peered":"2024-08-29T15:34:31.566045+0000","last_unstale":"2024-08-29T15:35:46.047757+0000","last_undegraded":"2024-08-29T15:35:46.047757+0000","last_fullsized":"2024-08-29T15:35:46.047757+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:36:21.006547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":562,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.055759+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:35:46.055759+0000","last_peered":"2024-08-29T15:35:46.055759+0000","last_clean":"2024-08-29T15:35:46.055759+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:35:46.055759+0000","last_undegraded":"2024-08-29T15:35:46.055759+0000","last_fullsized":"2024-08-29T15:35:46.055759+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":565,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047641+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:35:46.047641+0000","last_peered":"2024-08-29T15:35:46.047641+0000","last_clean":"2024-08-29T15:35:46.047641+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:35:46.047641+0000","last_undegraded":"2024-08-29T15:35:46.047641+0000","last_fullsized":"2024-08-29T15:35:46.047641+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":134,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.986222+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:35:35.986222+0000","last_peered":"2024-08-29T15:35:35.986222+0000","last_clean":"2024-08-29T15:35:35.986222+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:35:35.986222+0000","last_undegraded":"2024-08-29T15:35:35.986222+0000","last_fullsized":"2024-08-29T15:35:35.986222+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":593,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.056622+0000","last_change":"2024-08-29T15:32:56.055196+0000","last_active":"2024-08-29T15:35:46.056622+0000","last_peered":"2024-08-29T15:35:46.056622+0000","last_clean":"2024-08-29T15:35:46.056622+0000","last_became_active":"2024-08-29T15:32:56.054876+0000","last_became_peered":"2024-08-29T15:32:56.054876+0000","last_unstale":"2024-08-29T15:35:46.056622+0000","last_undegraded":"2024-08-29T15:35:46.056622+0000","last_fullsized":"2024-08-29T15:35:46.056622+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:46:26.627465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":68,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.987451+0000","last_change":"2024-08-29T15:31:01.734785+0000","last_active":"2024-08-29T15:35:35.987451+0000","last_peered":"2024-08-29T15:35:35.987451+0000","last_clean":"2024-08-29T15:35:35.987451+0000","last_became_active":"2024-08-29T15:31:01.734449+0000","last_became_peered":"2024-08-29T15:31:01.734449+0000","last_unstale":"2024-08-29T15:35:35.987451+0000","last_undegraded":"2024-08-29T15:35:35.987451+0000","last_fullsized":"2024-08-29T15:35:35.987451+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:18:36.797427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":134,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.986541+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:35:35.986541+0000","last_peered":"2024-08-29T15:35:35.986541+0000","last_clean":"2024-08-29T15:35:35.986541+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:35:35.986541+0000","last_undegraded":"2024-08-29T15:35:35.986541+0000","last_fullsized":"2024-08-29T15:35:35.986541+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":266,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.982971+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:35:35.982971+0000","last_peered":"2024-08-29T15:35:35.982971+0000","last_clean":"2024-08-29T15:35:35.982971+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:35:35.982971+0000","last_undegraded":"2024-08-29T15:35:35.982971+0000","last_fullsized":"2024-08-29T15:35:35.982971+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":134,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.986295+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:35:35.986295+0000","last_peered":"2024-08-29T15:35:35.986295+0000","last_clean":"2024-08-29T15:35:35.986295+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:35:35.986295+0000","last_undegraded":"2024-08-29T15:35:35.986295+0000","last_fullsized":"2024-08-29T15:35:35.986295+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":266,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.982862+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:35:35.982862+0000","last_peered":"2024-08-29T15:35:35.982862+0000","last_clean":"2024-08-29T15:35:35.982862+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:35:35.982862+0000","last_undegraded":"2024-08-29T15:35:35.982862+0000","last_fullsized":"2024-08-29T15:35:35.982862+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":597,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047569+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:35:46.047569+0000","last_peered":"2024-08-29T15:35:46.047569+0000","last_clean":"2024-08-29T15:35:46.047569+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:35:46.047569+0000","last_undegraded":"2024-08-29T15:35:46.047569+0000","last_fullsized":"2024-08-29T15:35:46.047569+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":599,"reported_epoch":351,"state":"active+clean","last_fresh":"2024-08-29T15:35:46.047943+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:35:46.047943+0000","last_peered":"2024-08-29T15:35:46.047943+0000","last_clean":"2024-08-29T15:35:46.047943+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:35:46.047943+0000","last_undegraded":"2024-08-29T15:35:46.047943+0000","last_fullsized":"2024-08-29T15:35:46.047943+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":134,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:35.985209+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:35:35.985209+0000","last_peered":"2024-08-29T15:35:35.985209+0000","last_clean":"2024-08-29T15:35:35.985209+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:35:35.985209+0000","last_undegraded":"2024-08-29T15:35:35.985209+0000","last_fullsized":"2024-08-29T15:35:35.985209+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":232,"reported_epoch":350,"state":"active+clean","last_fresh":"2024-08-29T15:35:36.520031+0000","last_change":"2024-08-29T15:32:56.042376+0000","last_active":"2024-08-29T15:35:36.520031+0000","last_peered":"2024-08-29T15:35:36.520031+0000","last_clean":"2024-08-29T15:35:36.520031+0000","last_became_active":"2024-08-29T15:32:56.036129+0000","last_became_peered":"2024-08-29T15:32:56.036129+0000","last_unstale":"2024-08-29T15:35:36.520031+0000","last_undegraded":"2024-08-29T15:35:36.520031+0000","last_fullsized":"2024-08-29T15:35:36.520031+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:33:55.308823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5931728896,"num_objects":90511,"num_object_clones":0,"num_object_copies":271533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90511,"num_write_kb":5792704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17743806464,"data_stored":17743806464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90511,"ondisk_log_size":90511,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":7132,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234495,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2351028,"kb_used_data":1802284,"kb_used_omap":1,"kb_used_meta":548734,"kb_avail":91394124,"statfs":{"total":95995035648,"available":93587582976,"internally_reserved":0,"allocated":1845538816,"data_stored":1843367612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":561904075},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496126,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2226736,"kb_used_data":1798568,"kb_used_omap":1,"kb_used_meta":428158,"kb_avail":91518416,"statfs":{"total":95995035648,"available":93714857984,"internally_reserved":0,"allocated":1841733632,"data_stored":1839566776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":438434247},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426713,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3525852,"kb_used_data":3294356,"kb_used_omap":8,"kb_used_meta":231479,"kb_avail":90219300,"statfs":{"total":95995035648,"available":92384563200,"internally_reserved":0,"allocated":3373420544,"data_stored":3371268740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":237035402},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019405,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2705264,"kb_used_data":2142568,"kb_used_omap":1,"kb_used_meta":562686,"kb_avail":91039888,"statfs":{"total":95995035648,"available":93224845312,"internally_reserved":0,"allocated":2193989632,"data_stored":2191822289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":576190918},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435589,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2368540,"kb_used_data":1777428,"kb_used_omap":4,"kb_used_meta":591099,"kb_avail":91376612,"statfs":{"total":95995035648,"available":93569650688,"internally_reserved":0,"allocated":1820086272,"data_stored":1817934300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":605285641},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903441,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2533716,"kb_used_data":2113228,"kb_used_omap":10,"kb_used_meta":420469,"kb_avail":91211436,"statfs":{"total":95995035648,"available":93400510464,"internally_reserved":0,"allocated":2163945472,"data_stored":2161800456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":430560784},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762643,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3095452,"kb_used_data":2540372,"kb_used_omap":7,"kb_used_meta":555064,"kb_avail":90649700,"statfs":{"total":95995035648,"available":92825292800,"internally_reserved":0,"allocated":2601340928,"data_stored":2599187762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":568385890},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926169,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2359000,"kb_used_data":1803920,"kb_used_omap":7,"kb_used_meta":555064,"kb_avail":91386152,"statfs":{"total":95995035648,"available":93579419648,"internally_reserved":0,"allocated":1847214080,"data_stored":1845065287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":568385679},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1853227008,"data_stored":1853227008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2615934976,"data_stored":2615934976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2176057344,"data_stored":2176057344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1820196864,"data_stored":1820196864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2199715840,"data_stored":2199715840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3390439424,"data_stored":3390439424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1847656448,"data_stored":1847656448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1840578560,"data_stored":1840578560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:36:01.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool set unique_pool_3 pgp_num 16 2024-08-29T15:36:01.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:01.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:01.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:01.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 59 16 96846 96830 102.559 65.4375 0.0146118 0.00974508 2024-08-29T15:36:01.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:01.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:01.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:01.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:01.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:02.003 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:02.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:02.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:02.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:02.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:02.255 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:36:02.255 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:36:02.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:02.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:02.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:02.339 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:02 smithi027 ceph-mon[93921]: pgmap v1301: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:36:02.339 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:02 smithi027 ceph-mon[98970]: pgmap v1301: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:36:02.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:02.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:02.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:02 smithi171 ceph-mon[90031]: pgmap v1301: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:36:02.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:02.611 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:02.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:02.696 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:02.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:02.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:36:02.739127+0000 min lat: 0.00117312 max lat: 0.294735 avg lat: 0.00980471 2024-08-29T15:36:02.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:36:02.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 60 16 97888 97872 101.935 65.125 0.0187831 0.00980471 2024-08-29T15:36:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:02.774 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:02.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:02.875 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:02.909 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:02.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:02.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:03.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:03.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:36:03.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:03.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:03.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:03.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:03.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:03.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:03.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:03.312 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:03.312 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:03.312 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:03.312 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:03.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:03.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:03.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:03.405 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:03.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:03.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:03.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:03.543 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:03.543 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:03.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:03.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:03.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:03.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:03.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 61 15 98934 98919 101.336 65.4375 0.0118314 0.00986319 2024-08-29T15:36:03.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:03.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:03.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:03.897 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:03.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:03.998 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:04.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:04.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:04.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:04.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:04.172 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:04.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:04.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:36:04.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:04.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:04.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:04.317 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:36:04.317 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:36:04.360 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:36:04.360 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:36:04.397 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:36:04.397 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:36:04.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:04.433 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:36:04.433 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:36:04.434 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:04.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:04.469 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:36:04.469 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:36:04.501 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:04 smithi027 ceph-mon[98970]: pgmap v1302: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:04.501 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3036757645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:04.502 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:04 smithi027 ceph-mon[93921]: pgmap v1302: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:04.502 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3036757645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:04.502 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:36:04.502 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:36:04.535 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:36:04.536 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:36:04.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:04.568 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:36:04.569 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:36:04.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:04.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:04.597 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:04 smithi171 ceph-mon[90031]: pgmap v1302: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:04.597 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3036757645' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:04.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:04.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:04.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:04.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 62 16 99973 99957 100.748 64.875 0.0081445 0.00992006 2024-08-29T15:36:04.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:04.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:04.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:04.984 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:05.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:05.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:05.146 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:05.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:05.285 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:05.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:05.425 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:36:05.426 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:36:05.426 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:36:05.426 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:36:05.426 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:36:05.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:05.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:05.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:05.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:05.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:05.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:05.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:05.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 63 16 100919 100903 100.087 59.125 0.12108 0.00997594 2024-08-29T15:36:05.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:05.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:05.920 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:05.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:05.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:06.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:06.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:06.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:06.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:06.223 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:06.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:06.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:06.308 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:06 smithi027 ceph-mon[93921]: pgmap v1303: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2024-08-29T15:36:06.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:06.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:06.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:06.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:06.500 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:06.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:06.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:06.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:06 smithi027 ceph-mon[98970]: pgmap v1303: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2024-08-29T15:36:06.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:06.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:06.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:06.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:06.739 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 64 16 102018 102002 99.5964 68.6875 0.0102681 0.0100348 2024-08-29T15:36:06.742 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:06.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:06.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:06 smithi171 ceph-mon[90031]: pgmap v1303: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2024-08-29T15:36:06.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:06.934 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:06.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:07.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:07.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:07.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:07.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:07.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:07.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:07.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:07.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:07.354 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:07.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:07.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:07.488 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:07.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:07.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:07.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:07.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:07.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:07.740 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 65 16 103053 103037 99.0593 64.6875 0.0162551 0.0100896 2024-08-29T15:36:07.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:07.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:07.888 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:07.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:07.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:08.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:08.047 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:08.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:08.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:08.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:08.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:08.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:08.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:08.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:08.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:08.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:36:08.455 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:08.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:08.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:08.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:08 smithi027 ceph-mon[98970]: pgmap v1304: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-29T15:36:08.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:08 smithi027 ceph-mon[93921]: pgmap v1304: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-29T15:36:08.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:08.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:08.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:08.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:08.731 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:08 smithi171 ceph-mon[90031]: pgmap v1304: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-29T15:36:08.740 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 66 16 104025 104009 98.4788 60.75 0.00522464 0.0101482 2024-08-29T15:36:08.783 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:08.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:08.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:08.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:08.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:36:08.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:08.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:08.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:09.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:09.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:09.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:09.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:09.161 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:36:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:36:09] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.51.0" 2024-08-29T15:36:09.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:09.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:09.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:09.319 INFO:teuthology.orchestra.run.smithi027.stderr:set pool 8 pgp_num to 16 2024-08-29T15:36:09.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:09.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:09.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:09.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:09.538 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:09.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:09.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:09 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4142351389' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-29T15:36:09.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:09 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4142351389' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-29T15:36:09.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:09.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:09.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:09.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:09.740 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 67 16 104956 104940 97.8774 58.1875 0.0194006 0.0102114 2024-08-29T15:36:09.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:09 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4142351389' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-29T15:36:09.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:09.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:09.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:09.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:09.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:09.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:09.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:10.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:10.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:10.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:10.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:10.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:10.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:10.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:10.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:10.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:10.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:10.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:10.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:10.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:10.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:10 smithi171 ceph-mon[90031]: pgmap v1305: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-08-29T15:36:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4142351389' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-29T15:36:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:10 smithi171 ceph-mon[90031]: osdmap e353: 8 total, 8 up, 8 in 2024-08-29T15:36:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:10 smithi027 ceph-mon[93921]: pgmap v1305: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-08-29T15:36:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4142351389' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-29T15:36:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:10 smithi027 ceph-mon[93921]: osdmap e353: 8 total, 8 up, 8 in 2024-08-29T15:36:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:10 smithi027 ceph-mon[98970]: pgmap v1305: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2024-08-29T15:36:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4142351389' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-29T15:36:10.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:10 smithi027 ceph-mon[98970]: osdmap e353: 8 total, 8 up, 8 in 2024-08-29T15:36:10.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:10.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:10.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:10.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:10.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 68 16 105983 105967 97.378 64.1875 0.0126861 0.0102637 2024-08-29T15:36:10.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:10.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:10.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:10.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:10.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:221963 6.3 6:c8074485:::benchmark_data_smithi027_169445_object96536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:36:10.929 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 525.517433944, 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.875189047, 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221963 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241084+0000", 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241082+0000", 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241090+0000", 2024-08-29T15:36:10.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6760000000000007e-06 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241103+0000", 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3097e-05 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.241110+0000", 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1189999999999999e-06 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.286106+0000", 2024-08-29T15:36:10.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.044995852000000003 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.289616+0000", 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035105090000000002 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.307511+0000", 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017894131000000001 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.309413+0000", 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019022659999999999 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.986973+0000", 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.67756063300000002 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.989188+0000", 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022149930000000002 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.993558+0000", 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043696730000000001 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.022139+0000", 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028581273000000001 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.043843+0000", 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021703999000000002 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.110257+0000", 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066413278000000006 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.936 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.111466+0000", 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001209578 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.116273+0000", 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0048067459999999998 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:221967 6.3 6:c4ba2f51:::benchmark_data_smithi027_169445_object96540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:36:10.937 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 525.49880527000005, 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86481450699999995, 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221967 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259713+0000", 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259711+0000", 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259717+0000", 2024-08-29T15:36:10.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4289999999999998e-06 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259726+0000", 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8389999999999996e-06 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.259731+0000", 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3930000000000001e-06 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.289939+0000", 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.03020888 2024-08-29T15:36:10.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.292731+0000", 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027917939999999998 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.344052+0000", 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.051320328999999998 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:10.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:25.345748+0000", 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016965680000000001 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.009894+0000", 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.66414622700000003 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.012206+0000", 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0023118190000000001 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.016501+0000", 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042943189999999996 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.076191+0000", 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.059690875999999997 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.944 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.117656+0000", 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041464320999999998 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.118388+0000", 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00073274099999999997 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.119598+0000", 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012098269999999999 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.124527+0000", 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049288099999999996 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:222226 6.3 6:c82e9241:::benchmark_data_smithi027_169445_object96799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 524.46616338299998, 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86422102700000003, 2024-08-29T15:36:10.946 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 222226 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292354+0000", 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292353+0000", 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292360+0000", 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4229999999999999e-06 2024-08-29T15:36:10.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292373+0000", 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.291e-05 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.292381+0000", 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1340000000000004e-06 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.308321+0000", 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015940122000000001 2024-08-29T15:36:10.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.310974+0000", 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026525699999999999 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.338098+0000", 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027124519999999999 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:26.339549+0000", 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001450724 2024-08-29T15:36:10.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.061922+0000", 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.72237274799999995 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.064096+0000", 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021742850000000002 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.068373+0000", 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042769779999999999 2024-08-29T15:36:10.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.099354+0000", 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030980508 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.100191+0000", 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00083792800000000005 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.149351+0000", 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049159714 2024-08-29T15:36:10.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.150692+0000", 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013410850000000001 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.156575+0000", 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0058832499999999996 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:222756 6.3 6:c4cc6026:::benchmark_data_smithi027_169445_object97329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 522.96262417499997, 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86913315599999996, 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 222756 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.954 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795894+0000", 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795892+0000", 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795898+0000", 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4450000000000001e-06 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795911+0000", 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3353000000000001e-05 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.795918+0000", 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0160000000000003e-06 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.800436+0000", 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045175850000000002 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.801672+0000", 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012355319999999999 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.890056+0000", 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.088384024000000005 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:27.891521+0000", 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014655689999999999 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.630976+0000", 2024-08-29T15:36:10.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.73945498399999998 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.633483+0000", 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025067330000000001 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.637874+0000", 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0043910590000000001 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.657485+0000", 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019610737 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.658230+0000", 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00074571299999999995 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.658976+0000", 2024-08-29T15:36:10.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00074594099999999997 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.660269+0000", 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012926579999999999 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:28.665027+0000", 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004757809 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:230075 6.3 6:c4c54f8a:::benchmark_data_smithi027_169445_object104648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 501.76474569200002, 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.89382495100000003, 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 230075 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.962 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993772+0000", 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993771+0000", 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993778+0000", 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8019999999999992e-06 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993792+0000", 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3959e-05 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:48.993799+0000", 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0040000000000002e-06 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.231044+0000", 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.237244607 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.232235+0000", 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001191454 2024-08-29T15:36:10.965 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.388426+0000", 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.156190521 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.389841+0000", 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001415531 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.868908+0000", 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.479066673 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.870800+0000", 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018920810000000001 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.874105+0000", 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033051000000000001 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.881899+0000", 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0077939180000000004 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.882530+0000", 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00063088800000000002 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.882723+0000", 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000193396 2024-08-29T15:36:10.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.883672+0000", 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00094804899999999996 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:49.887597+0000", 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039255740000000003 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.44912.0:231480 6.3 6:c1b858f2:::benchmark_data_smithi027_169445_object106053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 498.27343207299998, 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.92456820500000003, 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.44912", 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1813127562", 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 231480 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.970 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485086+0000", 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485084+0000", 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485091+0000", 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485105+0000", 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3947999999999999e-05 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.485113+0000", 2024-08-29T15:36:10.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5220000000000001e-06 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.563897+0000", 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.078784891999999995 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for missing object", 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.565127+0000", 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012290720000000001 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.667308+0000", 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10218187500000001 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:52.668461+0000", 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001152882 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.377239+0000", 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.70877815099999997 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.379283+0000", 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002043703 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.383071+0000", 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037881690000000001 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.402937+0000", 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019865765000000001 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.403638+0000", 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000700682 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.403872+0000", 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00023457 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.405059+0000", 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00118629 2024-08-29T15:36:10.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:27:53.409654+0000", 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045953820000000003 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:106267 7.d 7:b9baa89f:::benchmark_data_smithi027_271924_object106266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:29.580297+0000", 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 341.17822076599998, 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16656731, 2024-08-29T15:36:10.977 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 106267 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580297+0000", 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580297+0000", 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580295+0000", 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580346+0000", 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0374999999999998e-05 2024-08-29T15:36:10.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580362+0000", 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5965e-05 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580369+0000", 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7000000000000008e-06 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580414+0000", 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4737000000000002e-05 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580559+0000", 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014460800000000001 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.580695+0000", 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013610799999999999 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.583531+0000", 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028361409999999999 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.584462+0000", 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00093147399999999995 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.746807+0000", 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.162344087 2024-08-29T15:36:10.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.746840+0000", 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3581000000000003e-05 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:29.746864+0000", 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4261999999999999e-05 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107281 7.7 7:edf7da9f:::benchmark_data_smithi027_271924_object107280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.862099+0000", 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 339.89641906200001, 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.16102459599999999, 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107281 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.984 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862099+0000", 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862099+0000", 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862097+0000", 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862145+0000", 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8124000000000002e-05 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862164+0000", 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8284999999999999e-05 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862172+0000", 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9820000000000008e-06 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862217+0000", 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5518e-05 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862360+0000", 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014341899999999999 2024-08-29T15:36:10.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,3", 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.862497+0000", 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013688700000000001 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.866010+0000", 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035125099999999999 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.878345+0000", 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012334921 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023052+0000", 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14470682000000001 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023086+0000", 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4050000000000001e-05 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.989 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023123+0000", 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7694999999999997e-05 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107287 7.7 7:effdeacc:::benchmark_data_smithi027_271924_object107286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.867285+0000", 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 339.89123295500002, 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15597306499999999, 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.990 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107287 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867285+0000", 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867285+0000", 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867283+0000", 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867337+0000", 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3868e-05 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867349+0000", 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1878999999999999e-05 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867356+0000", 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3899999999999998e-06 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867396+0000", 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0698999999999998e-05 2024-08-29T15:36:10.993 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867532+0000", 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013618699999999999 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,3", 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.867687+0000", 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015432600000000001 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.994 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.870851+0000", 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031643180000000002 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.878387+0000", 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075357169999999999 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023171+0000", 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.144784568 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.995 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023214+0000", 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2509999999999998e-05 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.023258+0000", 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4023000000000002e-05 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.996 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:107292 7.c 7:33491277:::benchmark_data_smithi027_271924_object107291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 339.88886544600001, 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.141884811, 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 107292 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.997 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869652+0000", 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:10.998 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869732+0000", 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.017e-05 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869745+0000", 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3499000000000001e-05 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.869752+0000", 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4799999999999998e-06 2024-08-29T15:36:10.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.873032+0000", 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032800160000000002 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.873166+0000", 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013469400000000001 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,7", 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.873309+0000", 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014280900000000001 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.876602+0000", 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032927659999999999 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:30.876615+0000", 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3535e-05 2024-08-29T15:36:11.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.011451+0000", 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.134835548 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.011503+0000", 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2157000000000002e-05 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:31.011537+0000", 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4010999999999997e-05 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110071 7.d 7:b7d1d271:::benchmark_data_smithi027_271924_object110070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:34.619346+0000", 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.139172019, 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.259612114, 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.003 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110071 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619346+0000", 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619346+0000", 2024-08-29T15:36:11.004 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619345+0000", 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619412+0000", 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7166e-05 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.005 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619423+0000", 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1192000000000001e-05 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619429+0000", 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4359999999999999e-06 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619498+0000", 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8756000000000004e-05 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.006 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619639+0000", 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014112599999999999 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.619786+0000", 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014683200000000001 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.007 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.625738+0000", 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0059519899999999999 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.626604+0000", 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00086641699999999997 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878875+0000", 2024-08-29T15:36:11.008 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25227093099999998 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878915+0000", 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9335000000000002e-05 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878958+0000", 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3220999999999998e-05 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.009 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:110075 7.d 7:bd8da132:::benchmark_data_smithi027_271924_object110074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:30:34.622747+0000", 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.13577048000002, 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.256296413, 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 110075 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.010 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622747+0000", 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622747+0000", 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622746+0000", 2024-08-29T15:36:11.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622789+0000", 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3031000000000002e-05 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622800+0000", 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0584999999999999e-05 2024-08-29T15:36:11.012 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622807+0000", 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6760000000000001e-06 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622864+0000", 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7042e-05 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.622953+0000", 2024-08-29T15:36:11.013 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9240999999999995e-05 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.623047+0000", 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3597000000000005e-05 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.625811+0000", 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027641050000000002 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.014 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.626752+0000", 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00094161500000000003 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.878990+0000", 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25223777400000003 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.879014+0000", 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4241999999999999e-05 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:30:34.879044+0000", 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9402000000000001e-05 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148244 7.c 7:3ee65318:::benchmark_data_smithi027_271924_object31903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:25.315929+0000", 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 285.44258913900001, 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4513220809999998, 2024-08-29T15:36:11.016 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148244 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315929+0000", 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.017 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315929+0000", 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315927+0000", 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315934+0000", 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.46e-06 2024-08-29T15:36:11.018 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315947+0000", 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3696999999999999e-05 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.315956+0000", 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7849999999999997e-06 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.316004+0000", 2024-08-29T15:36:11.019 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7997999999999997e-05 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:25.316013+0000", 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5129999999999996e-06 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:26.185362+0000", 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86934909800000004 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:26.185371+0000", 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2089999999999994e-06 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.708601+0000", 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5232299839999999 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.708754+0000", 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015327 2024-08-29T15:36:11.021 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.709440+0000", 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000685437 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.736751+0000", 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027311275999999999 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.741158+0000", 2024-08-29T15:36:11.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044072649999999996 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763538+0000", 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022380015 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763587+0000", 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.833e-05 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.023 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.767251+0000", 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036641159999999998 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148261 7.c 7:3a32fcb3:::benchmark_data_smithi027_271924_object31920:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:27.521671+0000", 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 283.23684734199998, 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26602938700000001, 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.024 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148261 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521671+0000", 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521671+0000", 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.025 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521668+0000", 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521677+0000", 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4840000000000004e-06 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.026 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521692+0000", 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5509e-05 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521700+0000", 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3710000000000002e-06 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521762+0000", 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2642000000000001e-05 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:36:11.027 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.521775+0000", 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2765999999999999e-05 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.728975+0000", 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.20719976800000001 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.729117+0000", 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014175900000000001 2024-08-29T15:36:11.028 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.729268+0000", 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015089199999999999 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.741109+0000", 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011841841000000001 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748263+0000", 2024-08-29T15:36:11.029 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0071532009999999997 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.783481+0000", 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035218226999999998 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.783537+0000", 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5933999999999997e-05 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.030 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.787700+0000", 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041630599999999997 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148272 7.c 7:3b74bfd4:::benchmark_data_smithi027_271924_object31931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:27.620587+0000", 2024-08-29T15:36:11.031 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 283.13793093700002, 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17806313500000001, 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148272 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.032 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620587+0000", 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620587+0000", 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620586+0000", 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.033 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620592+0000", 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8549999999999996e-06 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620606+0000", 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4053000000000001e-05 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620612+0000", 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7490000000000004e-06 2024-08-29T15:36:11.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620656+0000", 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3389000000000001e-05 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.620664+0000", 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7320000000000004e-06 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.733242+0000", 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11257816599999999 2024-08-29T15:36:11.035 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.733397+0000", 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015458400000000001 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.733523+0000", 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012611600000000001 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.036 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.745008+0000", 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011484938 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748316+0000", 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033082559999999999 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.794883+0000", 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046566159000000003 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.037 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.794937+0000", 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4434000000000001e-05 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.798650+0000", 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037130890000000001 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:148280 7.c 7:36520c72:::benchmark_data_smithi027_271924_object31939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:27.658741+0000", 2024-08-29T15:36:11.038 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 283.09977650399998, 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15069249100000001, 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 148280 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658741+0000", 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658741+0000", 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658740+0000", 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.658746+0000", 2024-08-29T15:36:11.040 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3890000000000002e-06 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.665801+0000", 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070547589999999999 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.665817+0000", 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6188000000000002e-05 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748347+0000", 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.082530021999999995 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748449+0000", 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010179999999999999 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.042 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.043 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:36:11.043 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.748561+0000", 2024-08-29T15:36:11.043 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111557 2024-08-29T15:36:11.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.043 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.043 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.043 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763383+0000", 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014822343 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.763501+0000", 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117484 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.805791+0000", 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042290217999999997 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.805847+0000", 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5822000000000002e-05 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:27.809434+0000", 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035872529999999999 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:151566 7.7 7:e083386b:::benchmark_data_smithi027_271924_object35225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:33.614458+0000", 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 277.14406008899999, 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.14414993700000001, 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 151566 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.046 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614458+0000", 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614458+0000", 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614456+0000", 2024-08-29T15:36:11.047 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614463+0000", 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.224e-06 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614472+0000", 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3440000000000007e-06 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.048 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614477+0000", 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9479999999999999e-06 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614528+0000", 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1282999999999997e-05 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614624+0000", 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5488000000000005e-05 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1,3", 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.614719+0000", 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4926999999999994e-05 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.618050+0000", 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033317609999999999 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.620017+0000", 2024-08-29T15:36:11.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001966551 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.623032+0000", 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030154069999999999 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.758432+0000", 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13539964600000001 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.758490+0000", 2024-08-29T15:36:11.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8343e-05 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:33.758608+0000", 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117333 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:163430 7.7 7:e5fb515b:::benchmark_data_smithi027_271924_object47089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:31:55.523780+0000", 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 255.23473824000001, 2024-08-29T15:36:11.052 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13997130299999999, 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 163430 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.523780+0000", 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.523780+0000", 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.523779+0000", 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.523784+0000", 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.04e-06 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.054 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.523795+0000", 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1158999999999999e-05 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.523801+0000", 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4679999999999997e-06 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.528911+0000", 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051098810000000001 2024-08-29T15:36:11.055 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.529045+0000", 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000133416 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1,3", 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.529187+0000", 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000141958 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.536087+0000", 2024-08-29T15:36:11.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0069001790000000002 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.540287+0000", 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041999649999999999 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.555438+0000", 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015151324000000001 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.663581+0000", 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10814297 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.663633+0000", 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2269999999999999e-05 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:31:55.663751+0000", 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117544 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.058 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:166121 7.d 7:b3d4592d:::benchmark_data_smithi027_271924_object49780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:01.167526+0000", 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 249.59099179699999, 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.20043962900000001, 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 166121 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.059 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167526+0000", 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167526+0000", 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167525+0000", 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167531+0000", 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9789999999999998e-06 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167544+0000", 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2784e-05 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.061 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167550+0000", 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.917e-06 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167596+0000", 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5680999999999999e-05 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167687+0000", 2024-08-29T15:36:11.062 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0759000000000003e-05 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.167811+0000", 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012439100000000001 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.174773+0000", 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006961592 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.063 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.176900+0000", 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002126771 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.367905+0000", 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19100555999999999 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.367942+0000", 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6532000000000002e-05 2024-08-29T15:36:11.064 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.367966+0000", 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3966999999999999e-05 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:166123 7.d 7:bfba9792:::benchmark_data_smithi027_271924_object49782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:01.170018+0000", 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 249.58849938899999, 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19802821000000001, 2024-08-29T15:36:11.065 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 166123 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.170018+0000", 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.170018+0000", 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.170018+0000", 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.170022+0000", 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9299999999999996e-06 2024-08-29T15:36:11.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.170028+0000", 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1700000000000002e-06 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.170032+0000", 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1989999999999999e-06 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.171609+0000", 2024-08-29T15:36:11.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015775030000000001 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.171701+0000", 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1379999999999996e-05 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,1", 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.171781+0000", 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0668999999999996e-05 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:11.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.174917+0000", 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031360649999999999 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.177040+0000", 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021223119999999999 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.367997+0000", 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.19095740899999999 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.368026+0000", 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8670000000000002e-05 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:01.368047+0000", 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0780000000000001e-05 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:11.071 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:36:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:11.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:11.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:11.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:11.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:11.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:11.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:11.344 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:11.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:11.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:11.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:11.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:11.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:11.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:11.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:11.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:11.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 69 16 106958 106942 96.8499 60.9375 0.015794 0.0103203 2024-08-29T15:36:11.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:11.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:11.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:11.882 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:11.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:11.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:11.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:11.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:11.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:11.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[93921]: pgmap v1307: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:36:11.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:36:11.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:36:11.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:11.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:11.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:11.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[98970]: pgmap v1307: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:36:11.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:36:11.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:36:11.991 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:11 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:12.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:11 smithi171 ceph-mon[90031]: pgmap v1307: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-29T15:36:12.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:36:12.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:36:12.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:11 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:12.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:12.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:12.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:12.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:12.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:36:12.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:12.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:12.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:12.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:12.364 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:12.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:12.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:12.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:12.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:12.535 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:12.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:12.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:12.640 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:12.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:12.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:12.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 70 16 107949 107933 96.3511 61.9375 0.0161471 0.0103735 2024-08-29T15:36:12.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:12.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:12.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:12.988 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:12.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:13.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:13.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:13.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:13.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:13.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:13.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:13.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:13.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:13.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:13.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:13.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:13.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:13.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:13.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:13.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:13.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:13.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 71 16 108963 108947 95.8866 63.375 0.0203903 0.010424 2024-08-29T15:36:13.799 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:13.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:13.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:13.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:13.938 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:36:13.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:13.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:13 smithi171 ceph-mon[90031]: pgmap v1308: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3500424946' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:14.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[93921]: pgmap v1308: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3500424946' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[98970]: pgmap v1308: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:36:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3500424946' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:36:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:14.083 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:14.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:14.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:14.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:14.264 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:14.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:14.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:14.366 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:14.399 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:14.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:14.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:14.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:14.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:14.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:14.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:14.596 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:36:14.596 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:36:14.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.596 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:109781 8.5 e354/344)", 2024-08-29T15:36:14.596 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:36:14.587382+0000", 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0068582419999999996, 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006884884, 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "queued for pg", 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.587382+0000", 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.587382+0000", 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.587381+0000", 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.587403+0000", 2024-08-29T15:36:14.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2124e-05 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.587415+0000", 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2174000000000001e-05 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.587422+0000", 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2820000000000002e-06 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.599 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:109792 8.6 e354/344 8:645252d9:::benchmark_data_smithi027_306999_object109791:head v 354'6928, mlcod=354'6925)", 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:36:14.591757+0000", 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0024828649999999999, 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025611050000000002, 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591757+0000", 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:14.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591757+0000", 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591756+0000", 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591806+0000", 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9499999999999997e-05 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591819+0000", 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3591000000000001e-05 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591827+0000", 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1169999999999998e-06 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591924+0000", 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7921e-05 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:14.591985+0000", 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0241999999999997e-05 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 2 2024-08-29T15:36:14.603 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:36:14.640 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:14.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:14.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:14.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 72 16 109951 109935 95.4124 61.75 0.0326681 0.0104762 2024-08-29T15:36:14.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:14.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:14.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:14.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:14.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:14.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:14.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:14.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:15.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:15.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:15.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:15.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:15.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:15.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:15.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:14 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:15.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3500424946' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:15.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:14 smithi171 ceph-mon[90031]: osdmap e354: 8 total, 8 up, 8 in 2024-08-29T15:36:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:14 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3500424946' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:14 smithi027 ceph-mon[98970]: osdmap e354: 8 total, 8 up, 8 in 2024-08-29T15:36:15.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:14 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:15.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3500424946' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:15.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:14 smithi027 ceph-mon[93921]: osdmap e354: 8 total, 8 up, 8 in 2024-08-29T15:36:15.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:15.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:15.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:15.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:15.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:15.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:15.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:36:15.619 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:15.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:15.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:15.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:15.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 73 16 110918 110902 94.9333 60.4375 0.0201364 0.0105284 2024-08-29T15:36:15.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:15.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:15.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:15.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:15.923 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:15.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:15.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:16.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:16.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:16.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:16.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:16.200 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:16.204 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:15 smithi171 ceph-mon[90031]: pgmap v1310: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-29T15:36:16.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:16.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:16.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:16.237 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:36:16.237 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:36:16.237 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-08-29T15:36:16.237 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:36:16.237 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:36:16.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-08-29T15:36:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:15 smithi027 ceph-mon[98970]: pgmap v1310: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-29T15:36:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:15 smithi027 ceph-mon[93921]: pgmap v1310: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-29T15:36:16.332 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:16.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:16.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:16.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:16.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:16.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:16.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:16.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:16.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:16.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:16.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:16.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:16.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:16.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 74 16 111821 111805 94.4131 56.4375 0.014042 0.0105866 2024-08-29T15:36:16.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:16.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:16.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:16.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:16.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:16.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:17.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:17.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:17.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:17.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:17.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:17.185 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:36:17.186 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:36:17.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:17.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:17.225 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:36:17.225 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:36:17.260 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:36:17.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:36:17.293 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:36:17.293 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:36:17.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:17.329 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:36:17.329 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:36:17.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:17.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:17.367 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:36:17.367 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:36:17.396 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:36:17.396 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:36:17.430 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:36:17.430 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:36:17.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:17.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:17.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:17.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:17.635 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:17.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:17.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:17.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 75 16 112805 112789 93.9742 61.5 0.020103 0.0106357 2024-08-29T15:36:17.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:17.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:17.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:17.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:17.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:17.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:17.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:17.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:17 smithi027 ceph-mon[98970]: pgmap v1311: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-29T15:36:17.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3960101949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:17.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:17 smithi027 ceph-mon[93921]: pgmap v1311: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-29T15:36:17.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3960101949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:18.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:18.061 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:17 smithi171 ceph-mon[90031]: pgmap v1311: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-08-29T15:36:18.061 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3960101949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:18.070 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:18.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:18.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:18.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:18.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:18.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:18.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:18.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:18.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:18.474 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:36:18.475 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:36:18.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:18.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:18.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:18.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:18.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:18.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:18.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:18.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:18.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 76 16 113791 113775 93.5485 61.625 0.014403 0.0106845 2024-08-29T15:36:18.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:18.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:18.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:18.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:18.897 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:18.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:18.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:18.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:36:18.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:19.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:19.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:19.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:19.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:19.171 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:19.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:19.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:19.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:19.307 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:19.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:36:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:36:19] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-08-29T15:36:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:19.331 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:19.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:19.431 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:19.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:19.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:19.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:19.612 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:19.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:19.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:19.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:19.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 77 16 114774 114758 93.1315 61.4375 0.0279173 0.010732 2024-08-29T15:36:19.760 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:19.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:19.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:19.862 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:19.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:19.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:19.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:20.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:20.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:20.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:20.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:19 smithi027 ceph-mon[98970]: pgmap v1312: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:36:20.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:19 smithi027 ceph-mon[93921]: pgmap v1312: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:36:20.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:20.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:20.174 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:20.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:20.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:36:20.252 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:19 smithi171 ceph-mon[90031]: pgmap v1312: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-29T15:36:20.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:20.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:36:20.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:20.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:20.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:20.404 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:20.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:20.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:20.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:20.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:20.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:20.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:20.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:20.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:20.728 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:20.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:20.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:20.743 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 78 16 115781 115765 92.7443 62.9375 0.0110542 0.0107769 2024-08-29T15:36:20.830 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:20.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:20.869 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:20.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:20.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:20.986 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:21.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:21.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:21.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:21.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:21.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:21.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:21.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:21.321 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:21.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:21.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.420 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:21.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:21.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:21.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:21.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:21.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:21.665 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:36:21.665 INFO:teuthology.orchestra.run.smithi171.stdout: "success": "" 2024-08-29T15:36:21.665 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:36:21.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:21.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:21.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 79 16 116688 116672 92.2879 56.6875 0.011512 0.0108303 2024-08-29T15:36:21.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:21.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:21.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:21.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:21.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:21.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:22.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:22.109 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:22.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:22.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:22.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:21 smithi027 ceph-mon[98970]: pgmap v1313: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-29T15:36:22.226 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:21 smithi027 ceph-mon[93921]: pgmap v1313: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-29T15:36:22.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:22.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:22.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:22.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:21 smithi171 ceph-mon[90031]: pgmap v1313: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-29T15:36:22.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:22.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:22.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:22.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:22.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:22.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:22.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:22.678 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:22.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:22.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:36:22.743762+0000 min lat: 0.00117312 max lat: 0.294735 avg lat: 0.0108752 2024-08-29T15:36:22.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:36:22.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 80 16 117682 117666 91.9108 62.125 0.00950134 0.0108752 2024-08-29T15:36:22.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:22.845 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:22.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:22.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:22.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:22.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:22.997 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:36:23.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:23.119 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:23.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:23.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:23.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:23.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:23.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:23.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:23.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3494457929' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:36:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3494457929' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:36:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3494457929' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:36:23.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:23.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:23.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:23.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:23.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:23.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:23.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:23.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:23.664 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:23.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:23.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:23.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 81 16 118693 118677 91.5562 63.1875 0.00890953 0.010917 2024-08-29T15:36:23.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:23.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:23.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:23.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:23.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:23.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:23.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:24.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:24.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:24.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:24.117 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:36:24.118 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:36:24.118 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[98970]: pgmap v1314: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2024-08-29T15:36:24.118 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:24.118 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3494457929' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:36:24.118 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[98970]: osdmap e355: 8 total, 8 up, 8 in 2024-08-29T15:36:24.118 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[93921]: pgmap v1314: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2024-08-29T15:36:24.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:24.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3494457929' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:36:24.119 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:23 smithi027 ceph-mon[93921]: osdmap e355: 8 total, 8 up, 8 in 2024-08-29T15:36:24.144 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:36:24.144 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:36:24.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:24.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:36:24.178 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:36:24.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:24.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:24.222 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:36:24.222 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:36:24.260 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:36:24.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:36:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:23 smithi171 ceph-mon[90031]: pgmap v1314: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2024-08-29T15:36:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:23 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3494457929' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:36:24.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:23 smithi171 ceph-mon[90031]: osdmap e355: 8 total, 8 up, 8 in 2024-08-29T15:36:24.294 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:36:24.294 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:36:24.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:24.338 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:36:24.338 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:36:24.349 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:24.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:24.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:24.375 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:36:24.375 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:36:24.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:24.490 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:24.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:24.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:24.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:24.630 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:24.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:24.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:24.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 82 15 119621 119606 91.1475 58.0625 0.027972 0.0109663 2024-08-29T15:36:24.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:24.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:24.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:24.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:24.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:24.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:25.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:25.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:25.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:25.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:25.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/982621312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:25.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:25.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:25.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:25.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/982621312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:25.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/982621312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:25.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:25.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:25.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:25.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:25.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:25.724 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:25.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 83 16 120619 120603 90.8001 62.3125 0.0088149 0.0110081 2024-08-29T15:36:25.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:25.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:36:25.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:25.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:25.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:25.986 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:25.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:26.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:26.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:26.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:26.201 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:26.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:26.270 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:26 smithi171 ceph-mon[90031]: pgmap v1316: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-29T15:36:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:26.276 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:26.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:26.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:26 smithi027 ceph-mon[98970]: pgmap v1316: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-29T15:36:26.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:26 smithi027 ceph-mon[93921]: pgmap v1316: 89 pgs: 89 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-29T15:36:26.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:26.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:26.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:26.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:26.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:26.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:26.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:26.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:26.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:26.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:26.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:26.744 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 84 16 121480 121464 90.3597 53.8125 0.00439985 0.0110547 2024-08-29T15:36:26.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:26.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:26.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:26.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:26.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:26.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:26.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:27.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:36:27.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:27.166 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:27.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:27.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:27.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:27.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:27.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:181363 7.c 7:374819ae:::benchmark_data_smithi027_271924_object65022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:31.121438+0000", 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.462115686, 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12429384, 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.317 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 181363 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121438+0000", 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121438+0000", 2024-08-29T15:36:27.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121437+0000", 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121444+0000", 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.613e-06 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121455+0000", 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1379999999999999e-05 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121460+0000", 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1959999999999999e-06 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.127088+0000", 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056279779999999996 2024-08-29T15:36:27.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.127714+0000", 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00062543899999999999 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.196681+0000", 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.068967428999999997 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.196801+0000", 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012019 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.196954+0000", 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000152941 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.209476+0000", 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012521947 2024-08-29T15:36:27.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.209485+0000", 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0669999999999998e-06 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.243529+0000", 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034044267000000003 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.243589+0000", 2024-08-29T15:36:27.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9919e-05 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.245732+0000", 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021428319999999999 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:183981 7.c 7:37b24d37:::benchmark_data_smithi027_271924_object67640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:35.790310+0000", 2024-08-29T15:36:27.324 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 230.793243636, 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.142854971, 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 183981 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790310+0000", 2024-08-29T15:36:27.325 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790310+0000", 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790309+0000", 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790316+0000", 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1199999999999996e-06 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790331+0000", 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.449e-05 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790338+0000", 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4070000000000004e-06 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.807843+0000", 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01750521 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.807891+0000", 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7818999999999997e-05 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.854100+0000", 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046209159 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.854159+0000", 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9163999999999998e-05 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.854266+0000", 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106943 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.864760+0000", 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010493593000000001 2024-08-29T15:36:27.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.867054+0000", 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022938609999999999 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.931263+0000", 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.064209242 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.931297+0000", 2024-08-29T15:36:27.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3673000000000001e-05 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.933165+0000", 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001868682 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:185906 7.c 7:392ad6ee:::benchmark_data_smithi027_271924_object69565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.000130+0000", 2024-08-29T15:36:27.331 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 227.58342425500001, 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12084690200000001, 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 185906 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000130+0000", 2024-08-29T15:36:27.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000130+0000", 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000128+0000", 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000135+0000", 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5490000000000003e-06 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000148+0000", 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3553999999999999e-05 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000155+0000", 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8970000000000002e-06 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.007287+0000", 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0071321889999999997 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.007830+0000", 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00054281100000000003 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.062993+0000", 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.055163104999999997 2024-08-29T15:36:27.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.063115+0000", 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000121966 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.063280+0000", 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000164699 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.079375+0000", 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016094799999999999 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.091591+0000", 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012216401999999999 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.118746+0000", 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027154729999999998 2024-08-29T15:36:27.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.118845+0000", 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9225e-05 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.120977+0000", 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002131473 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:186004 7.c 7:3c022e11:::benchmark_data_smithi027_271924_object69663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.201273+0000", 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 227.38228142099999, 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13242031000000001, 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 186004 2024-08-29T15:36:27.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201273+0000", 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201273+0000", 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201271+0000", 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201278+0000", 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1969999999999999e-06 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201293+0000", 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4674e-05 2024-08-29T15:36:27.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201300+0000", 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8210000000000004e-06 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.210716+0000", 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0094159840000000005 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.210774+0000", 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8548999999999998e-05 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.291659+0000", 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080884510000000007 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.291736+0000", 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7298000000000004e-05 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.291830+0000", 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4399000000000002e-05 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.299628+0000", 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00779745 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.299638+0000", 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0093e-05 2024-08-29T15:36:27.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.331393+0000", 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031755115 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.331453+0000", 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9846999999999998e-05 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.333693+0000", 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022401209999999999 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:186059 7.c 7:3f2c4a83:::benchmark_data_smithi027_271924_object69718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.308963+0000", 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 227.27459119100001, 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.120818908, 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.346 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 186059 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308963+0000", 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308963+0000", 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308962+0000", 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308965+0000", 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0070000000000002e-06 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308972+0000", 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9040000000000003e-06 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308977+0000", 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0950000000000003e-06 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.314312+0000", 2024-08-29T15:36:27.349 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0053341250000000003 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.314833+0000", 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00052143199999999995 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.357405+0000", 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042572461999999998 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.357472+0000", 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6929999999999998e-05 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.357559+0000", 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6762999999999996e-05 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363038+0000", 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0054784350000000002 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363810+0000", 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00077244800000000001 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.427824+0000", 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.064014038999999995 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.427901+0000", 2024-08-29T15:36:27.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7435999999999995e-05 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.429782+0000", 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001880271 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:186072 7.c 7:3e3c8b97:::benchmark_data_smithi027_271924_object69731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.337065+0000", 2024-08-29T15:36:27.353 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 227.246489209, 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12659251599999999, 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 186072 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337065+0000", 2024-08-29T15:36:27.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337065+0000", 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337064+0000", 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337067+0000", 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3470000000000001e-06 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337072+0000", 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.797e-06 2024-08-29T15:36:27.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337076+0000", 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9319999999999998e-06 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363071+0000", 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025995829000000002 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363792+0000", 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000720238 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.435485+0000", 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.071693713000000006 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.435577+0000", 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2023999999999998e-05 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.435713+0000", 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000135227 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.447400+0000", 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011686999999999999 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.461301+0000", 2024-08-29T15:36:27.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013901939 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.461311+0000", 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2390000000000008e-06 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.461395+0000", 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3772999999999999e-05 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.463657+0000", 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002262834 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193430 7.c 7:3582ae89:::benchmark_data_smithi027_271924_object77089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.570981+0000", 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 214.012573229, 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12667557800000001, 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.361 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193430 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570981+0000", 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570981+0000", 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570979+0000", 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570986+0000", 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2980000000000003e-06 2024-08-29T15:36:27.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.571004+0000", 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7306e-05 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.571012+0000", 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4449999999999996e-06 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.576859+0000", 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0058465210000000004 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.577402+0000", 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00054362699999999998 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.630804+0000", 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.053401692000000001 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.630900+0000", 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5832e-05 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.631043+0000", 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014345800000000001 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.642639+0000", 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011596245999999999 2024-08-29T15:36:27.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.642694+0000", 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4666999999999997e-05 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.695939+0000", 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.053244646 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.695995+0000", 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6042999999999999e-05 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.697656+0000", 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016616160000000001 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193477 7.c 7:3b7390b6:::benchmark_data_smithi027_271924_object77136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.654544+0000", 2024-08-29T15:36:27.368 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 213.929009976, 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.121148282, 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193477 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654544+0000", 2024-08-29T15:36:27.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654544+0000", 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654543+0000", 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654548+0000", 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7149999999999997e-06 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654558+0000", 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6169999999999996e-06 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654563+0000", 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0019999999999997e-06 2024-08-29T15:36:27.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654596+0000", 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3031000000000003e-05 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.655082+0000", 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00048625900000000001 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.707741+0000", 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.052658917999999999 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.707808+0000", 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7167000000000003e-05 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.707901+0000", 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2989e-05 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720410+0000", 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012508594 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.738610+0000", 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018200213 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.773726+0000", 2024-08-29T15:36:27.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035116742999999999 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.773782+0000", 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5612000000000002e-05 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775692+0000", 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001910246 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.375 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193483 7.c 7:32868c1b:::benchmark_data_smithi027_271924_object77142:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.656968+0000", 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 213.926586087, 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.122521351, 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193483 2024-08-29T15:36:27.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656968+0000", 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656968+0000", 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656967+0000", 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656972+0000", 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.417e-06 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656985+0000", 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3152999999999999e-05 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656994+0000", 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.867e-06 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.657562+0000", 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000567302 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.658150+0000", 2024-08-29T15:36:27.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00058876200000000005 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720458+0000", 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062307686000000001 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720562+0000", 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000104076 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720715+0000", 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015248700000000001 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.732477+0000", 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011761984 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.738685+0000", 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0062084130000000003 2024-08-29T15:36:27.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.777488+0000", 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038803294000000002 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.777537+0000", 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8822000000000001e-05 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.779489+0000", 2024-08-29T15:36:27.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00195211 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193484 7.c 7:3eb7034a:::benchmark_data_smithi027_271924_object77143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.659073+0000", 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 213.924480735, 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12237255900000001, 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.383 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193484 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659073+0000", 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659073+0000", 2024-08-29T15:36:27.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659072+0000", 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659081+0000", 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4440000000000008e-06 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659094+0000", 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3171e-05 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659101+0000", 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3669999999999999e-06 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659152+0000", 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0470999999999998e-05 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659730+0000", 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00057846700000000004 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.726851+0000", 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067120432999999993 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.726917+0000", 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6421999999999995e-05 2024-08-29T15:36:27.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.727028+0000", 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111351 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.737340+0000", 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010311694 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.738725+0000", 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001384758 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.779539+0000", 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040814069000000001 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.779586+0000", 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6828e-05 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.781446+0000", 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018601900000000001 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193536 7.c 7:3cbbe8ac:::benchmark_data_smithi027_271924_object77195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.775463+0000", 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 213.808091515, 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13050738200000001, 2024-08-29T15:36:27.390 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193536 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775463+0000", 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775463+0000", 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775461+0000", 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775467+0000", 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7910000000000001e-06 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775478+0000", 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1139e-05 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775484+0000", 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.127e-06 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.796483+0000", 2024-08-29T15:36:27.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020999522999999999 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.797785+0000", 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013018859999999999 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.837846+0000", 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040060428000000002 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.837925+0000", 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9503000000000001e-05 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.838042+0000", 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011702299999999999 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.848706+0000", 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010664237 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.864670+0000", 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015963537999999999 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.904123+0000", 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039452698000000001 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.904159+0000", 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6752999999999999e-05 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.905970+0000", 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018105020000000001 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193545 7.c 7:3ea5e9f6:::benchmark_data_smithi027_271924_object77204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.783647+0000", 2024-08-29T15:36:27.397 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 213.79990680500001, 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.124103535, 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193545 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783647+0000", 2024-08-29T15:36:27.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783647+0000", 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783646+0000", 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:27.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783654+0000", 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.055e-06 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783669+0000", 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5077e-05 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783676+0000", 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6040000000000004e-06 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.797824+0000", 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014147730000000001 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.802988+0000", 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005164302 2024-08-29T15:36:27.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.845319+0000", 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042330378000000002 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.845395+0000", 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6686000000000004e-05 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.845534+0000", 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013818500000000001 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.864676+0000", 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019142306000000001 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.864727+0000", 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.126e-05 2024-08-29T15:36:27.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.906028+0000", 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041300986999999997 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.906081+0000", 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2518000000000001e-05 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.907751+0000", 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016700230000000001 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:206192 7.c 7:3fe59ff6:::benchmark_data_smithi027_271924_object89851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:13.748591+0000", 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 192.83496305899999, 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12247730699999999, 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.405 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 206192 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748591+0000", 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748591+0000", 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748589+0000", 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748597+0000", 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1589999999999993e-06 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748611+0000", 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4183e-05 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748628+0000", 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6101999999999999e-05 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748657+0000", 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9575999999999999e-05 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748720+0000", 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2524e-05 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.839050+0000", 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090330220000000003 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.839156+0000", 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106018 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.839317+0000", 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016095700000000001 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.862968+0000", 2024-08-29T15:36:27.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023650786 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.862980+0000", 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2702e-05 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.869517+0000", 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006536549 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.869557+0000", 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9705e-05 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.871068+0000", 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001511667 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.412 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209048 7.c 7:3c105219:::benchmark_data_smithi027_271924_object92707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.323326+0000", 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 188.260228117, 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.151098872, 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209048 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323326+0000", 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323326+0000", 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323325+0000", 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323329+0000", 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6449999999999998e-06 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323337+0000", 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3329999999999997e-06 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323343+0000", 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.733e-06 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.328539+0000", 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051963679999999998 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.328600+0000", 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1122000000000002e-05 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.357721+0000", 2024-08-29T15:36:27.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029121030999999999 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.357828+0000", 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010610799999999999 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.357977+0000", 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014897700000000001 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.368487+0000", 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010509977 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.368496+0000", 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6970000000000005e-06 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.472833+0000", 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.104336963 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.472883+0000", 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9663999999999997e-05 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.474425+0000", 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015419279999999999 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.419 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209068 7.c 7:30640b78:::benchmark_data_smithi027_271924_object92727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.344896+0000", 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 188.23865792300001, 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.130916859, 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209068 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344896+0000", 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344896+0000", 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344895+0000", 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344900+0000", 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2320000000000001e-06 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344914+0000", 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4229e-05 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344921+0000", 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7499999999999997e-06 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.345970+0000", 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0010485729999999999 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.346631+0000", 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00066114899999999998 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.364287+0000", 2024-08-29T15:36:27.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017656505999999999 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.364390+0000", 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010242399999999999 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.364535+0000", 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0001449 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.374351+0000", 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0098159440000000001 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.374372+0000", 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1821e-05 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.474493+0000", 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100120244 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.474548+0000", 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4885000000000001e-05 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.475813+0000", 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001265399 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209071 7.c 7:3a2a109c:::benchmark_data_smithi027_271924_object92730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.426 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.351037+0000", 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 188.23251747500001, 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15185913200000001, 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209071 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351037+0000", 2024-08-29T15:36:27.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351037+0000", 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351034+0000", 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351041+0000", 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0230000000000004e-06 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351057+0000", 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5682999999999999e-05 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351065+0000", 2024-08-29T15:36:27.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7800000000000001e-06 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351216+0000", 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015084399999999999 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.355370+0000", 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004154189 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.462040+0000", 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10666985399999999 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.462107+0000", 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7359000000000003e-05 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.462197+0000", 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9567000000000005e-05 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.475879+0000", 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013681858 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.475890+0000", 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1669e-05 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501444+0000", 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025554132 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501518+0000", 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4066999999999994e-05 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.502896+0000", 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001377234 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.433 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209088 7.c 7:30b45f45:::benchmark_data_smithi027_271924_object92747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.407525+0000", 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 188.176029019, 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17467528700000001, 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209088 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407525+0000", 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407525+0000", 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407523+0000", 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407532+0000", 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8449999999999992e-06 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407551+0000", 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9578000000000001e-05 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407560+0000", 2024-08-29T15:36:27.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3950000000000001e-06 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407619+0000", 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8794e-05 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407695+0000", 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5962000000000002e-05 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.507558+0000", 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.099863482000000003 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.507663+0000", 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010514999999999999 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.507823+0000", 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015936600000000001 2024-08-29T15:36:27.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.518533+0000", 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010709984000000001 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.518596+0000", 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3349999999999995e-05 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.580676+0000", 2024-08-29T15:36:27.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062079955999999999 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.580732+0000", 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6558000000000003e-05 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.582200+0000", 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014678110000000001 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209145 7.c 7:3ba8fe40:::benchmark_data_smithi027_271924_object92804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.501713+0000", 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 188.08184129700001, 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12807848699999999, 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209145 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501713+0000", 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501713+0000", 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501711+0000", 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501718+0000", 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0380000000000002e-06 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501730+0000", 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2211999999999999e-05 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501736+0000", 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9660000000000001e-06 2024-08-29T15:36:27.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.504637+0000", 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002900812 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.505279+0000", 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00064199000000000005 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.576672+0000", 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.071392707 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.576750+0000", 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8045000000000002e-05 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.576841+0000", 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1044000000000004e-05 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.589018+0000", 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012176910000000001 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.589075+0000", 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7438999999999997e-05 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.628573+0000", 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039498097000000003 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.628615+0000", 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1423e-05 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.629791+0000", 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0011766649999999999 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209157 7.c 7:36d544a5:::benchmark_data_smithi027_271924_object92816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.517313+0000", 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 188.06624116399999, 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12589757600000001, 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209157 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517313+0000", 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517313+0000", 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517311+0000", 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:36:27.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517318+0000", 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6669999999999999e-06 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517335+0000", 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7294000000000001e-05 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517343+0000", 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9719999999999992e-06 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.518677+0000", 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013335179999999999 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.519253+0000", 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00057647500000000001 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.584808+0000", 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.065554772999999997 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.584931+0000", 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000123538 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.585071+0000", 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000139469 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.595031+0000", 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.009960666 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.595041+0000", 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.431e-06 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.642176+0000", 2024-08-29T15:36:27.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.047135336 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.642206+0000", 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9872000000000001e-05 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.643210+0000", 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00100435 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.454 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:212171 7.c 7:338e7bd7:::benchmark_data_smithi027_271924_object95830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:23.410992+0000", 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 183.17256252300001, 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12099120100000001, 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:36:27.455 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 212171 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410992+0000", 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410992+0000", 2024-08-29T15:36:27.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410990+0000", 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410996+0000", 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3119999999999999e-06 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.411008+0000", 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1187e-05 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.411013+0000", 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8420000000000004e-06 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.419818+0000", 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0088056160000000005 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.419915+0000", 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7058000000000005e-05 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.420032+0000", 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117299 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.426619+0000", 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0065867820000000002 2024-08-29T15:36:27.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.427738+0000", 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001119117 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.530575+0000", 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.102836512 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.530628+0000", 2024-08-29T15:36:27.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2670000000000002e-05 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.531983+0000", 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001355141 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:27.461 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:36:27.462 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:27.469 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:36:27.469 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:36:27.470 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-08-29T15:36:27.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:36:27.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:27.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:27.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:27.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:27.632 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:27.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:27.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:27.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:27.749 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 85 16 122393 122377 89.9626 57.0625 0.0376721 0.011111 2024-08-29T15:36:27.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:27.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:27.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:27.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:27.902 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:27.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:28.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:28.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:28.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:28 smithi027 ceph-mon[98970]: pgmap v1317: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-29T15:36:28.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:28 smithi027 ceph-mon[93921]: pgmap v1317: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-29T15:36:28.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:28.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:28.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:28.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:28.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:28 smithi171 ceph-mon[90031]: pgmap v1317: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-29T15:36:28.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:28.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:28.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:28.427 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:28.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:36:28.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:28.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:28.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:28.560 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:28.608 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:28.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:28.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:28.749 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 86 16 123312 123296 89.5844 57.4375 0.0246154 0.0111573 2024-08-29T15:36:28.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:28.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:28.775 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:28.789 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:28.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:28.921 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:28.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:29.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:29.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:29.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:29.132 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:29.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:29.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:29.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:29.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:29.277 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:29.277 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:29.277 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:36:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:36:29] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-08-29T15:36:29.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:29.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:29.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:29.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:29.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:29.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:29.595 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:29.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:29.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:29.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:29.749 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 87 16 124143 124127 89.1517 51.9375 0.0093976 0.0112111 2024-08-29T15:36:29.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:29.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:29.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:29.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:29.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:30.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:30.046 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:36:30.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:30 smithi027 ceph-mon[98970]: pgmap v1318: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-08-29T15:36:30.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1702114296' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:30.047 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:30.047 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:30 smithi027 ceph-mon[93921]: pgmap v1318: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-08-29T15:36:30.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1702114296' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:30.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:30.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:30.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:30.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:30.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:30.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:30.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:30.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:30 smithi171 ceph-mon[90031]: pgmap v1318: 89 pgs: 89 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-08-29T15:36:30.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1702114296' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:30.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:30.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:30.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:30.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:30.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:30.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:30.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:30.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:30.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:30.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:30.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:30.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:30.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:30.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:30.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 88 16 125016 125000 88.7585 54.5625 0.0182338 0.0112615 2024-08-29T15:36:30.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:30.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:30.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:30.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:30.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:31.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:31.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:31.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:31.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:31.156 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:31.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:31.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:31.256 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:31 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:36:31.256 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:31 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:36:31.256 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:31.257 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:31 smithi171 ceph-mon[90031]: osdmap e356: 8 total, 8 up, 8 in 2024-08-29T15:36:31.297 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:31.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:36:31.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:36:31.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:31.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[93921]: osdmap e356: 8 total, 8 up, 8 in 2024-08-29T15:36:31.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:36:31.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:36:31.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:31.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:31 smithi027 ceph-mon[98970]: osdmap e356: 8 total, 8 up, 8 in 2024-08-29T15:36:31.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:31.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:31.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:31.421 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:36:31.421 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:125606 8.2 e356/344 8:43316568:::benchmark_data_smithi027_306999_object125605:head v 356'7948, mlcod=356'7946)", 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:36:31.415785+0000", 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0028050779999999999, 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028346220000000002, 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415785+0000", 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:31.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415785+0000", 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415784+0000", 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415806+0000", 2024-08-29T15:36:31.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1647000000000001e-05 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415817+0000", 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0651e-05 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415822+0000", 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0629999999999996e-06 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415870+0000", 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8878000000000002e-05 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415889+0000", 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8179e-05 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:125601 8.0 e356/344 8:030d2332:::benchmark_data_smithi027_306999_object125600:head v 356'7876, mlcod=356'7873)", 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:36:31.415912+0000", 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0026784780000000002, 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027697899999999998, 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415912+0000", 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415912+0000", 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415911+0000", 2024-08-29T15:36:31.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.415999+0000", 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7689999999999996e-05 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416009+0000", 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0044000000000001e-05 2024-08-29T15:36:31.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416014+0000", 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1290000000000004e-06 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416050+0000", 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5664999999999998e-05 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416063+0000", 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3434e-05 2024-08-29T15:36:31.429 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:125607 8.c e356/344 8:38dbdcef:::benchmark_data_smithi027_306999_object125606:head v 356'8008, mlcod=356'8006)", 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:36:31.416635+0000", 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0019546849999999998, 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021070849999999999, 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416635+0000", 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416635+0000", 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416635+0000", 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416746+0000", 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011154000000000001 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416764+0000", 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7419999999999999e-05 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416772+0000", 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0600000000000008e-06 2024-08-29T15:36:31.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416825+0000", 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2673999999999997e-05 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:36:31.416844+0000", 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9610999999999999e-05 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:36:31.433 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:36:31.434 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 3 2024-08-29T15:36:31.434 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:36:31.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:31.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:31.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:31.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:31.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:31.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:31.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:31.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:31.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 89 16 125939 125923 88.4093 57.6875 0.0143521 0.0113061 2024-08-29T15:36:31.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:31.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:31.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:31.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:31.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:31.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:31.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:32.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:32.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:32.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:32.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:32.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:32.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:32.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:32.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:36:32.298 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:32 smithi171 ceph-mon[90031]: pgmap v1320: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-08-29T15:36:32.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:32.308 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:32 smithi027 ceph-mon[98970]: pgmap v1320: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-08-29T15:36:32.308 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:32 smithi027 ceph-mon[93921]: pgmap v1320: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2024-08-29T15:36:32.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:32.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:32.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:32.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:32.487 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:32.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:32.581 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:32.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:32.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:32.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:32.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:32.748 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:32.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:32.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total time run: 90.0121 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total writes made: 126776 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Write size: 65536 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object size: 65536 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Bandwidth (MB/sec): 88.027 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Bandwidth: 40.0247 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max bandwidth (MB/sec): 241.938 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min bandwidth (MB/sec): 39.5625 2024-08-29T15:36:32.750 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average IOPS: 1408 2024-08-29T15:36:32.751 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev IOPS: 640.395 2024-08-29T15:36:32.751 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max IOPS: 3871 2024-08-29T15:36:32.751 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min IOPS: 633 2024-08-29T15:36:32.751 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average Latency(s): 0.0113556 2024-08-29T15:36:32.751 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Latency(s): 0.0112274 2024-08-29T15:36:32.751 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max latency(s): 0.294735 2024-08-29T15:36:32.751 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min latency(s): 0.00117312 2024-08-29T15:36:32.755 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Cleaning up (deleting benchmark objects) 2024-08-29T15:36:32.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:32.884 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:32.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:32.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:33.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:33.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:33.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:33.162 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:33.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:36:33.175 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:36:33.176 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:36:33.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:33.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:33.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:33.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2387148470' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:33.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2387148470' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:33.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:33.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2387148470' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:33.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:33.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:33.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.722 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:33.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:33.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:33.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:33.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:33.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:33.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:34.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:34.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:34.103 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:34.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:34.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:34.242 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:34.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:34.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:34.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:34 smithi027 ceph-mon[93921]: pgmap v1321: 89 pgs: 89 active+clean; 7.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-08-29T15:36:34.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:34 smithi027 ceph-mon[98970]: pgmap v1321: 89 pgs: 89 active+clean; 7.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-08-29T15:36:34.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:34.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:34.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:34.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:34.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:34.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:34.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:34.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:34 smithi171 ceph-mon[90031]: pgmap v1321: 89 pgs: 89 active+clean; 7.6 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-08-29T15:36:34.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:34.641 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:34.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:34.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:34.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:34.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:34.878 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:34.917 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:34.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:34.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:35.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:35.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:35.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:35.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:35.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:35.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:35.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:35.242 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:36:35.242 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:36:35.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:35.329 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:35.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:35.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:35.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:35.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:35.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:35.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:35.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:35.698 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:35.728 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:35.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:35.830 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:35.861 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:35.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:35.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:35.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:35.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:36:36.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:36.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:36.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:36.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:36.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:36.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:36.216 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:36 smithi027 ceph-mon[93921]: pgmap v1322: 89 pgs: 89 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 921 op/s 2024-08-29T15:36:36.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:36 smithi027 ceph-mon[98970]: pgmap v1322: 89 pgs: 89 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 921 op/s 2024-08-29T15:36:36.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:36.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:36 smithi171 ceph-mon[90031]: pgmap v1322: 89 pgs: 89 active+clean; 7.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 55 MiB/s wr, 921 op/s 2024-08-29T15:36:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:36.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:36.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:36.377 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:36:36.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:36.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:36.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:36.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:36.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:36.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:36.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:36.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:36.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:36.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:36.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:36.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:36.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:36.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:36.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:37.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:37.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:37.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:37.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:37.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:37.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:37.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:37.439 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:37.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:37.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:37.618 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:36:37.619 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:36:37.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:37.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:37.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:37.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:37.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:37.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:37.986 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:37.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:38.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:38.126 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:38.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:38.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:36:38.228 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:38.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:38 smithi171 ceph-mon[90031]: pgmap v1323: 89 pgs: 89 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 896 op/s 2024-08-29T15:36:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:38.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:38.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:38 smithi027 ceph-mon[98970]: pgmap v1323: 89 pgs: 89 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 896 op/s 2024-08-29T15:36:38.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:38 smithi027 ceph-mon[93921]: pgmap v1323: 89 pgs: 89 active+clean; 7.6 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 896 op/s 2024-08-29T15:36:38.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:38.404 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:38.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:38.502 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:36:38.502 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:36:38.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:38.537 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1323,"stamp":"2024-08-29T15:36:36.876886+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8157009917,"num_objects":124422,"num_object_clones":0,"num_object_copies":373266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":124422,"num_whiteouts":0,"num_read":6366,"num_read_kb":13823,"num_write":130283,"num_write_kb":8128706,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":10166863,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123342,"ondisk_log_size":123342,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":28186528,"kb_used_data":24075936,"kb_used_omap":43,"kb_used_meta":4110356,"kb_avail":721774688,"statfs":{"total":767960285184,"available":739097280512,"internally_reserved":0,"allocated":24653758464,"data_stored":24636176294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":4209005036},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":275054616,"num_objects":4198,"num_object_clones":0,"num_object_copies":12594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4198,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8964,"num_write_kb":421121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003273"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":601,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047243+0000","last_change":"2024-08-29T15:18:33.993493+0000","last_active":"2024-08-29T15:36:30.047243+0000","last_peered":"2024-08-29T15:36:30.047243+0000","last_clean":"2024-08-29T15:36:30.047243+0000","last_became_active":"2024-08-29T15:18:33.993228+0000","last_became_peered":"2024-08-29T15:18:33.993228+0000","last_unstale":"2024-08-29T15:36:30.047243+0000","last_undegraded":"2024-08-29T15:36:30.047243+0000","last_fullsized":"2024-08-29T15:36:30.047243+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:16.906311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":147,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.015376+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:36:23.015376+0000","last_peered":"2024-08-29T15:36:23.015376+0000","last_clean":"2024-08-29T15:36:23.015376+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:36:23.015376+0000","last_undegraded":"2024-08-29T15:36:23.015376+0000","last_fullsized":"2024-08-29T15:36:23.015376+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007166+0000","last_change":"2024-08-29T15:17:40.735303+0000","last_active":"2024-08-29T15:36:23.007166+0000","last_peered":"2024-08-29T15:36:23.007166+0000","last_clean":"2024-08-29T15:36:23.007166+0000","last_became_active":"2024-08-29T15:17:40.735036+0000","last_became_peered":"2024-08-29T15:17:40.735036+0000","last_unstale":"2024-08-29T15:36:23.007166+0000","last_undegraded":"2024-08-29T15:36:23.007166+0000","last_fullsized":"2024-08-29T15:36:23.007166+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:17:15.614240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":575,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.058154+0000","last_change":"2024-08-29T15:31:26.199672+0000","last_active":"2024-08-29T15:36:30.058154+0000","last_peered":"2024-08-29T15:36:30.058154+0000","last_clean":"2024-08-29T15:36:30.058154+0000","last_became_active":"2024-08-29T15:31:26.199057+0000","last_became_peered":"2024-08-29T15:31:26.199057+0000","last_unstale":"2024-08-29T15:36:30.058154+0000","last_undegraded":"2024-08-29T15:36:30.058154+0000","last_fullsized":"2024-08-29T15:36:30.058154+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:21:31.006988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":568,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.056457+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:36:30.056457+0000","last_peered":"2024-08-29T15:36:30.056457+0000","last_clean":"2024-08-29T15:36:30.056457+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:36:30.056457+0000","last_undegraded":"2024-08-29T15:36:30.056457+0000","last_fullsized":"2024-08-29T15:36:30.056457+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":140,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007282+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:36:23.007282+0000","last_peered":"2024-08-29T15:36:23.007282+0000","last_clean":"2024-08-29T15:36:23.007282+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:36:23.007282+0000","last_undegraded":"2024-08-29T15:36:23.007282+0000","last_fullsized":"2024-08-29T15:36:23.007282+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":386,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.048267+0000","last_change":"2024-08-29T15:31:26.189733+0000","last_active":"2024-08-29T15:36:30.048267+0000","last_peered":"2024-08-29T15:36:30.048267+0000","last_clean":"2024-08-29T15:36:30.048267+0000","last_became_active":"2024-08-29T15:31:26.189414+0000","last_became_peered":"2024-08-29T15:31:26.189414+0000","last_unstale":"2024-08-29T15:36:30.048267+0000","last_undegraded":"2024-08-29T15:36:30.048267+0000","last_fullsized":"2024-08-29T15:36:30.048267+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:12:46.739048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":200,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.001013+0000","last_change":"2024-08-29T15:32:26.143498+0000","last_active":"2024-08-29T15:36:23.001013+0000","last_peered":"2024-08-29T15:36:23.001013+0000","last_clean":"2024-08-29T15:36:23.001013+0000","last_became_active":"2024-08-29T15:32:26.143154+0000","last_became_peered":"2024-08-29T15:32:26.143154+0000","last_unstale":"2024-08-29T15:36:23.001013+0000","last_undegraded":"2024-08-29T15:36:23.001013+0000","last_fullsized":"2024-08-29T15:36:23.001013+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:44.284293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.4","version":"356'8084","reported_seq":8105,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.662550+0000","last_change":"2024-08-29T15:34:49.663635+0000","last_active":"2024-08-29T15:36:36.662550+0000","last_peered":"2024-08-29T15:36:36.662550+0000","last_clean":"2024-08-29T15:36:36.662550+0000","last_became_active":"2024-08-29T15:34:49.663325+0000","last_became_peered":"2024-08-29T15:34:49.663325+0000","last_unstale":"2024-08-29T15:36:36.662550+0000","last_undegraded":"2024-08-29T15:36:36.662550+0000","last_fullsized":"2024-08-29T15:36:36.662550+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":8084,"log_dups_size":0,"ondisk_log_size":8084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:01:09.917949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":499253248,"num_objects":7618,"num_object_clones":0,"num_object_copies":22854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8084,"num_write_kb":502464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":157,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007570+0000","last_change":"2024-08-29T15:31:26.215108+0000","last_active":"2024-08-29T15:36:23.007570+0000","last_peered":"2024-08-29T15:36:23.007570+0000","last_clean":"2024-08-29T15:36:23.007570+0000","last_became_active":"2024-08-29T15:31:26.197355+0000","last_became_peered":"2024-08-29T15:31:26.197355+0000","last_unstale":"2024-08-29T15:36:23.007570+0000","last_undegraded":"2024-08-29T15:36:23.007570+0000","last_fullsized":"2024-08-29T15:36:23.007570+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:10:37.515092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.000931+0000","last_change":"2024-08-29T15:17:40.720181+0000","last_active":"2024-08-29T15:36:23.000931+0000","last_peered":"2024-08-29T15:36:23.000931+0000","last_clean":"2024-08-29T15:36:23.000931+0000","last_became_active":"2024-08-29T15:17:40.719951+0000","last_became_peered":"2024-08-29T15:17:40.719951+0000","last_unstale":"2024-08-29T15:36:23.000931+0000","last_undegraded":"2024-08-29T15:36:23.000931+0000","last_fullsized":"2024-08-29T15:36:23.000931+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:48:25.059934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"356'8074","reported_seq":8095,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:34.409446+0000","last_change":"2024-08-29T15:34:49.878392+0000","last_active":"2024-08-29T15:36:34.409446+0000","last_peered":"2024-08-29T15:36:34.409446+0000","last_clean":"2024-08-29T15:36:34.409446+0000","last_became_active":"2024-08-29T15:34:49.878081+0000","last_became_peered":"2024-08-29T15:34:49.878081+0000","last_unstale":"2024-08-29T15:36:34.409446+0000","last_undegraded":"2024-08-29T15:36:34.409446+0000","last_fullsized":"2024-08-29T15:36:34.409446+0000","mapping_epoch":344,"log_start":"348'700","ondisk_log_start":"348'700","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:39:43.629402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":516292608,"num_objects":7878,"num_object_clones":0,"num_object_copies":23634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8074,"num_write_kb":510464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":144,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007742+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:36:23.007742+0000","last_peered":"2024-08-29T15:36:23.007742+0000","last_clean":"2024-08-29T15:36:23.007742+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:36:23.007742+0000","last_undegraded":"2024-08-29T15:36:23.007742+0000","last_fullsized":"2024-08-29T15:36:23.007742+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":149,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007076+0000","last_change":"2024-08-29T15:31:26.195631+0000","last_active":"2024-08-29T15:36:23.007076+0000","last_peered":"2024-08-29T15:36:23.007076+0000","last_clean":"2024-08-29T15:36:23.007076+0000","last_became_active":"2024-08-29T15:31:26.195293+0000","last_became_peered":"2024-08-29T15:31:26.195293+0000","last_unstale":"2024-08-29T15:36:23.007076+0000","last_undegraded":"2024-08-29T15:36:23.007076+0000","last_fullsized":"2024-08-29T15:36:23.007076+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:52:28.638575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"356'8107","reported_seq":8143,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:34.872711+0000","last_change":"2024-08-29T15:34:49.877713+0000","last_active":"2024-08-29T15:36:34.872711+0000","last_peered":"2024-08-29T15:36:34.872711+0000","last_clean":"2024-08-29T15:36:34.872711+0000","last_became_active":"2024-08-29T15:34:49.877477+0000","last_became_peered":"2024-08-29T15:34:49.877477+0000","last_unstale":"2024-08-29T15:36:34.872711+0000","last_undegraded":"2024-08-29T15:36:34.872711+0000","last_fullsized":"2024-08-29T15:36:34.872711+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":8107,"log_dups_size":0,"ondisk_log_size":8107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:43:56.536739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":516751360,"num_objects":7885,"num_object_clones":0,"num_object_copies":23655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8107,"num_write_kb":511744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":384,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.054448+0000","last_change":"2024-08-29T15:18:36.387232+0000","last_active":"2024-08-29T15:36:30.054448+0000","last_peered":"2024-08-29T15:36:30.054448+0000","last_clean":"2024-08-29T15:36:30.054448+0000","last_became_active":"2024-08-29T15:18:36.384797+0000","last_became_peered":"2024-08-29T15:18:36.384797+0000","last_unstale":"2024-08-29T15:36:30.054448+0000","last_undegraded":"2024-08-29T15:36:30.054448+0000","last_fullsized":"2024-08-29T15:36:30.054448+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:05:34.159921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":203,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046930+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:36:30.046930+0000","last_peered":"2024-08-29T15:36:30.046930+0000","last_clean":"2024-08-29T15:36:30.046930+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:36:30.046930+0000","last_undegraded":"2024-08-29T15:36:30.046930+0000","last_fullsized":"2024-08-29T15:36:30.046930+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"356'8024","reported_seq":8059,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.341215+0000","last_change":"2024-08-29T15:34:49.403476+0000","last_active":"2024-08-29T15:36:36.341215+0000","last_peered":"2024-08-29T15:36:36.341215+0000","last_clean":"2024-08-29T15:36:36.341215+0000","last_became_active":"2024-08-29T15:34:49.403150+0000","last_became_peered":"2024-08-29T15:34:49.403150+0000","last_unstale":"2024-08-29T15:36:36.341215+0000","last_undegraded":"2024-08-29T15:36:36.341215+0000","last_fullsized":"2024-08-29T15:36:36.341215+0000","mapping_epoch":344,"log_start":"348'1200","ondisk_log_start":"348'1200","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:34:46.231176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":500170752,"num_objects":7632,"num_object_clones":0,"num_object_copies":22896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8024,"num_write_kb":500992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":247,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.003151+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:36:23.003151+0000","last_peered":"2024-08-29T15:36:23.003151+0000","last_clean":"2024-08-29T15:36:23.003151+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:36:23.003151+0000","last_undegraded":"2024-08-29T15:36:23.003151+0000","last_fullsized":"2024-08-29T15:36:23.003151+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":583,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.054113+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:36:30.054113+0000","last_peered":"2024-08-29T15:36:30.054113+0000","last_clean":"2024-08-29T15:36:30.054113+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:36:30.054113+0000","last_undegraded":"2024-08-29T15:36:30.054113+0000","last_fullsized":"2024-08-29T15:36:30.054113+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"356'7990","reported_seq":8026,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:33.650563+0000","last_change":"2024-08-29T15:34:49.400123+0000","last_active":"2024-08-29T15:36:33.650563+0000","last_peered":"2024-08-29T15:36:33.650563+0000","last_clean":"2024-08-29T15:36:33.650563+0000","last_became_active":"2024-08-29T15:34:49.399790+0000","last_became_peered":"2024-08-29T15:34:49.399790+0000","last_unstale":"2024-08-29T15:36:33.650563+0000","last_undegraded":"2024-08-29T15:36:33.650563+0000","last_fullsized":"2024-08-29T15:36:33.650563+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":7990,"log_dups_size":0,"ondisk_log_size":7990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:47.263466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":518127616,"num_objects":7906,"num_object_clones":0,"num_object_copies":23718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7990,"num_write_kb":508672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":587,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.049260+0000","last_change":"2024-08-29T15:31:26.213123+0000","last_active":"2024-08-29T15:36:30.049260+0000","last_peered":"2024-08-29T15:36:30.049260+0000","last_clean":"2024-08-29T15:36:30.049260+0000","last_became_active":"2024-08-29T15:31:26.196584+0000","last_became_peered":"2024-08-29T15:31:26.196584+0000","last_unstale":"2024-08-29T15:36:30.049260+0000","last_undegraded":"2024-08-29T15:36:30.049260+0000","last_fullsized":"2024-08-29T15:36:30.049260+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:51:23.571768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":140,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007561+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:36:23.007561+0000","last_peered":"2024-08-29T15:36:23.007561+0000","last_clean":"2024-08-29T15:36:23.007561+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:36:23.007561+0000","last_undegraded":"2024-08-29T15:36:23.007561+0000","last_fullsized":"2024-08-29T15:36:23.007561+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"356'8120","reported_seq":8156,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.330798+0000","last_change":"2024-08-29T15:34:49.664231+0000","last_active":"2024-08-29T15:36:36.330798+0000","last_peered":"2024-08-29T15:36:36.330798+0000","last_clean":"2024-08-29T15:36:36.330798+0000","last_became_active":"2024-08-29T15:34:49.663889+0000","last_became_peered":"2024-08-29T15:34:49.663889+0000","last_unstale":"2024-08-29T15:36:36.330798+0000","last_undegraded":"2024-08-29T15:36:36.330798+0000","last_fullsized":"2024-08-29T15:36:36.330798+0000","mapping_epoch":344,"log_start":"348'1300","ondisk_log_start":"348'1300","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":6820,"log_dups_size":0,"ondisk_log_size":6820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:04:25.563324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":502923264,"num_objects":7674,"num_object_clones":0,"num_object_copies":23022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8120,"num_write_kb":505408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007187+0000","last_change":"2024-08-29T15:32:26.140568+0000","last_active":"2024-08-29T15:36:23.007187+0000","last_peered":"2024-08-29T15:36:23.007187+0000","last_clean":"2024-08-29T15:36:23.007187+0000","last_became_active":"2024-08-29T15:32:26.140257+0000","last_became_peered":"2024-08-29T15:32:26.140257+0000","last_unstale":"2024-08-29T15:36:23.007187+0000","last_undegraded":"2024-08-29T15:36:23.007187+0000","last_fullsized":"2024-08-29T15:36:23.007187+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:04:33.833909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.009958+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:36:23.009958+0000","last_peered":"2024-08-29T15:36:23.009958+0000","last_clean":"2024-08-29T15:36:23.009958+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:36:23.009958+0000","last_undegraded":"2024-08-29T15:36:23.009958+0000","last_fullsized":"2024-08-29T15:36:23.009958+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"356'8136","reported_seq":8156,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:34.393928+0000","last_change":"2024-08-29T15:34:49.398857+0000","last_active":"2024-08-29T15:36:34.393928+0000","last_peered":"2024-08-29T15:36:34.393928+0000","last_clean":"2024-08-29T15:36:34.393928+0000","last_became_active":"2024-08-29T15:34:49.398263+0000","last_became_peered":"2024-08-29T15:34:49.398263+0000","last_unstale":"2024-08-29T15:36:34.393928+0000","last_undegraded":"2024-08-29T15:36:34.393928+0000","last_fullsized":"2024-08-29T15:36:34.393928+0000","mapping_epoch":344,"log_start":"348'800","ondisk_log_start":"348'800","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":7336,"log_dups_size":0,"ondisk_log_size":7336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:23.024563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":519569408,"num_objects":7928,"num_object_clones":0,"num_object_copies":23784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8136,"num_write_kb":514048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":140,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007145+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:36:23.007145+0000","last_peered":"2024-08-29T15:36:23.007145+0000","last_clean":"2024-08-29T15:36:23.007145+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:36:23.007145+0000","last_undegraded":"2024-08-29T15:36:23.007145+0000","last_fullsized":"2024-08-29T15:36:23.007145+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":346,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007822+0000","last_change":"2024-08-29T15:16:59.034843+0000","last_active":"2024-08-29T15:36:23.007822+0000","last_peered":"2024-08-29T15:36:23.007822+0000","last_clean":"2024-08-29T15:36:23.007822+0000","last_became_active":"2024-08-29T15:16:59.034289+0000","last_became_peered":"2024-08-29T15:16:59.034289+0000","last_unstale":"2024-08-29T15:36:23.007822+0000","last_undegraded":"2024-08-29T15:36:23.007822+0000","last_fullsized":"2024-08-29T15:36:23.007822+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:30:19.298112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"356'8093","reported_seq":8129,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.250579+0000","last_change":"2024-08-29T15:34:49.399082+0000","last_active":"2024-08-29T15:36:36.250579+0000","last_peered":"2024-08-29T15:36:36.250579+0000","last_clean":"2024-08-29T15:36:36.250579+0000","last_became_active":"2024-08-29T15:34:49.398776+0000","last_became_peered":"2024-08-29T15:34:49.398776+0000","last_unstale":"2024-08-29T15:36:36.250579+0000","last_undegraded":"2024-08-29T15:36:36.250579+0000","last_fullsized":"2024-08-29T15:36:36.250579+0000","mapping_epoch":344,"log_start":"348'1200","ondisk_log_start":"348'1200","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":6893,"log_dups_size":0,"ondisk_log_size":6893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:59:01.377768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":506003456,"num_objects":7721,"num_object_clones":0,"num_object_copies":23163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8093,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":171,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047659+0000","last_change":"2024-08-29T15:31:26.192931+0000","last_active":"2024-08-29T15:36:30.047659+0000","last_peered":"2024-08-29T15:36:30.047659+0000","last_clean":"2024-08-29T15:36:30.047659+0000","last_became_active":"2024-08-29T15:31:26.191104+0000","last_became_peered":"2024-08-29T15:31:26.191104+0000","last_unstale":"2024-08-29T15:36:30.047659+0000","last_undegraded":"2024-08-29T15:36:30.047659+0000","last_fullsized":"2024-08-29T15:36:30.047659+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:06:43.150576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":572,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.048384+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:36:30.048384+0000","last_peered":"2024-08-29T15:36:30.048384+0000","last_clean":"2024-08-29T15:36:30.048384+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:36:30.048384+0000","last_undegraded":"2024-08-29T15:36:30.048384+0000","last_fullsized":"2024-08-29T15:36:30.048384+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"356'8111","reported_seq":8132,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.649250+0000","last_change":"2024-08-29T15:34:49.398582+0000","last_active":"2024-08-29T15:36:36.649250+0000","last_peered":"2024-08-29T15:36:36.649250+0000","last_clean":"2024-08-29T15:36:36.649250+0000","last_became_active":"2024-08-29T15:34:49.398256+0000","last_became_peered":"2024-08-29T15:34:49.398256+0000","last_unstale":"2024-08-29T15:36:36.649250+0000","last_undegraded":"2024-08-29T15:36:36.649250+0000","last_fullsized":"2024-08-29T15:36:36.649250+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":8111,"log_dups_size":0,"ondisk_log_size":8111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:35:41.421188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":502333440,"num_objects":7665,"num_object_clones":0,"num_object_copies":22995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8111,"num_write_kb":504832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":674,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.054193+0000","last_change":"2024-08-29T15:18:10.849323+0000","last_active":"2024-08-29T15:36:30.054193+0000","last_peered":"2024-08-29T15:36:30.054193+0000","last_clean":"2024-08-29T15:36:30.054193+0000","last_became_active":"2024-08-29T15:18:10.848939+0000","last_became_peered":"2024-08-29T15:18:10.848939+0000","last_unstale":"2024-08-29T15:36:30.054193+0000","last_undegraded":"2024-08-29T15:36:30.054193+0000","last_fullsized":"2024-08-29T15:36:30.054193+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T14:51:49.042786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:19:19.804801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":832,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:26.651988+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:36:26.651988+0000","last_peered":"2024-08-29T15:36:26.651988+0000","last_clean":"2024-08-29T15:36:26.651988+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:36:26.651988+0000","last_undegraded":"2024-08-29T15:36:26.651988+0000","last_fullsized":"2024-08-29T15:36:26.651988+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":349,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.000609+0000","last_change":"2024-08-29T15:31:26.189487+0000","last_active":"2024-08-29T15:36:23.000609+0000","last_peered":"2024-08-29T15:36:23.000609+0000","last_clean":"2024-08-29T15:36:23.000609+0000","last_became_active":"2024-08-29T15:31:26.188972+0000","last_became_peered":"2024-08-29T15:31:26.188972+0000","last_unstale":"2024-08-29T15:36:23.000609+0000","last_undegraded":"2024-08-29T15:36:23.000609+0000","last_fullsized":"2024-08-29T15:36:23.000609+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:46:31.878072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":106,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.009808+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:36:23.009808+0000","last_peered":"2024-08-29T15:36:23.009808+0000","last_clean":"2024-08-29T15:36:23.009808+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:36:23.009808+0000","last_undegraded":"2024-08-29T15:36:23.009808+0000","last_fullsized":"2024-08-29T15:36:23.009808+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.a","version":"356'8192","reported_seq":8228,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.338569+0000","last_change":"2024-08-29T15:34:49.665429+0000","last_active":"2024-08-29T15:36:36.338569+0000","last_peered":"2024-08-29T15:36:36.338569+0000","last_clean":"2024-08-29T15:36:36.338569+0000","last_became_active":"2024-08-29T15:34:49.664936+0000","last_became_peered":"2024-08-29T15:34:49.664936+0000","last_unstale":"2024-08-29T15:36:36.338569+0000","last_undegraded":"2024-08-29T15:36:36.338569+0000","last_fullsized":"2024-08-29T15:36:36.338569+0000","mapping_epoch":344,"log_start":"348'1300","ondisk_log_start":"348'1300","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:16:47.799797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":510525440,"num_objects":7790,"num_object_clones":0,"num_object_copies":23370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8192,"num_write_kb":511424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1046,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047507+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:36:30.047507+0000","last_peered":"2024-08-29T15:36:30.047507+0000","last_clean":"2024-08-29T15:36:30.047507+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:36:30.047507+0000","last_undegraded":"2024-08-29T15:36:30.047507+0000","last_fullsized":"2024-08-29T15:36:30.047507+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":591,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.062679+0000","last_change":"2024-08-29T15:31:26.193377+0000","last_active":"2024-08-29T15:36:30.062679+0000","last_peered":"2024-08-29T15:36:30.062679+0000","last_clean":"2024-08-29T15:36:30.062679+0000","last_became_active":"2024-08-29T15:31:26.193034+0000","last_became_peered":"2024-08-29T15:31:26.193034+0000","last_unstale":"2024-08-29T15:36:30.062679+0000","last_undegraded":"2024-08-29T15:36:30.062679+0000","last_fullsized":"2024-08-29T15:36:30.062679+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:25:51.414695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":402,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047346+0000","last_change":"2024-08-29T15:31:26.189633+0000","last_active":"2024-08-29T15:36:30.047346+0000","last_peered":"2024-08-29T15:36:30.047346+0000","last_clean":"2024-08-29T15:36:30.047346+0000","last_became_active":"2024-08-29T15:31:26.189312+0000","last_became_peered":"2024-08-29T15:31:26.189312+0000","last_unstale":"2024-08-29T15:36:30.047346+0000","last_undegraded":"2024-08-29T15:36:30.047346+0000","last_fullsized":"2024-08-29T15:36:30.047346+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:53:49.593868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"356'7905","reported_seq":7926,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:34.414410+0000","last_change":"2024-08-29T15:34:49.665017+0000","last_active":"2024-08-29T15:36:34.414410+0000","last_peered":"2024-08-29T15:36:34.414410+0000","last_clean":"2024-08-29T15:36:34.414410+0000","last_became_active":"2024-08-29T15:34:49.664771+0000","last_became_peered":"2024-08-29T15:34:49.664771+0000","last_unstale":"2024-08-29T15:36:34.414410+0000","last_undegraded":"2024-08-29T15:36:34.414410+0000","last_fullsized":"2024-08-29T15:36:34.414410+0000","mapping_epoch":344,"log_start":"348'500","ondisk_log_start":"348'500","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":7405,"log_dups_size":0,"ondisk_log_size":7405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:12:10.349198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":505872384,"num_objects":7719,"num_object_clones":0,"num_object_copies":23157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7905,"num_write_kb":499968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":609,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.062932+0000","last_change":"2024-08-29T15:31:29.300646+0000","last_active":"2024-08-29T15:36:30.062932+0000","last_peered":"2024-08-29T15:36:30.062932+0000","last_clean":"2024-08-29T15:36:30.062932+0000","last_became_active":"2024-08-29T15:31:29.296994+0000","last_became_peered":"2024-08-29T15:31:29.296994+0000","last_unstale":"2024-08-29T15:36:30.062932+0000","last_undegraded":"2024-08-29T15:36:30.062932+0000","last_fullsized":"2024-08-29T15:36:30.062932+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:04:42.548079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":92,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:29.429933+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:36:29.429933+0000","last_peered":"2024-08-29T15:36:29.429933+0000","last_clean":"2024-08-29T15:36:29.429933+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:36:29.429933+0000","last_undegraded":"2024-08-29T15:36:29.429933+0000","last_fullsized":"2024-08-29T15:36:29.429933+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":618,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.048118+0000","last_change":"2024-08-29T15:17:40.712150+0000","last_active":"2024-08-29T15:36:30.048118+0000","last_peered":"2024-08-29T15:36:30.048118+0000","last_clean":"2024-08-29T15:36:30.048118+0000","last_became_active":"2024-08-29T15:17:40.710782+0000","last_became_peered":"2024-08-29T15:17:40.710782+0000","last_unstale":"2024-08-29T15:36:30.048118+0000","last_undegraded":"2024-08-29T15:36:30.048118+0000","last_fullsized":"2024-08-29T15:36:30.048118+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:45:10.194180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"356'8122","reported_seq":8141,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.615428+0000","last_change":"2024-08-29T15:34:49.878086+0000","last_active":"2024-08-29T15:36:36.615428+0000","last_peered":"2024-08-29T15:36:36.615428+0000","last_clean":"2024-08-29T15:36:36.615428+0000","last_became_active":"2024-08-29T15:34:49.877786+0000","last_became_peered":"2024-08-29T15:34:49.877786+0000","last_unstale":"2024-08-29T15:36:36.615428+0000","last_undegraded":"2024-08-29T15:36:36.615428+0000","last_fullsized":"2024-08-29T15:36:36.615428+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:35:28.352102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":501088256,"num_objects":7646,"num_object_clones":0,"num_object_copies":22938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8122,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":300,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.000526+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:36:23.000526+0000","last_peered":"2024-08-29T15:36:23.000526+0000","last_clean":"2024-08-29T15:36:23.000526+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:36:23.000526+0000","last_undegraded":"2024-08-29T15:36:23.000526+0000","last_fullsized":"2024-08-29T15:36:23.000526+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":141,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:29.428910+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:36:29.428910+0000","last_peered":"2024-08-29T15:36:29.428910+0000","last_clean":"2024-08-29T15:36:29.428910+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:36:29.428910+0000","last_undegraded":"2024-08-29T15:36:29.428910+0000","last_fullsized":"2024-08-29T15:36:29.428910+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":207,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:26.652015+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:36:26.652015+0000","last_peered":"2024-08-29T15:36:26.652015+0000","last_clean":"2024-08-29T15:36:26.652015+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:36:26.652015+0000","last_undegraded":"2024-08-29T15:36:26.652015+0000","last_fullsized":"2024-08-29T15:36:26.652015+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.e","version":"356'7958","reported_seq":7994,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:34.882443+0000","last_change":"2024-08-29T15:34:49.663668+0000","last_active":"2024-08-29T15:36:34.882443+0000","last_peered":"2024-08-29T15:36:34.882443+0000","last_clean":"2024-08-29T15:36:34.882443+0000","last_became_active":"2024-08-29T15:34:49.663353+0000","last_became_peered":"2024-08-29T15:34:49.663353+0000","last_unstale":"2024-08-29T15:36:34.882443+0000","last_undegraded":"2024-08-29T15:36:34.882443+0000","last_fullsized":"2024-08-29T15:36:34.882443+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":7958,"log_dups_size":0,"ondisk_log_size":7958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:19:23.168228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":506986496,"num_objects":7736,"num_object_clones":0,"num_object_copies":23208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7958,"num_write_kb":502208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":232,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:26.652054+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:36:26.652054+0000","last_peered":"2024-08-29T15:36:26.652054+0000","last_clean":"2024-08-29T15:36:26.652054+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:36:26.652054+0000","last_undegraded":"2024-08-29T15:36:26.652054+0000","last_fullsized":"2024-08-29T15:36:26.652054+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1010,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046898+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:36:30.046898+0000","last_peered":"2024-08-29T15:36:30.046898+0000","last_clean":"2024-08-29T15:36:30.046898+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:36:30.046898+0000","last_undegraded":"2024-08-29T15:36:30.046898+0000","last_fullsized":"2024-08-29T15:36:30.046898+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":587,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.049030+0000","last_change":"2024-08-29T15:19:46.810976+0000","last_active":"2024-08-29T15:36:30.049030+0000","last_peered":"2024-08-29T15:36:30.049030+0000","last_clean":"2024-08-29T15:36:30.049030+0000","last_became_active":"2024-08-29T15:19:46.810446+0000","last_became_peered":"2024-08-29T15:19:46.810446+0000","last_unstale":"2024-08-29T15:36:30.049030+0000","last_undegraded":"2024-08-29T15:36:30.049030+0000","last_fullsized":"2024-08-29T15:36:30.049030+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:47:10.191546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"356'7937","reported_seq":7972,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:33.648424+0000","last_change":"2024-08-29T15:34:49.400720+0000","last_active":"2024-08-29T15:36:33.648424+0000","last_peered":"2024-08-29T15:36:33.648424+0000","last_clean":"2024-08-29T15:36:33.648424+0000","last_became_active":"2024-08-29T15:34:49.400373+0000","last_became_peered":"2024-08-29T15:34:49.400373+0000","last_unstale":"2024-08-29T15:36:33.648424+0000","last_undegraded":"2024-08-29T15:36:33.648424+0000","last_fullsized":"2024-08-29T15:36:33.648424+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":7937,"log_dups_size":0,"ondisk_log_size":7937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:36:39.942123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":513343488,"num_objects":7833,"num_object_clones":0,"num_object_copies":23499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7937,"num_write_kb":504640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":592,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046722+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:36:30.046722+0000","last_peered":"2024-08-29T15:36:30.046722+0000","last_clean":"2024-08-29T15:36:30.046722+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:36:30.046722+0000","last_undegraded":"2024-08-29T15:36:30.046722+0000","last_fullsized":"2024-08-29T15:36:30.046722+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":812,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.010608+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:36:23.010608+0000","last_peered":"2024-08-29T15:36:23.010608+0000","last_clean":"2024-08-29T15:36:23.010608+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:36:23.010608+0000","last_undegraded":"2024-08-29T15:36:23.010608+0000","last_fullsized":"2024-08-29T15:36:23.010608+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":584,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.053992+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:36:30.053992+0000","last_peered":"2024-08-29T15:36:30.053992+0000","last_clean":"2024-08-29T15:36:30.053992+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:36:30.053992+0000","last_undegraded":"2024-08-29T15:36:30.053992+0000","last_fullsized":"2024-08-29T15:36:30.053992+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"356'8139","reported_seq":8175,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:33.605699+0000","last_change":"2024-08-29T15:34:49.878565+0000","last_active":"2024-08-29T15:36:33.605699+0000","last_peered":"2024-08-29T15:36:33.605699+0000","last_clean":"2024-08-29T15:36:33.605699+0000","last_became_active":"2024-08-29T15:34:49.878202+0000","last_became_peered":"2024-08-29T15:34:49.878202+0000","last_unstale":"2024-08-29T15:36:33.605699+0000","last_undegraded":"2024-08-29T15:36:33.605699+0000","last_fullsized":"2024-08-29T15:36:33.605699+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":8139,"log_dups_size":0,"ondisk_log_size":8139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:54:47.624475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":526974976,"num_objects":8041,"num_object_clones":0,"num_object_copies":24123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8139,"num_write_kb":517760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":576,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047701+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:36:30.047701+0000","last_peered":"2024-08-29T15:36:30.047701+0000","last_clean":"2024-08-29T15:36:30.047701+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:36:30.047701+0000","last_undegraded":"2024-08-29T15:36:30.047701+0000","last_fullsized":"2024-08-29T15:36:30.047701+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1229,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047666+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:36:30.047666+0000","last_peered":"2024-08-29T15:36:30.047666+0000","last_clean":"2024-08-29T15:36:30.047666+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:36:30.047666+0000","last_undegraded":"2024-08-29T15:36:30.047666+0000","last_fullsized":"2024-08-29T15:36:30.047666+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047301+0000","last_change":"2024-08-29T15:31:26.189321+0000","last_active":"2024-08-29T15:36:30.047301+0000","last_peered":"2024-08-29T15:36:30.047301+0000","last_clean":"2024-08-29T15:36:30.047301+0000","last_became_active":"2024-08-29T15:31:26.188778+0000","last_became_peered":"2024-08-29T15:31:26.188778+0000","last_unstale":"2024-08-29T15:36:30.047301+0000","last_undegraded":"2024-08-29T15:36:30.047301+0000","last_fullsized":"2024-08-29T15:36:30.047301+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:11:57.644178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.d","version":"356'8168","reported_seq":8190,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:36.605159+0000","last_change":"2024-08-29T15:34:49.878253+0000","last_active":"2024-08-29T15:36:36.605159+0000","last_peered":"2024-08-29T15:36:36.605159+0000","last_clean":"2024-08-29T15:36:36.605159+0000","last_became_active":"2024-08-29T15:34:49.877999+0000","last_became_peered":"2024-08-29T15:34:49.877999+0000","last_unstale":"2024-08-29T15:36:36.605159+0000","last_undegraded":"2024-08-29T15:36:36.605159+0000","last_fullsized":"2024-08-29T15:36:36.605159+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:34:48.383395+0000","last_clean_scrub_stamp":"2024-08-29T15:34:48.383395+0000","objects_scrubbed":0,"log_size":8168,"log_dups_size":0,"ondisk_log_size":8168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:49:48.499809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":506003480,"num_objects":7722,"num_object_clones":0,"num_object_copies":23166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7722,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8168,"num_write_kb":508417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":130,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047440+0000","last_change":"2024-08-29T15:31:26.211099+0000","last_active":"2024-08-29T15:36:30.047440+0000","last_peered":"2024-08-29T15:36:30.047440+0000","last_clean":"2024-08-29T15:36:30.047440+0000","last_became_active":"2024-08-29T15:31:26.196998+0000","last_became_peered":"2024-08-29T15:31:26.196998+0000","last_unstale":"2024-08-29T15:36:30.047440+0000","last_undegraded":"2024-08-29T15:36:30.047440+0000","last_fullsized":"2024-08-29T15:36:30.047440+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:07:19.391161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":213,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.055016+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:36:30.055016+0000","last_peered":"2024-08-29T15:36:30.055016+0000","last_clean":"2024-08-29T15:36:30.055016+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:36:30.055016+0000","last_undegraded":"2024-08-29T15:36:30.055016+0000","last_fullsized":"2024-08-29T15:36:30.055016+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:29.428827+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:36:29.428827+0000","last_peered":"2024-08-29T15:36:29.428827+0000","last_clean":"2024-08-29T15:36:29.428827+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:36:29.428827+0000","last_undegraded":"2024-08-29T15:36:29.428827+0000","last_fullsized":"2024-08-29T15:36:29.428827+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":140,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.015454+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:36:23.015454+0000","last_peered":"2024-08-29T15:36:23.015454+0000","last_clean":"2024-08-29T15:36:23.015454+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:36:23.015454+0000","last_undegraded":"2024-08-29T15:36:23.015454+0000","last_fullsized":"2024-08-29T15:36:23.015454+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":150,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007640+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:36:23.007640+0000","last_peered":"2024-08-29T15:36:23.007640+0000","last_clean":"2024-08-29T15:36:23.007640+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:36:23.007640+0000","last_undegraded":"2024-08-29T15:36:23.007640+0000","last_fullsized":"2024-08-29T15:36:23.007640+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":581,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.048662+0000","last_change":"2024-08-29T15:18:34.990191+0000","last_active":"2024-08-29T15:36:30.048662+0000","last_peered":"2024-08-29T15:36:30.048662+0000","last_clean":"2024-08-29T15:36:30.048662+0000","last_became_active":"2024-08-29T15:18:34.989761+0000","last_became_peered":"2024-08-29T15:18:34.989761+0000","last_unstale":"2024-08-29T15:36:30.048662+0000","last_undegraded":"2024-08-29T15:36:30.048662+0000","last_fullsized":"2024-08-29T15:36:30.048662+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:08:32.149708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":339,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046758+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:36:30.046758+0000","last_peered":"2024-08-29T15:36:30.046758+0000","last_clean":"2024-08-29T15:36:30.046758+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:36:30.046758+0000","last_undegraded":"2024-08-29T15:36:30.046758+0000","last_fullsized":"2024-08-29T15:36:30.046758+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.000591+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:36:23.000591+0000","last_peered":"2024-08-29T15:36:23.000591+0000","last_clean":"2024-08-29T15:36:23.000591+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:36:23.000591+0000","last_undegraded":"2024-08-29T15:36:23.000591+0000","last_fullsized":"2024-08-29T15:36:23.000591+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":539,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046175+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:36:30.046175+0000","last_peered":"2024-08-29T15:36:30.046175+0000","last_clean":"2024-08-29T15:36:30.046175+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:36:30.046175+0000","last_undegraded":"2024-08-29T15:36:30.046175+0000","last_fullsized":"2024-08-29T15:36:30.046175+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":594,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.054648+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:36:30.054648+0000","last_peered":"2024-08-29T15:36:30.054648+0000","last_clean":"2024-08-29T15:36:30.054648+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:36:30.054648+0000","last_undegraded":"2024-08-29T15:36:30.054648+0000","last_fullsized":"2024-08-29T15:36:30.054648+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":591,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046676+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:36:30.046676+0000","last_peered":"2024-08-29T15:36:30.046676+0000","last_clean":"2024-08-29T15:36:30.046676+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:36:30.046676+0000","last_undegraded":"2024-08-29T15:36:30.046676+0000","last_fullsized":"2024-08-29T15:36:30.046676+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":210,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046839+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:36:30.046839+0000","last_peered":"2024-08-29T15:36:30.046839+0000","last_clean":"2024-08-29T15:36:30.046839+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:36:30.046839+0000","last_undegraded":"2024-08-29T15:36:30.046839+0000","last_fullsized":"2024-08-29T15:36:30.046839+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046584+0000","last_change":"2024-08-29T15:31:26.195212+0000","last_active":"2024-08-29T15:36:30.046584+0000","last_peered":"2024-08-29T15:36:30.046584+0000","last_clean":"2024-08-29T15:36:30.046584+0000","last_became_active":"2024-08-29T15:31:26.194866+0000","last_became_peered":"2024-08-29T15:31:26.194866+0000","last_unstale":"2024-08-29T15:36:30.046584+0000","last_undegraded":"2024-08-29T15:36:30.046584+0000","last_fullsized":"2024-08-29T15:36:30.046584+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:50:45.933437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":584,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.057248+0000","last_change":"2024-08-29T15:34:31.566349+0000","last_active":"2024-08-29T15:36:30.057248+0000","last_peered":"2024-08-29T15:36:30.057248+0000","last_clean":"2024-08-29T15:36:30.057248+0000","last_became_active":"2024-08-29T15:34:31.566045+0000","last_became_peered":"2024-08-29T15:34:31.566045+0000","last_unstale":"2024-08-29T15:36:30.057248+0000","last_undegraded":"2024-08-29T15:36:30.057248+0000","last_fullsized":"2024-08-29T15:36:30.057248+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:36:21.006547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":572,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.048096+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:36:30.048096+0000","last_peered":"2024-08-29T15:36:30.048096+0000","last_clean":"2024-08-29T15:36:30.048096+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:36:30.048096+0000","last_undegraded":"2024-08-29T15:36:30.048096+0000","last_fullsized":"2024-08-29T15:36:30.048096+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":576,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046561+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:36:30.046561+0000","last_peered":"2024-08-29T15:36:30.046561+0000","last_clean":"2024-08-29T15:36:30.046561+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:36:30.046561+0000","last_undegraded":"2024-08-29T15:36:30.046561+0000","last_fullsized":"2024-08-29T15:36:30.046561+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":139,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007265+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:36:23.007265+0000","last_peered":"2024-08-29T15:36:23.007265+0000","last_clean":"2024-08-29T15:36:23.007265+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:36:23.007265+0000","last_undegraded":"2024-08-29T15:36:23.007265+0000","last_fullsized":"2024-08-29T15:36:23.007265+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":604,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047056+0000","last_change":"2024-08-29T15:32:56.055196+0000","last_active":"2024-08-29T15:36:30.047056+0000","last_peered":"2024-08-29T15:36:30.047056+0000","last_clean":"2024-08-29T15:36:30.047056+0000","last_became_active":"2024-08-29T15:32:56.054876+0000","last_became_peered":"2024-08-29T15:32:56.054876+0000","last_unstale":"2024-08-29T15:36:30.047056+0000","last_undegraded":"2024-08-29T15:36:30.047056+0000","last_fullsized":"2024-08-29T15:36:30.047056+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:46:26.627465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":74,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.015537+0000","last_change":"2024-08-29T15:31:01.734785+0000","last_active":"2024-08-29T15:36:23.015537+0000","last_peered":"2024-08-29T15:36:23.015537+0000","last_clean":"2024-08-29T15:36:23.015537+0000","last_became_active":"2024-08-29T15:31:01.734449+0000","last_became_peered":"2024-08-29T15:31:01.734449+0000","last_unstale":"2024-08-29T15:36:23.015537+0000","last_undegraded":"2024-08-29T15:36:23.015537+0000","last_fullsized":"2024-08-29T15:36:23.015537+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:18:36.797427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.015049+0000","last_change":"2024-08-29T15:19:46.809041+0000","last_active":"2024-08-29T15:36:23.015049+0000","last_peered":"2024-08-29T15:36:23.015049+0000","last_clean":"2024-08-29T15:36:23.015049+0000","last_became_active":"2024-08-29T15:19:46.808632+0000","last_became_peered":"2024-08-29T15:19:46.808632+0000","last_unstale":"2024-08-29T15:36:23.015049+0000","last_undegraded":"2024-08-29T15:36:23.015049+0000","last_fullsized":"2024-08-29T15:36:23.015049+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:48.002804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":271,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.003070+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:36:23.003070+0000","last_peered":"2024-08-29T15:36:23.003070+0000","last_clean":"2024-08-29T15:36:23.003070+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:36:23.003070+0000","last_undegraded":"2024-08-29T15:36:23.003070+0000","last_fullsized":"2024-08-29T15:36:23.003070+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":139,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007788+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:36:23.007788+0000","last_peered":"2024-08-29T15:36:23.007788+0000","last_clean":"2024-08-29T15:36:23.007788+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:36:23.007788+0000","last_undegraded":"2024-08-29T15:36:23.007788+0000","last_fullsized":"2024-08-29T15:36:23.007788+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":271,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.007239+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:36:23.007239+0000","last_peered":"2024-08-29T15:36:23.007239+0000","last_clean":"2024-08-29T15:36:23.007239+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:36:23.007239+0000","last_undegraded":"2024-08-29T15:36:23.007239+0000","last_fullsized":"2024-08-29T15:36:23.007239+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":607,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.046348+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:36:30.046348+0000","last_peered":"2024-08-29T15:36:30.046348+0000","last_clean":"2024-08-29T15:36:30.046348+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:36:30.046348+0000","last_undegraded":"2024-08-29T15:36:30.046348+0000","last_fullsized":"2024-08-29T15:36:30.046348+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":609,"reported_epoch":356,"state":"active+clean","last_fresh":"2024-08-29T15:36:30.047067+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:36:30.047067+0000","last_peered":"2024-08-29T15:36:30.047067+0000","last_clean":"2024-08-29T15:36:30.047067+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:36:30.047067+0000","last_undegraded":"2024-08-29T15:36:30.047067+0000","last_fullsized":"2024-08-29T15:36:30.047067+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":139,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.006883+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:36:23.006883+0000","last_peered":"2024-08-29T15:36:23.006883+0000","last_clean":"2024-08-29T15:36:23.006883+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:36:23.006883+0000","last_undegraded":"2024-08-29T15:36:23.006883+0000","last_fullsized":"2024-08-29T15:36:23.006883+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":237,"reported_epoch":355,"state":"active+clean","last_fresh":"2024-08-29T15:36:23.000495+0000","last_change":"2024-08-29T15:32:56.042376+0000","last_active":"2024-08-29T15:36:23.000495+0000","last_peered":"2024-08-29T15:36:23.000495+0000","last_clean":"2024-08-29T15:36:23.000495+0000","last_became_active":"2024-08-29T15:32:56.036129+0000","last_became_peered":"2024-08-29T15:32:56.036129+0000","last_unstale":"2024-08-29T15:36:23.000495+0000","last_undegraded":"2024-08-29T15:36:23.000495+0000","last_fullsized":"2024-08-29T15:36:23.000495+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:33:55.308823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":8152219672,"num_objects":124394,"num_object_clones":0,"num_object_copies":373182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":124394,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129160,"num_write_kb":8113665,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24511131648,"data_stored":24511119432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122160,"ondisk_log_size":122160,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":7132,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234503,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3096776,"kb_used_data":2534080,"kb_used_omap":1,"kb_used_meta":562686,"kb_avail":90648376,"statfs":{"total":95995035648,"available":92823937024,"internally_reserved":0,"allocated":2594897920,"data_stored":2592683419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":576190923},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496134,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2941056,"kb_used_data":2500536,"kb_used_omap":1,"kb_used_meta":440446,"kb_avail":90804096,"statfs":{"total":95995035648,"available":92983394304,"internally_reserved":0,"allocated":2560548864,"data_stored":2558342783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":451017159},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426721,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4776944,"kb_used_data":4465000,"kb_used_omap":8,"kb_used_meta":311863,"kb_avail":88968208,"statfs":{"total":95995035648,"available":91103444992,"internally_reserved":0,"allocated":4572160000,"data_stored":4569964899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":319348618},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019413,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3609472,"kb_used_data":3024632,"kb_used_omap":1,"kb_used_meta":584830,"kb_avail":90135680,"statfs":{"total":95995035648,"available":92298936320,"internally_reserved":0,"allocated":3097223168,"data_stored":3095016600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":598866374},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435597,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3140076,"kb_used_data":2535076,"kb_used_omap":4,"kb_used_meta":604987,"kb_avail":90605076,"statfs":{"total":95995035648,"available":92779597824,"internally_reserved":0,"allocated":2595917824,"data_stored":2593726627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":619506953},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903449,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3469096,"kb_used_data":3026400,"kb_used_omap":10,"kb_used_meta":442677,"kb_avail":90276056,"statfs":{"total":95995035648,"available":92442681344,"internally_reserved":0,"allocated":3099033600,"data_stored":3096845287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":453301776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762651,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4045164,"kb_used_data":3463716,"kb_used_omap":7,"kb_used_meta":581432,"kb_avail":89699988,"statfs":{"total":95995035648,"available":91852787712,"internally_reserved":0,"allocated":3546845184,"data_stored":3544652793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":595386722},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926177,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3107944,"kb_used_data":2526496,"kb_used_omap":7,"kb_used_meta":581432,"kb_avail":90637208,"statfs":{"total":95995035648,"available":92812500992,"internally_reserved":0,"allocated":2587131904,"data_stored":2584943886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":595386511},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2558984192,"data_stored":2558984192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3528130560,"data_stored":3528130560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3085832192,"data_stored":3085828120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2579496960,"data_stored":2579496960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3069444096,"data_stored":3069444096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4554362880,"data_stored":4554358808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2549940224,"data_stored":2549940224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2584940544,"data_stored":2584936472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:36:38.541 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-29T15:36:38.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-08-29T15:36:38.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:38.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:38.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:38.652 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:38.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:36:38.676 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:38.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:38.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:38.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:38.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:38.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:38.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:38.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:38.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:38.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:38.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:39.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:39.109 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:39.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:39.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:39.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:39.253 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:39.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:39 smithi171 ceph-mon[90031]: from='client.56311 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:36:39.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:39 smithi027 ceph-mon[93921]: from='client.56311 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:36:39.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:39 smithi027 ceph-mon[98970]: from='client.56311 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:36:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:36:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:36:39] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-08-29T15:36:39.355 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:39.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:39.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:39.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:39.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:39.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:39.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:39.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:39.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:39.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:39.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:39.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:39.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:39.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:39.882 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:39.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:39.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:40.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:40.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:40.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:40.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:40.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:40.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:40.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:40.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:40 smithi027 ceph-mon[93921]: pgmap v1324: 89 pgs: 89 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 893 op/s 2024-08-29T15:36:40.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:40 smithi027 ceph-mon[98970]: pgmap v1324: 89 pgs: 89 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 893 op/s 2024-08-29T15:36:40.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:40.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:40.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:40.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:40.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:40.449 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:40 smithi171 ceph-mon[90031]: pgmap v1324: 89 pgs: 89 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 35 MiB/s wr, 893 op/s 2024-08-29T15:36:40.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:40.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:40.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:40.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:40.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:40.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:40.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:40.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:40.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:40.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:40.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:40.852 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:40.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:40.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:40.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:40.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:36:40.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:41.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:41.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:41.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:41.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:41.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:41.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:41.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:41.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:41.432 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:41.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:41.474 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:41.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:41.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:41.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:41.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:41.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:41.735 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:41.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:41.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:41.769 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:41.769 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:41.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:41.866 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:41.867 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:41.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:41.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:42.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:42.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:42.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:42.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:42.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:42.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:42.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:42.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:42 smithi027 ceph-mon[98970]: pgmap v1325: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 94 B/s rd, 25 MiB/s wr, 835 op/s 2024-08-29T15:36:42.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:42 smithi027 ceph-mon[93921]: pgmap v1325: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 94 B/s rd, 25 MiB/s wr, 835 op/s 2024-08-29T15:36:42.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:42.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:42.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:42 smithi171 ceph-mon[90031]: pgmap v1325: 89 pgs: 89 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 94 B/s rd, 25 MiB/s wr, 835 op/s 2024-08-29T15:36:42.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:42.567 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:42.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:42.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:42.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:42.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:42.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:42.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:42.950 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:42.984 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:42.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:43.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:43.118 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:36:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:43.119 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:43.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:43.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:43.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:43.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:43.357 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:43.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:43.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:43.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:43.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1790881233' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:36:43.532 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1790881233' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:36:43.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1790881233' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:36:43.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:43.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:43.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:43.640 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:43.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:43.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:43.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:43.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:43.821 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:43.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:43.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:43.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:43.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:44.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:44.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:44.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:44.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:44.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:44.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:44.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:44.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:44.363 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:44.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:44.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[93921]: pgmap v1326: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 936 op/s 2024-08-29T15:36:44.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:44.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1790881233' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:36:44.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[93921]: osdmap e357: 8 total, 8 up, 8 in 2024-08-29T15:36:44.397 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:44.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[98970]: pgmap v1326: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 936 op/s 2024-08-29T15:36:44.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:44.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1790881233' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:36:44.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[98970]: osdmap e357: 8 total, 8 up, 8 in 2024-08-29T15:36:44.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:44.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:44.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:44.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:44 smithi171 ceph-mon[90031]: pgmap v1326: 89 pgs: 89 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 936 op/s 2024-08-29T15:36:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:44 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1790881233' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:36:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:44 smithi171 ceph-mon[90031]: osdmap e357: 8 total, 8 up, 8 in 2024-08-29T15:36:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:44.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:44.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:44.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:44.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:44.767 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:44.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:44.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:44.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:44.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:44.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:45.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:45.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:45.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:45.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:45.145 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:45.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:45.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:45.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:45.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:45.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:45.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:45.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:45.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:45.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:45.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:45.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:45.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:45.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:45.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:45.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:45.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:45.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:45.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:36:45.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:45.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:45.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:45.970 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:46.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:46.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:46.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:46.142 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:46.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:46.244 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:46.276 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:46.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:46 smithi027 ceph-mon[93921]: pgmap v1328: 89 pgs: 89 active+clean; 7.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 905 op/s 2024-08-29T15:36:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:46 smithi027 ceph-mon[98970]: pgmap v1328: 89 pgs: 89 active+clean; 7.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 905 op/s 2024-08-29T15:36:46.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:46.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:46 smithi171 ceph-mon[90031]: pgmap v1328: 89 pgs: 89 active+clean; 7.2 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 5.3 MiB/s wr, 905 op/s 2024-08-29T15:36:46.517 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:46.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:46.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:46.688 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:46.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:46.828 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:46.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:46.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:46.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:46.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:47.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:47.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:47.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:47.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:47.224 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:47.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:47.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:47.270 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:36:47.270 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:36:47.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:47.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:47.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:47.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:47.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:47.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:47.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:47.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:47.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:47.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:47.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:47.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:47.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:47.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:47.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:48.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:48.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:48.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:48.201 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:48.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:48.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:48.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:48.460 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:48.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:48 smithi171 ceph-mon[90031]: pgmap v1329: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2024-08-29T15:36:48.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:48.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:48 smithi027 ceph-mon[98970]: pgmap v1329: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2024-08-29T15:36:48.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:48 smithi027 ceph-mon[93921]: pgmap v1329: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2024-08-29T15:36:48.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:48.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:48.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:48.823 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:48.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:48.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:48.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:48.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:49.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:49.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:49.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:49.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:49.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:49.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:49.292 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:36:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:36:49] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.51.0" 2024-08-29T15:36:49.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:49.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:49.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:49.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:49.550 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:49.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:49.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:49.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:49.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:49.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:36:49.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:49.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:49.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:49.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:49.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:49.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:49.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:49.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:49.999 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:50.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:50.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:50.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:50.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:50.261 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:50.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:50.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:50.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:50.371 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:50 smithi027 ceph-mon[93921]: pgmap v1330: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-08-29T15:36:50.371 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:50 smithi027 ceph-mon[98970]: pgmap v1330: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-08-29T15:36:50.403 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:50.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:50.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:50.505 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:50.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:50 smithi171 ceph-mon[90031]: pgmap v1330: 89 pgs: 89 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-08-29T15:36:50.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:50.543 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:50.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:50.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:50.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:50.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:50.743 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:36:50.744 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:36:50.751 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:36:50.751 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:36:50.784 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:36:50.784 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:36:50.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:50.816 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:36:50.816 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:36:50.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:50.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:50.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:50.846 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:36:50.847 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:36:50.880 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:36:50.880 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:36:50.920 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:36:50.921 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:36:50.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:50.953 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:36:50.953 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:36:50.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:50.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:51.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:51.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:51.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:51.140 INFO:teuthology.orchestra.run.smithi027.stderr:set pool 1 pg_num to 11 2024-08-29T15:36:51.198 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2787919936' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-29T15:36:51.198 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1705743662' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:51.199 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2787919936' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-29T15:36:51.199 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1705743662' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:51.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:51.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:51.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:51.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:51.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:51.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:51.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:51.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:51.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2787919936' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-29T15:36:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1705743662' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:36:51.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:51.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:51.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:51.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:51.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:51.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:51.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:51.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:51.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:51.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:51.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:51.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:51.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:52.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:52.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:52.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:52.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:52.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:52.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:52.282 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:52 smithi027 ceph-mon[98970]: pgmap v1331: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-08-29T15:36:52.282 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2787919936' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-29T15:36:52.282 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:52 smithi027 ceph-mon[98970]: osdmap e358: 8 total, 8 up, 8 in 2024-08-29T15:36:52.283 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:52 smithi027 ceph-mon[93921]: pgmap v1331: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-08-29T15:36:52.283 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2787919936' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-29T15:36:52.283 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:52 smithi027 ceph-mon[93921]: osdmap e358: 8 total, 8 up, 8 in 2024-08-29T15:36:52.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:52.376 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:52.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:52.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:52.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:52.511 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:52.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:52.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:52 smithi171 ceph-mon[90031]: pgmap v1331: 89 pgs: 89 active+clean; 6.9 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-08-29T15:36:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2787919936' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-29T15:36:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:52 smithi171 ceph-mon[90031]: osdmap e358: 8 total, 8 up, 8 in 2024-08-29T15:36:52.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:52.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:52.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:52.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:52.752 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:52.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:52.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:52.891 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:52.931 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:52.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:52.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:53.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:53.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:53.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:53.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:53.203 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:53.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:53.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:53.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:53.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:53.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:53.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-29T15:36:53.392 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-29T15:36:53.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:53.483 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:53.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:53.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-29T15:36:53.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:53.656 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:53.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:53.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:53.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:53.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:53.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:53.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:53.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:53.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:53.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:53.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:54.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:54.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:54.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:54.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:54.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:54 smithi027 ceph-mon[93921]: pgmap v1333: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-08-29T15:36:54.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-29T15:36:54.176 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:54 smithi027 ceph-mon[93921]: osdmap e359: 8 total, 8 up, 8 in 2024-08-29T15:36:54.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:54 smithi027 ceph-mon[98970]: pgmap v1333: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-08-29T15:36:54.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-29T15:36:54.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:54 smithi027 ceph-mon[98970]: osdmap e359: 8 total, 8 up, 8 in 2024-08-29T15:36:54.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:54.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:54.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:54.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:54.349 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:54.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:54.450 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:54.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:54.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:54 smithi171 ceph-mon[90031]: pgmap v1333: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-08-29T15:36:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-29T15:36:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:54 smithi171 ceph-mon[90031]: osdmap e359: 8 total, 8 up, 8 in 2024-08-29T15:36:54.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:54.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:54.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:54.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:54.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:54.751 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:54.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:54.852 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:54.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:54.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:54.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:55.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:55.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:55.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:55.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:55.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:55.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:55.173 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:36:55.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:55.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:55.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:55.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:55.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:36:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:55.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:55.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:55.500 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:55 smithi027 ceph-mon[98970]: osdmap e360: 8 total, 8 up, 8 in 2024-08-29T15:36:55.500 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1839965057' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:55.500 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:55 smithi027 ceph-mon[93921]: osdmap e360: 8 total, 8 up, 8 in 2024-08-29T15:36:55.500 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1839965057' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:55 smithi171 ceph-mon[90031]: osdmap e360: 8 total, 8 up, 8 in 2024-08-29T15:36:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1839965057' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:36:55.552 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:55.589 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:55.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:55.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:55.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:55.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:55.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:55.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:55.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:55.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:55.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:55.973 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:56.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:56.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:56.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:56.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:56.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:56.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:56.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:56.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:56.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[98970]: pgmap v1336: 99 pgs: 10 unknown, 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[98970]: 1.7 scrub starts 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1839965057' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[98970]: osdmap e361: 8 total, 8 up, 8 in 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[93921]: pgmap v1336: 99 pgs: 10 unknown, 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[93921]: 1.7 scrub starts 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1839965057' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:56.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:56 smithi027 ceph-mon[93921]: osdmap e361: 8 total, 8 up, 8 in 2024-08-29T15:36:56.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:56.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:56.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:56.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:56 smithi171 ceph-mon[90031]: pgmap v1336: 99 pgs: 10 unknown, 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-08-29T15:36:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:56 smithi171 ceph-mon[90031]: 1.7 scrub starts 2024-08-29T15:36:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:56 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:36:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1839965057' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:36:56.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:56 smithi171 ceph-mon[90031]: osdmap e361: 8 total, 8 up, 8 in 2024-08-29T15:36:56.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:56.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:56.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:56.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:56.704 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:36:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:36:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:36:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:36:56.704 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:36:56.748 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:56.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:56.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:56.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:56.888 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:56.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:56.971 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:56.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:57.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:57.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:57.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:57.051 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:36:57.051 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:36:57.052 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-08-29T15:36:57.052 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:36:57.052 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:36:57.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-08-29T15:36:57.137 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:57.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:57.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:57.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:57.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:57.316 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:57.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:57.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:57.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:57.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:57.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:57 smithi171 ceph-mon[90031]: 1.4 scrub starts 2024-08-29T15:36:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:57 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-29T15:36:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:57 smithi027 ceph-mon[93921]: 1.4 scrub starts 2024-08-29T15:36:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:57 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-29T15:36:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:57 smithi027 ceph-mon[98970]: 1.4 scrub starts 2024-08-29T15:36:57.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:57 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-29T15:36:57.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:57.607 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:57.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:57.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:36:57.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:57.751 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:57.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:36:57.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:57.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:57.886 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:57.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:57.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:36:57.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:58.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:58.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:58.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:58.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:36:58.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:58.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:58.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:58.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:58.349 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:58.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:58.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:58.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:58.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:58.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:58.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[93921]: pgmap v1338: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-29T15:36:58.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[93921]: 1.2 scrub starts 2024-08-29T15:36:58.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-29T15:36:58.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[93921]: osdmap e362: 8 total, 8 up, 8 in 2024-08-29T15:36:58.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[98970]: pgmap v1338: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-29T15:36:58.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[98970]: 1.2 scrub starts 2024-08-29T15:36:58.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-29T15:36:58.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:58 smithi027 ceph-mon[98970]: osdmap e362: 8 total, 8 up, 8 in 2024-08-29T15:36:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:58 smithi171 ceph-mon[90031]: pgmap v1338: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-29T15:36:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:58 smithi171 ceph-mon[90031]: 1.2 scrub starts 2024-08-29T15:36:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:58 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-29T15:36:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:58 smithi171 ceph-mon[90031]: osdmap e362: 8 total, 8 up, 8 in 2024-08-29T15:36:58.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:36:58.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:58.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:36:58.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:58.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:58.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:58.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:36:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:58.914 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:58.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:36:59.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:59.080 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:59.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:59.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:36:59.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:59.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:59.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:36:59.274 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:59.274 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:59 smithi027 ceph-mon[93921]: 1.0 scrub starts 2024-08-29T15:36:59.274 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:59 smithi027 ceph-mon[93921]: osdmap e363: 8 total, 8 up, 8 in 2024-08-29T15:36:59.274 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:36:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:59.275 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:36:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:36:59] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.51.0" 2024-08-29T15:36:59.275 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:59 smithi027 ceph-mon[98970]: 1.0 scrub starts 2024-08-29T15:36:59.275 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:59 smithi027 ceph-mon[98970]: osdmap e363: 8 total, 8 up, 8 in 2024-08-29T15:36:59.275 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:36:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:59.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:59.327 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:36:59.386 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:59.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:59.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:36:59.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:36:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:59 smithi171 ceph-mon[90031]: 1.0 scrub starts 2024-08-29T15:36:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:59 smithi171 ceph-mon[90031]: osdmap e363: 8 total, 8 up, 8 in 2024-08-29T15:36:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:36:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:36:59.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:59.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:36:59.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:36:59.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:36:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:59.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:36:59.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:36:59.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:59.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:36:59.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:36:59.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:36:59.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:36:59.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:00.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:00.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:00.098 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:00.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:00.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:00.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:00.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:00.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:00.370 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:00 smithi027 ceph-mon[93921]: pgmap v1341: 99 pgs: 99 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-08-29T15:37:00.370 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:00 smithi027 ceph-mon[93921]: osdmap e364: 8 total, 8 up, 8 in 2024-08-29T15:37:00.370 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:00 smithi027 ceph-mon[98970]: pgmap v1341: 99 pgs: 99 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-08-29T15:37:00.370 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:00 smithi027 ceph-mon[98970]: osdmap e364: 8 total, 8 up, 8 in 2024-08-29T15:37:00.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:00.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:00.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:00.520 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:00.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:00.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:00.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:00.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:00.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:00.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:00.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:00 smithi171 ceph-mon[90031]: pgmap v1341: 99 pgs: 99 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-08-29T15:37:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:00 smithi171 ceph-mon[90031]: osdmap e364: 8 total, 8 up, 8 in 2024-08-29T15:37:00.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:00.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:00.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:00.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:00.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:00.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:01.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:01.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:01.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:01.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:01.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:01.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:01.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:01.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:01.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:01.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:01.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:01.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:01.488 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:01.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:01.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:01.513 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:01 smithi027 ceph-mon[93921]: osdmap e365: 8 total, 8 up, 8 in 2024-08-29T15:37:01.513 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3558945508' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:01.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:01 smithi027 ceph-mon[98970]: osdmap e365: 8 total, 8 up, 8 in 2024-08-29T15:37:01.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3558945508' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:01.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:01.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:01.624 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:01.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:01.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:01.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:01.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:01.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:01 smithi171 ceph-mon[90031]: osdmap e365: 8 total, 8 up, 8 in 2024-08-29T15:37:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3558945508' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:01.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:01.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:01.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:02.004 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:02.038 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:02.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:02.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:02.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:02.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:02.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:02.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:02.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:02.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:02.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:02.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:02.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:02.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:02.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:02.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[98970]: pgmap v1344: 99 pgs: 1 active+remapped, 9 active+undersized+remapped, 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 144 KiB/s, 0 objects/s recovering 2024-08-29T15:37:02.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[98970]: osdmap e366: 8 total, 8 up, 8 in 2024-08-29T15:37:02.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[98970]: 1.3 scrub starts 2024-08-29T15:37:02.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[98970]: 1.9 scrub starts 2024-08-29T15:37:02.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[93921]: pgmap v1344: 99 pgs: 1 active+remapped, 9 active+undersized+remapped, 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 144 KiB/s, 0 objects/s recovering 2024-08-29T15:37:02.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[93921]: osdmap e366: 8 total, 8 up, 8 in 2024-08-29T15:37:02.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[93921]: 1.3 scrub starts 2024-08-29T15:37:02.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:02 smithi027 ceph-mon[93921]: 1.9 scrub starts 2024-08-29T15:37:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:02.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:02.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:02.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:02.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:02.732 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:02.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:02 smithi171 ceph-mon[90031]: pgmap v1344: 99 pgs: 1 active+remapped, 9 active+undersized+remapped, 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 144 KiB/s, 0 objects/s recovering 2024-08-29T15:37:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:02 smithi171 ceph-mon[90031]: osdmap e366: 8 total, 8 up, 8 in 2024-08-29T15:37:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:02 smithi171 ceph-mon[90031]: 1.3 scrub starts 2024-08-29T15:37:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:02 smithi171 ceph-mon[90031]: 1.9 scrub starts 2024-08-29T15:37:02.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:02.876 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:02.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:02.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:03.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:03.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:03.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:03.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:03.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:03.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:03.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:03.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:03.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:03.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:03 smithi027 ceph-mon[93921]: 1.2 scrub starts 2024-08-29T15:37:03.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:03 smithi027 ceph-mon[93921]: 1.4 scrub starts 2024-08-29T15:37:03.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:03 smithi027 ceph-mon[98970]: 1.2 scrub starts 2024-08-29T15:37:03.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:03 smithi027 ceph-mon[98970]: 1.4 scrub starts 2024-08-29T15:37:03.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:03.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:03.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:03 smithi171 ceph-mon[90031]: 1.2 scrub starts 2024-08-29T15:37:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:03 smithi171 ceph-mon[90031]: 1.4 scrub starts 2024-08-29T15:37:03.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:03.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:03.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:03.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:03.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:04.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:04.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:04.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:04.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:04.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:04.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:04.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:04.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:04.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:04.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:04.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:04.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:04.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:04.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:04 smithi027 ceph-mon[93921]: pgmap v1346: 99 pgs: 5 active+undersized+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 123 KiB/s, 0 objects/s recovering 2024-08-29T15:37:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:04 smithi027 ceph-mon[93921]: 1.0 scrub starts 2024-08-29T15:37:04.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:04 smithi027 ceph-mon[98970]: pgmap v1346: 99 pgs: 5 active+undersized+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 123 KiB/s, 0 objects/s recovering 2024-08-29T15:37:04.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:04 smithi027 ceph-mon[98970]: 1.0 scrub starts 2024-08-29T15:37:04.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:04.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:04.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:04.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:04.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:04.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:04 smithi171 ceph-mon[90031]: pgmap v1346: 99 pgs: 5 active+undersized+remapped, 94 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 123 KiB/s, 0 objects/s recovering 2024-08-29T15:37:04.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:04 smithi171 ceph-mon[90031]: 1.0 scrub starts 2024-08-29T15:37:04.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:04.791 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:04.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:04.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:04.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:04.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:05.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:05.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:05.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:05.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:05.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:05.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:05.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:05.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:05.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:37:05.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:37:05.259 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:37:05.260 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:37:05.261 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:37:05.261 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:37:05.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:05.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:05.337 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:05.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:05.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:05.477 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:05.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:05.566 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:05 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:37:05.566 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:05 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:37:05.577 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:05.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:05.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:05.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:05.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:05.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:05.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:05.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:05 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:37:05.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:05.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:05.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:05.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:05.994 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:06.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:06.068 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:06.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:06.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:06.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:06.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:06.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:06.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:06.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:06.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:06.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:06 smithi027 ceph-mon[98970]: pgmap v1347: 99 pgs: 2 active+undersized+remapped, 97 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 96 KiB/s, 0 objects/s recovering 2024-08-29T15:37:06.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:06 smithi027 ceph-mon[93921]: pgmap v1347: 99 pgs: 2 active+undersized+remapped, 97 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 96 KiB/s, 0 objects/s recovering 2024-08-29T15:37:06.544 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:06.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.575 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:06.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:06.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:06.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:06.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:06 smithi171 ceph-mon[90031]: pgmap v1347: 99 pgs: 2 active+undersized+remapped, 97 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 96 KiB/s, 0 objects/s recovering 2024-08-29T15:37:06.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:06.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:06.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:06.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:06.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:07.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:07.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:07.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:07.219 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:07.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:07.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:07.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:07.340 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:37:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:37:07.341 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:37:07.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:07.404 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:07.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:07.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:07.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:07.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:07.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:07.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:07.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:07.700 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:07.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:07.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:07.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:07.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:07.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:07.970 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:07.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:08.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:08.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:37:08.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:08.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:08.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:08.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:08.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:08.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:08.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:08.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:08.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:08.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:08.531 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:37:08.531 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:37:08.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:08 smithi027 ceph-mon[98970]: pgmap v1348: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 75 KiB/s, 0 objects/s recovering 2024-08-29T15:37:08.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:08 smithi027 ceph-mon[98970]: 1.3 scrub starts 2024-08-29T15:37:08.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:08 smithi027 ceph-mon[98970]: 1.9 scrub starts 2024-08-29T15:37:08.532 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:08 smithi027 ceph-mon[93921]: pgmap v1348: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 75 KiB/s, 0 objects/s recovering 2024-08-29T15:37:08.532 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:08 smithi027 ceph-mon[93921]: 1.3 scrub starts 2024-08-29T15:37:08.532 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:08 smithi027 ceph-mon[93921]: 1.9 scrub starts 2024-08-29T15:37:08.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:08.568 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:08.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:08.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:08.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:08.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:08 smithi171 ceph-mon[90031]: pgmap v1348: 99 pgs: 4 active+clean+scrubbing, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 75 KiB/s, 0 objects/s recovering 2024-08-29T15:37:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:08 smithi171 ceph-mon[90031]: 1.3 scrub starts 2024-08-29T15:37:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:08 smithi171 ceph-mon[90031]: 1.9 scrub starts 2024-08-29T15:37:08.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:08.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:08.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:08.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:08.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:08.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:09.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:09.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:09.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:09.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:09.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:09.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:09.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:37:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:37:09] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.51.0" 2024-08-29T15:37:09.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:09.377 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:09.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:09.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:09.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:09 smithi171 ceph-mon[90031]: 1.2 scrub starts 2024-08-29T15:37:09.508 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:09 smithi171 ceph-mon[90031]: 1.4 scrub starts 2024-08-29T15:37:09.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:09.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:09.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:09.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:09.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:09.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:09.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:09.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:09 smithi027 ceph-mon[98970]: 1.2 scrub starts 2024-08-29T15:37:09.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:09 smithi027 ceph-mon[98970]: 1.4 scrub starts 2024-08-29T15:37:09.689 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:09 smithi027 ceph-mon[93921]: 1.2 scrub starts 2024-08-29T15:37:09.689 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:09 smithi027 ceph-mon[93921]: 1.4 scrub starts 2024-08-29T15:37:09.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:09.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:09.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:09.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:09.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:09.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:10.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:10.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:10.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:10.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:37:10.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:10.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:10.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:10.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:10.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:10.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:10.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:10.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:10.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:10.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:10.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:10.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:10 smithi027 ceph-mon[93921]: pgmap v1349: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2024-08-29T15:37:10.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:10 smithi027 ceph-mon[93921]: 1.0 scrub starts 2024-08-29T15:37:10.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:10 smithi027 ceph-mon[98970]: pgmap v1349: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2024-08-29T15:37:10.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:10 smithi027 ceph-mon[98970]: 1.0 scrub starts 2024-08-29T15:37:10.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:10.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:10.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:10.679 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:10 smithi171 ceph-mon[90031]: pgmap v1349: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2024-08-29T15:37:10.679 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:10 smithi171 ceph-mon[90031]: 1.0 scrub starts 2024-08-29T15:37:10.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:10.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:10.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:10.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:10.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:10.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:11.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:11.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:11.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:11.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:11.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:11.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:11.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:11.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:11.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:11.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:11.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:11.477 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:11.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:11.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:11.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:11 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:37:11.610 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:11 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:37:11.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:11.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:11.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:11.711 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:11.718 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:11.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:11 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:37:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:11.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:11.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:11.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:11.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:11.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:12.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:12.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:12.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:12.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:12.266 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:12.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:12.368 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:12.415 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:12.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:12.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:12.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:12.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:12.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:12.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:12 smithi027 ceph-mon[93921]: pgmap v1350: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 927 op/s 2024-08-29T15:37:12.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:12 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:37:12.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:12 smithi027 ceph-mon[98970]: pgmap v1350: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 927 op/s 2024-08-29T15:37:12.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:12 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:37:12.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:12.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:12.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:12 smithi171 ceph-mon[90031]: pgmap v1350: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 927 op/s 2024-08-29T15:37:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:12 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:37:12.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:12.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:12.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:12.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:12.984 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:12.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:13.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:13.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:13.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:13.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:13.225 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:13.248 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:13.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:13.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:13.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:13.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:13.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:13.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:13.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:13.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:13.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:13 smithi027 ceph-mon[98970]: 1.9 scrub starts 2024-08-29T15:37:13.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:13 smithi027 ceph-mon[93921]: 1.9 scrub starts 2024-08-29T15:37:13.604 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:13.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:13.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:13.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:13.673 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:13 smithi171 ceph-mon[90031]: 1.9 scrub starts 2024-08-29T15:37:13.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:13.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:13.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:13.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:13.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:13.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:13.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:14.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:14.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:14.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:14.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:14.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:14.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:14.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:14.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:14.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:14.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:14.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:14.374 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:37:14.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:14.477 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:14.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:14.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:14.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:14.623 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:14.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:14.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:14.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:14.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:14 smithi171 ceph-mon[90031]: pgmap v1351: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2024-08-29T15:37:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:14 smithi171 ceph-mon[90031]: 1.2 scrub starts 2024-08-29T15:37:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:14 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2904707128' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:37:14.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:14 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:37:14.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:14 smithi171 ceph-mon[90031]: 1.3 scrub starts 2024-08-29T15:37:14.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:14.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:14 smithi171 ceph-mon[90031]: 1.4 scrub starts 2024-08-29T15:37:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[93921]: pgmap v1351: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[93921]: 1.2 scrub starts 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2904707128' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[93921]: 1.3 scrub starts 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[93921]: 1.4 scrub starts 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[98970]: pgmap v1351: 99 pgs: 5 active+clean+scrubbing, 94 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[98970]: 1.2 scrub starts 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2904707128' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[98970]: 1.3 scrub starts 2024-08-29T15:37:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:14.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:14 smithi027 ceph-mon[98970]: 1.4 scrub starts 2024-08-29T15:37:14.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:14.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:14.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:14.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:14.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:15.044 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:15.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:15.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:15.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:15.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:15.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:15.278 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:15.322 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:15.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:15.424 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:15.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:15.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:15.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:15.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[93921]: 1.0 scrub starts 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[93921]: osdmap e367: 8 total, 8 up, 8 in 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[98970]: 1.0 scrub starts 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:37:15.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:15 smithi027 ceph-mon[98970]: osdmap e367: 8 total, 8 up, 8 in 2024-08-29T15:37:15.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:15.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:15.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:15.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:15 smithi171 ceph-mon[90031]: 1.0 scrub starts 2024-08-29T15:37:15.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:15 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:37:15.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:15 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:37:15.678 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:15 smithi171 ceph-mon[90031]: osdmap e367: 8 total, 8 up, 8 in 2024-08-29T15:37:15.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:15.735 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:15.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:15.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:15.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:15.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:15.890 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:15.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:15.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:16.000 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:37:16.000 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:37:16.000 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-29T15:37:16.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 1000 0.05 2024-08-29T15:37:16.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:16.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:16.133 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:16.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:16.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:16.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:16.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:16.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:16.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:16.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:16.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:16.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:16 smithi027 ceph-mon[98970]: pgmap v1353: 99 pgs: 6 active+clean+scrubbing, 93 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-08-29T15:37:16.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:16 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:37:16.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:16 smithi027 ceph-mon[98970]: 1.6 scrub ok 2024-08-29T15:37:16.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:16 smithi027 ceph-mon[93921]: pgmap v1353: 99 pgs: 6 active+clean+scrubbing, 93 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-08-29T15:37:16.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:16 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:37:16.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:16 smithi027 ceph-mon[93921]: 1.6 scrub ok 2024-08-29T15:37:16.572 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:16.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:16.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:16.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:16.682 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:16 smithi171 ceph-mon[90031]: pgmap v1353: 99 pgs: 6 active+clean+scrubbing, 93 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-08-29T15:37:16.682 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:16 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:37:16.682 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:16 smithi171 ceph-mon[90031]: 1.6 scrub ok 2024-08-29T15:37:16.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:16.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:16.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:16.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:16.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:16.890 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:16.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:16.992 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:17.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:17.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:17.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:17.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:17.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:17.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:17.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:17.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:17.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:17.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:17.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:17.440 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:17.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:17.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:17.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:17.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:17.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:17.625 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:37:17.625 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:37:17.663 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:37:17.663 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:37:17.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:17.696 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:37:17.697 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:37:17.729 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:37:17.729 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:37:17.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:17.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:17.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:17.769 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:37:17.769 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:37:17.807 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:37:17.808 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:37:17.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:17.861 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:37:17.861 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:168154 8.6 8:66b88237:::benchmark_data_smithi027_306999_object41375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:17.841621+0000", 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.034159951000000001, 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.034196108000000003, 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:37:17.878 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 168154 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.841621+0000", 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.841621+0000", 2024-08-29T15:37:17.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.841622+0000", 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.409e-06 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.841628+0000", 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8390000000000003e-06 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.841643+0000", 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4847e-05 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.841653+0000", 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9450000000000005e-06 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.860416+0000", 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018763610999999999 2024-08-29T15:37:17.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.860531+0000", 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114286 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.860633+0000", 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102497 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.867204+0000", 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065711540000000001 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.867368+0000", 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016344999999999999 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:37:17.883 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:168162 8.e 8:737f9de3:::benchmark_data_smithi027_306999_object41383:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:17.870005+0000", 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0057761749999999997, 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059171190000000002, 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:37:17.884 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 168162 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870005+0000", 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870005+0000", 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870006+0000", 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.567e-06 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870014+0000", 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9230000000000002e-06 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:37:17.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870027+0000", 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3417e-05 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870036+0000", 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4230000000000005e-06 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870216+0000", 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018052199999999999 2024-08-29T15:37:17.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870352+0000", 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013516600000000001 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870479+0000", 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012728800000000001 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:168167 8.6 8:619d2d10:::benchmark_data_smithi027_306999_object41388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:17.870080+0000", 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0057011270000000003, 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059033860000000001, 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 168167 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.889 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870080+0000", 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870080+0000", 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:37:17.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870080+0000", 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.13e-07 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870083+0000", 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2780000000000002e-06 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870092+0000", 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4770000000000003e-06 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.870098+0000", 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2029999999999997e-06 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.871460+0000", 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013616279999999999 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.871784+0000", 2024-08-29T15:37:17.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00032443199999999999 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 6,7", 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.872168+0000", 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038357099999999998 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46475.0:168168 8.2 e367/344 8:406ca821:::benchmark_data_smithi027_306999_object41389:head v 367'10656, mlcod=367'10654)", 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:17.873279+0000", 2024-08-29T15:37:17.893 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.002502003, 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027646649999999999, 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873279+0000", 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:37:17.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873279+0000", 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873280+0000", 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9299999999999997e-07 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873284+0000", 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6759999999999998e-06 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873301+0000", 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6619000000000001e-05 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873309+0000", 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0770000000000008e-06 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873461+0000", 2024-08-29T15:37:17.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015229100000000001 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873480+0000", 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8680000000000001e-05 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46475.0:168166 8.1 e367/344)", 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:17.873351+0000", 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.002429839, 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027358700000000001, 2024-08-29T15:37:17.897 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873351+0000", 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873351+0000", 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873351+0000", 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7e-07 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873355+0000", 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3560000000000001e-06 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873362+0000", 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5259999999999996e-06 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:37:17.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:17.873367+0000", 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1719999999999998e-06 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 5 2024-08-29T15:37:17.900 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:37:17.902 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:37:17.902 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:37:17.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:17.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:17.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:18.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:18.042 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:18.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:18.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:18.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:18.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:37:18.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:18.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:18.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:18.316 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:18.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:18.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:18.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:18.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:18.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:18 smithi171 ceph-mon[90031]: 1.7 scrub starts 2024-08-29T15:37:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:18 smithi171 ceph-mon[90031]: 1.7 scrub ok 2024-08-29T15:37:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:18 smithi171 ceph-mon[90031]: pgmap v1354: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-08-29T15:37:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:18 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:18 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:18 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/887943506' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[93921]: 1.7 scrub starts 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[93921]: 1.7 scrub ok 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[93921]: pgmap v1354: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/887943506' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[98970]: 1.7 scrub starts 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[98970]: 1.7 scrub ok 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[98970]: pgmap v1354: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-08-29T15:37:18.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:18.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:18.571 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:18 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/887943506' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:18.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:18.601 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:18.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:18.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:18.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:18.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:18.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:18.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:18.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:18.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:18.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:19.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:19.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:19.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:19.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:19.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:19.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:19.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:19.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:19.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:19.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:37:19] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.51.0" 2024-08-29T15:37:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[98970]: 1.9 scrub starts 2024-08-29T15:37:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[98970]: 1.9 scrub ok 2024-08-29T15:37:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[98970]: 1.2 scrub starts 2024-08-29T15:37:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[98970]: 1.2 scrub ok 2024-08-29T15:37:19.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[93921]: 1.9 scrub starts 2024-08-29T15:37:19.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[93921]: 1.9 scrub ok 2024-08-29T15:37:19.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[93921]: 1.2 scrub starts 2024-08-29T15:37:19.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:19 smithi027 ceph-mon[93921]: 1.2 scrub ok 2024-08-29T15:37:19.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:19.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:19.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:19 smithi171 ceph-mon[90031]: 1.9 scrub starts 2024-08-29T15:37:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:19 smithi171 ceph-mon[90031]: 1.9 scrub ok 2024-08-29T15:37:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:19 smithi171 ceph-mon[90031]: 1.2 scrub starts 2024-08-29T15:37:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:19 smithi171 ceph-mon[90031]: 1.2 scrub ok 2024-08-29T15:37:19.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:19.604 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:19.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:19.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:19.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:19.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:19.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:19.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:19.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:19.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:19.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:19.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:37:19.989 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:20.020 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:20.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:20.122 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:20.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:20.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:20.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:20.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:20.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:20.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:20.431 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:20.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:20.501 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:20.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:20.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[93921]: pgmap v1355: 99 pgs: 99 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-08-29T15:37:20.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[93921]: 1.0 scrub starts 2024-08-29T15:37:20.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[93921]: 1.0 scrub ok 2024-08-29T15:37:20.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[93921]: 1.5 scrub starts 2024-08-29T15:37:20.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[93921]: 1.5 scrub ok 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[93921]: 1.3 scrub starts 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[93921]: 1.3 scrub ok 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[98970]: pgmap v1355: 99 pgs: 99 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[98970]: 1.0 scrub starts 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[98970]: 1.0 scrub ok 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[98970]: 1.5 scrub starts 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[98970]: 1.5 scrub ok 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[98970]: 1.3 scrub starts 2024-08-29T15:37:20.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:20 smithi027 ceph-mon[98970]: 1.3 scrub ok 2024-08-29T15:37:20.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:20.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:20.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:20.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:20.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:20 smithi171 ceph-mon[90031]: pgmap v1355: 99 pgs: 99 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-08-29T15:37:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:20 smithi171 ceph-mon[90031]: 1.0 scrub starts 2024-08-29T15:37:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:20 smithi171 ceph-mon[90031]: 1.0 scrub ok 2024-08-29T15:37:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:20 smithi171 ceph-mon[90031]: 1.5 scrub starts 2024-08-29T15:37:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:20 smithi171 ceph-mon[90031]: 1.5 scrub ok 2024-08-29T15:37:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:20 smithi171 ceph-mon[90031]: 1.3 scrub starts 2024-08-29T15:37:20.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:20 smithi171 ceph-mon[90031]: 1.3 scrub ok 2024-08-29T15:37:20.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:20.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:20.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:20.855 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:20.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:20.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:20.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:20.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:21.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:21.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:21.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:21.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:21.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:21.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:21.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:21.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:21.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:21.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:21.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:21.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:21.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:21 smithi027 ceph-mon[98970]: 1.4 scrub starts 2024-08-29T15:37:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:21 smithi027 ceph-mon[98970]: 1.4 scrub ok 2024-08-29T15:37:21.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:21 smithi027 ceph-mon[93921]: 1.4 scrub starts 2024-08-29T15:37:21.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:21 smithi027 ceph-mon[93921]: 1.4 scrub ok 2024-08-29T15:37:21.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:21.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:21.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:21.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:21 smithi171 ceph-mon[90031]: 1.4 scrub starts 2024-08-29T15:37:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:21 smithi171 ceph-mon[90031]: 1.4 scrub ok 2024-08-29T15:37:21.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:21.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:21.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:21.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:21.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:21.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:21.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:21.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:22.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:22.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:22.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:22.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:22.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:22.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:22.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:22.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:22.403 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:22.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:22.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:22.504 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:22.509 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:22.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:22.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:22.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:22.549 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:37:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:22 smithi027 ceph-mon[93921]: pgmap v1356: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:37:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:22 smithi027 ceph-mon[98970]: pgmap v1356: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:37:22.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:22.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:22.688 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:22.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:22 smithi171 ceph-mon[90031]: pgmap v1356: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-08-29T15:37:22.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:22.815 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:22.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:22.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:22.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:22.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:22.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:23.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:23.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:23.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:23.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:23.222 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:23.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:23.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:23.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:23.363 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:23.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:23.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:23.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:23.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:23.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:23.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:23.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:23.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:23.808 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:23.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:23.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:23.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:23.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:24.054 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:24.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:24.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:24.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:24.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:24.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:24.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:24.357 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:24.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:24.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:24.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:24.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:24.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:24.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:24 smithi027 ceph-mon[93921]: 1.8 scrub starts 2024-08-29T15:37:24.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:24 smithi027 ceph-mon[93921]: 1.8 scrub ok 2024-08-29T15:37:24.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:24 smithi027 ceph-mon[93921]: pgmap v1357: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-08-29T15:37:24.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:24 smithi027 ceph-mon[98970]: 1.8 scrub starts 2024-08-29T15:37:24.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:24 smithi027 ceph-mon[98970]: 1.8 scrub ok 2024-08-29T15:37:24.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:24 smithi027 ceph-mon[98970]: pgmap v1357: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-08-29T15:37:24.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:24.628 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:24.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:24.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:24.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:24.764 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:24.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:24 smithi171 ceph-mon[90031]: 1.8 scrub starts 2024-08-29T15:37:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:24 smithi171 ceph-mon[90031]: 1.8 scrub ok 2024-08-29T15:37:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:24 smithi171 ceph-mon[90031]: pgmap v1357: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-08-29T15:37:24.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:24.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:24.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:24.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:25.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:25.047 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:25.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:25.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:25.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:25.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:25.297 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:25.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:25.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:25.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:25.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:25.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:25.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:25 smithi027 ceph-mon[98970]: 1.1 scrub starts 2024-08-29T15:37:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:25 smithi027 ceph-mon[98970]: 1.1 scrub ok 2024-08-29T15:37:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'52332','2':'53895','3':'45532'}"} : dispatch 2024-08-29T15:37:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:25 smithi027 ceph-mon[93921]: 1.1 scrub starts 2024-08-29T15:37:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:25 smithi027 ceph-mon[93921]: 1.1 scrub ok 2024-08-29T15:37:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'52332','2':'53895','3':'45532'}"} : dispatch 2024-08-29T15:37:25.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:25.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:25.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:25.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:25.740 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:25.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:25 smithi171 ceph-mon[90031]: 1.1 scrub starts 2024-08-29T15:37:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:25 smithi171 ceph-mon[90031]: 1.1 scrub ok 2024-08-29T15:37:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'52332','2':'53895','3':'45532'}"} : dispatch 2024-08-29T15:37:25.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:25.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:25.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:25.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:25.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:26.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:26.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:26.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:26.250 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:26.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:26.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:26.420 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:26.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:26.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:26.557 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:26.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:26.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: 1.a scrub starts 2024-08-29T15:37:26.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: 1.a scrub ok 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: from='client.56344 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: pgmap v1358: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'52332','2':'53895','3':'45532'}"}]': finished 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: osdmap e368: 8 total, 8 up, 8 in 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3438789551' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: 1.a scrub starts 2024-08-29T15:37:26.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: 1.a scrub ok 2024-08-29T15:37:26.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: from='client.56344 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:26.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: pgmap v1358: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2024-08-29T15:37:26.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'52332','2':'53895','3':'45532'}"}]': finished 2024-08-29T15:37:26.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: osdmap e368: 8 total, 8 up, 8 in 2024-08-29T15:37:26.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3438789551' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:37:26.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:26 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:37:26.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:26.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:26.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: 1.a scrub starts 2024-08-29T15:37:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: 1.a scrub ok 2024-08-29T15:37:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: from='client.56344 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: pgmap v1358: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 969 op/s 2024-08-29T15:37:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'52332','2':'53895','3':'45532'}"}]': finished 2024-08-29T15:37:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: osdmap e368: 8 total, 8 up, 8 in 2024-08-29T15:37:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3438789551' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:37:26.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:26 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:37:26.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:26.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:26.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:26.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:26.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:26.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:26.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:27.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:27.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:27.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:27.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:27.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:27.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:27.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:27.383 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:27.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:27.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:27.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:27.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:27.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:27.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:27.594 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:37:27.595 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:37:27.595 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:37:27.595 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:37:27.595 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[93921]: osdmap e369: 8 total, 8 up, 8 in 2024-08-29T15:37:27.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:37:27.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:37:27.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:37:27.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:27 smithi027 ceph-mon[98970]: osdmap e369: 8 total, 8 up, 8 in 2024-08-29T15:37:27.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:27.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:27.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:27.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:27.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:27 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:37:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:27 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:37:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:27 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:37:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:27 smithi171 ceph-mon[90031]: osdmap e369: 8 total, 8 up, 8 in 2024-08-29T15:37:27.800 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:27.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:27.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:27.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:27.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:27.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:28.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:28.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:28.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:28.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:28.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:28.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:28.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:28.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:28.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:28.354 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:28.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:28.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:28.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:28.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:28.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:28.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:28.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:28.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:28.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:28.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:28.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:28 smithi171 ceph-mon[90031]: pgmap v1361: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-08-29T15:37:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:28 smithi171 ceph-mon[90031]: osdmap e370: 8 total, 8 up, 8 in 2024-08-29T15:37:28.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:28 smithi027 ceph-mon[93921]: pgmap v1361: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-08-29T15:37:28.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:28 smithi027 ceph-mon[93921]: osdmap e370: 8 total, 8 up, 8 in 2024-08-29T15:37:28.814 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:28 smithi027 ceph-mon[98970]: pgmap v1361: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-08-29T15:37:28.814 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:28 smithi027 ceph-mon[98970]: osdmap e370: 8 total, 8 up, 8 in 2024-08-29T15:37:28.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:28.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:28.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:28.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:29.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:29.035 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:29.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:29.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:37:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:37:29] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-08-29T15:37:29.137 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:29.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:29.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:29.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:29.306 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:29.323 INFO:teuthology.orchestra.run.smithi027.stdout:moved 5 / 297 (1.6835%) 2024-08-29T15:37:29.323 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:37:29.323 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 6.43113 -> 5.01093 (expected baseline 5.69951) 2024-08-29T15:37:29.323 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.0 with 28 -> 31 pgs (0.754209 -> 0.835017 * mean) 2024-08-29T15:37:29.323 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.1 with 45 -> 42 pgs (1.21212 -> 1.13131 * mean) 2024-08-29T15:37:29.323 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:37:29.323 INFO:teuthology.orchestra.run.smithi027.stdout:oload 1000 2024-08-29T15:37:29.324 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 0.05 2024-08-29T15:37:29.324 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:37:29.324 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0273 2024-08-29T15:37:29.324 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.2726 2024-08-29T15:37:29.324 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 0.7485 -> 0.7985 2024-08-29T15:37:29.324 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.6448 -> 0.6948 2024-08-29T15:37:29.324 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 weight 0.7975 -> 0.8224 2024-08-29T15:37:29.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:29.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:29.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:29.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:29.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:29 smithi027 ceph-mon[93921]: osdmap e371: 8 total, 8 up, 8 in 2024-08-29T15:37:29.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:29.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:29 smithi027 ceph-mon[98970]: osdmap e371: 8 total, 8 up, 8 in 2024-08-29T15:37:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:29.581 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:29.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:29.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:29.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:29 smithi171 ceph-mon[90031]: osdmap e371: 8 total, 8 up, 8 in 2024-08-29T15:37:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:29.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:29.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:29.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:29.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:29.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:29.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:30.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:30.094 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-29T15:37:30.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 110 1.0 2024-08-29T15:37:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:30.127 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:30.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:30.228 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:30.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:30.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:30.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:30.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:30.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:30.581 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:30.614 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:30.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:30.643 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:30 smithi027 ceph-mon[98970]: pgmap v1364: 99 pgs: 1 peering, 98 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 0 B/s, 3 objects/s recovering 2024-08-29T15:37:30.644 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:30 smithi027 ceph-mon[98970]: osdmap e372: 8 total, 8 up, 8 in 2024-08-29T15:37:30.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:30 smithi027 ceph-mon[93921]: pgmap v1364: 99 pgs: 1 peering, 98 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 0 B/s, 3 objects/s recovering 2024-08-29T15:37:30.644 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:30 smithi027 ceph-mon[93921]: osdmap e372: 8 total, 8 up, 8 in 2024-08-29T15:37:30.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:30.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:30.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:30 smithi171 ceph-mon[90031]: pgmap v1364: 99 pgs: 1 peering, 98 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 0 B/s, 3 objects/s recovering 2024-08-29T15:37:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:30 smithi171 ceph-mon[90031]: osdmap e372: 8 total, 8 up, 8 in 2024-08-29T15:37:30.837 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:37:30.837 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:37:30.837 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:37:30.837 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:37:30.837 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:37:30.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:30.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:30.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:30.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:31.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:31.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:31.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:31.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:31.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:37:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:31.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:31.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:31.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:31.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:31.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:31.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:31.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:31.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:31.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:31.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:31.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:31.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:31.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:31.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:31.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:31.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:31.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:32.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:32.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:32.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:32.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:32.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:32.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:32.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:32.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:32.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:32.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:32.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:32.450 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:32.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:37:32.488 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:32.524 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:32.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:32.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:32.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:32.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:32 smithi027 ceph-mon[98970]: pgmap v1366: 99 pgs: 1 peering, 98 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 0 B/s, 3 objects/s recovering 2024-08-29T15:37:32.597 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:32 smithi027 ceph-mon[93921]: pgmap v1366: 99 pgs: 1 peering, 98 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 0 B/s, 3 objects/s recovering 2024-08-29T15:37:32.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:32.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:32.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:32 smithi171 ceph-mon[90031]: pgmap v1366: 99 pgs: 1 peering, 98 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 0 B/s, 3 objects/s recovering 2024-08-29T15:37:32.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:32.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:32.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:32.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:33.007 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:33.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:33.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:33.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:33.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:33.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:33.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:33.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:33.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:33.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:33.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:33.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:33.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:33.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:33.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:33.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:33.943 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:33.978 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:33.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:34.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:34.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:34.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:34.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:34.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:34.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:34.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:34.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:34.392 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:34.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:34.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:34.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:34 smithi027 ceph-mon[93921]: pgmap v1367: 99 pgs: 1 peering, 98 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 3 B/s, 2 objects/s recovering 2024-08-29T15:37:34.528 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:34 smithi027 ceph-mon[98970]: pgmap v1367: 99 pgs: 1 peering, 98 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 3 B/s, 2 objects/s recovering 2024-08-29T15:37:34.538 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:34.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:34.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:34.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:34.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:34.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:34.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:34 smithi171 ceph-mon[90031]: pgmap v1367: 99 pgs: 1 peering, 98 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 3 B/s, 2 objects/s recovering 2024-08-29T15:37:34.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:34.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:34.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:34.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:34.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:34.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:34.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:34.989 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:35.080 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:35.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:35.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:35.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:35.258 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:35.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:35.282 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:35.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:35.397 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:35.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:35.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:35.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:35.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:35.543 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:35.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:35.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:35.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:35.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:35.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:35.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:35.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:35.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:35.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:35.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:35.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:36.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:36.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:36.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:36.166 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:36.202 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:36.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:36.304 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:36.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:36.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:36.357 INFO:teuthology.orchestra.run.smithi027.stdout:moved 21 / 297 (7.07071%) 2024-08-29T15:37:36.357 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:37:36.357 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 5.01093 -> 5.66651 (expected baseline 5.69951) 2024-08-29T15:37:36.357 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.2 with 30 -> 32 pgs (0.808081 -> 0.861953 * mean) 2024-08-29T15:37:36.357 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.4 with 44 -> 45 pgs (1.18519 -> 1.21212 * mean) 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 1 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0242 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0267 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 weight 1.0000 -> 0.7645 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 weight 1.0000 -> 0.8577 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 0.7985 -> 0.8964 2024-08-29T15:37:36.358 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.6948 -> 0.7621 2024-08-29T15:37:36.442 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:36.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:36.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:36.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:36.646 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:36 smithi027 ceph-mon[98970]: pgmap v1368: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 2 B/s, 2 objects/s recovering 2024-08-29T15:37:36.646 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:36 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58747','1':'56213','3':'49945','5':'50104'}"} : dispatch 2024-08-29T15:37:36.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:36 smithi027 ceph-mon[93921]: pgmap v1368: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 2 B/s, 2 objects/s recovering 2024-08-29T15:37:36.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:36 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58747','1':'56213','3':'49945','5':'50104'}"} : dispatch 2024-08-29T15:37:36.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:36.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:36.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:36 smithi171 ceph-mon[90031]: pgmap v1368: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 921 op/s; 2 B/s, 2 objects/s recovering 2024-08-29T15:37:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:36 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58747','1':'56213','3':'49945','5':'50104'}"} : dispatch 2024-08-29T15:37:36.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:36.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:36.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:36.933 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:36.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:36.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:36.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:37.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:37.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:37.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:37.204 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:37.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:37.328 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:37.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:37.430 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:37.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:37.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:37.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:37.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[98970]: from='client.46607 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:37.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58747','1':'56213','3':'49945','5':'50104'}"}]': finished 2024-08-29T15:37:37.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[98970]: osdmap e373: 8 total, 8 up, 8 in 2024-08-29T15:37:37.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[98970]: osdmap e374: 8 total, 8 up, 8 in 2024-08-29T15:37:37.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[93921]: from='client.46607 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:37.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58747','1':'56213','3':'49945','5':'50104'}"}]': finished 2024-08-29T15:37:37.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[93921]: osdmap e373: 8 total, 8 up, 8 in 2024-08-29T15:37:37.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:37 smithi027 ceph-mon[93921]: osdmap e374: 8 total, 8 up, 8 in 2024-08-29T15:37:37.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:37.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:37.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:37.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:37 smithi171 ceph-mon[90031]: from='client.46607 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:37 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58747','1':'56213','3':'49945','5':'50104'}"}]': finished 2024-08-29T15:37:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:37 smithi171 ceph-mon[90031]: osdmap e373: 8 total, 8 up, 8 in 2024-08-29T15:37:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:37 smithi171 ceph-mon[90031]: osdmap e374: 8 total, 8 up, 8 in 2024-08-29T15:37:37.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:37.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:37.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:37.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:37.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:37.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:38.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:38.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:38.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:38.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:38.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:38.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:38.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:38.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:38.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:38.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:38.435 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:38.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:38.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[98970]: pgmap v1370: 99 pgs: 1 unknown, 2 peering, 96 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 2 B/s, 0 objects/s recovering 2024-08-29T15:37:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:37:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[98970]: osdmap e375: 8 total, 8 up, 8 in 2024-08-29T15:37:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[98970]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 372'11852 2024-08-29T15:37:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[93921]: pgmap v1370: 99 pgs: 1 unknown, 2 peering, 96 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 2 B/s, 0 objects/s recovering 2024-08-29T15:37:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:37:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[93921]: osdmap e375: 8 total, 8 up, 8 in 2024-08-29T15:37:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:38 smithi027 ceph-mon[93921]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 372'11852 2024-08-29T15:37:38.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:38.612 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:38.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:38.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:38.644 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-08-29T15:37:38.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 3.0 2024-08-29T15:37:38.668 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:38 smithi171 ceph-mon[90031]: pgmap v1370: 99 pgs: 1 unknown, 2 peering, 96 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 2 B/s, 0 objects/s recovering 2024-08-29T15:37:38.668 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:38 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:37:38.668 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:38 smithi171 ceph-mon[90031]: osdmap e375: 8 total, 8 up, 8 in 2024-08-29T15:37:38.668 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:38 smithi171 ceph-mon[90031]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 372'11852 2024-08-29T15:37:38.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:38.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:38.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:38.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:38.911 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:38.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:39.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:39.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:39.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:39.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:39.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:39.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:39.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:37:39] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.51.0" 2024-08-29T15:37:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:39.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:39.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:39.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:39.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:39.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:39.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:39.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:39.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:39.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:39.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:39.671 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[93921]: 8.b starting backfill to osd.7 from (0'0,0'0] MAX to 372'11523 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[93921]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 372'11560 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[93921]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 372'11568 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[93921]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 372'11627 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[93921]: osdmap e376: 8 total, 8 up, 8 in 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[98970]: 8.b starting backfill to osd.7 from (0'0,0'0] MAX to 372'11523 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[98970]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 372'11560 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[98970]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 372'11568 2024-08-29T15:37:39.672 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[98970]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 372'11627 2024-08-29T15:37:39.673 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:39 smithi027 ceph-mon[98970]: osdmap e376: 8 total, 8 up, 8 in 2024-08-29T15:37:39.729 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:39.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:39.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:39 smithi171 ceph-mon[90031]: 8.b starting backfill to osd.7 from (0'0,0'0] MAX to 372'11523 2024-08-29T15:37:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:39 smithi171 ceph-mon[90031]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 372'11560 2024-08-29T15:37:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:39 smithi171 ceph-mon[90031]: 8.9 starting backfill to osd.3 from (0'0,0'0] MAX to 372'11568 2024-08-29T15:37:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:39 smithi171 ceph-mon[90031]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 372'11627 2024-08-29T15:37:39.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:39 smithi171 ceph-mon[90031]: osdmap e376: 8 total, 8 up, 8 in 2024-08-29T15:37:39.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:39.907 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:39.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:40.035 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:40.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:40.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:40.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:40.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:40.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:40.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:40.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:40.354 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:40.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:40.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:40.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:40.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:40.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:40.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[98970]: pgmap v1373: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 1 unknown, 2 peering, 91 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 897 op/s; 8454/205203 objects misplaced (4.120%); 527 KiB/s, 8 objects/s recovering 2024-08-29T15:37:40.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[98970]: osdmap e377: 8 total, 8 up, 8 in 2024-08-29T15:37:40.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[98970]: 1.8 scrub starts 2024-08-29T15:37:40.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[98970]: 1.8 scrub ok 2024-08-29T15:37:40.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[93921]: pgmap v1373: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 1 unknown, 2 peering, 91 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 897 op/s; 8454/205203 objects misplaced (4.120%); 527 KiB/s, 8 objects/s recovering 2024-08-29T15:37:40.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[93921]: osdmap e377: 8 total, 8 up, 8 in 2024-08-29T15:37:40.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[93921]: 1.8 scrub starts 2024-08-29T15:37:40.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:40 smithi027 ceph-mon[93921]: 1.8 scrub ok 2024-08-29T15:37:40.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:40.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:40.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:40.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:40 smithi171 ceph-mon[90031]: pgmap v1373: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 1 unknown, 2 peering, 91 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 897 op/s; 8454/205203 objects misplaced (4.120%); 527 KiB/s, 8 objects/s recovering 2024-08-29T15:37:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:40 smithi171 ceph-mon[90031]: osdmap e377: 8 total, 8 up, 8 in 2024-08-29T15:37:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:40 smithi171 ceph-mon[90031]: 1.8 scrub starts 2024-08-29T15:37:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:40 smithi171 ceph-mon[90031]: 1.8 scrub ok 2024-08-29T15:37:40.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:40.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:40.844 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:37:40.844 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:37:40.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:40.883 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:37:40.883 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:37:40.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:40.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:40.938 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:37:40.938 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:37:40.981 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:37:40.984 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:37:41.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:41.018 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:37:41.018 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:37:41.036 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:37:41.037 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:37:41.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:41.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:41.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:41.074 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:37:41.074 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:37:41.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:37:41.112 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:37:41.149 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:37:41.149 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:37:41.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:41.201 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:41.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:41.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:41.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:41.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:41.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:41.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:41.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:41.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:41.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:41.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:41.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:41.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/465826379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:41.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/465826379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:41.767 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/465826379' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:41.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:41.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:41.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:41.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:41.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:42.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:42.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:42.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:42.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:42.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:42.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:42.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:42.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:42.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:42.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:42.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:42.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:42.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:42.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:42.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:42.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:42.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:42.726 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:42 smithi027 ceph-mon[98970]: pgmap v1376: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 1 unknown, 2 peering, 90 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 441 op/s; 12243/203607 objects misplaced (6.013%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:37:42.727 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:42 smithi027 ceph-mon[93921]: pgmap v1376: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 1 unknown, 2 peering, 90 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 441 op/s; 12243/203607 objects misplaced (6.013%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:37:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:42 smithi171 ceph-mon[90031]: pgmap v1376: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 1 unknown, 2 peering, 90 active+clean; 4.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 441 op/s; 12243/203607 objects misplaced (6.013%); 4.6 MiB/s, 73 objects/s recovering 2024-08-29T15:37:42.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:37:42.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:42.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:42.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:42.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:42.917 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:42.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:43.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:43.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:43.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:43.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:43.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:43.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:43.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:43.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:43.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:43.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:43.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:43.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:43.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:43.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:43.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:43.618 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:43.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:43.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:43.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:43.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:43.774 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:43.795 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:43.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:43.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:43.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:43.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:43.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:44.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:44.097 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:44.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:44.105 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:37:44.199 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:44.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:44.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:44.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:44.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:44.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: pgmap v1377: 99 pgs: 4 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 91 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 13 KiB/s rd, 38 KiB/s wr, 504 op/s; 19757/200103 objects misplaced (9.873%); 12 MiB/s, 186 objects/s recovering 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:37:44.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.455 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/818080150' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:37:44.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: pgmap v1377: 99 pgs: 4 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 91 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 13 KiB/s rd, 38 KiB/s wr, 504 op/s; 19757/200103 objects misplaced (9.873%); 12 MiB/s, 186 objects/s recovering 2024-08-29T15:37:44.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:37:44.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:37:44.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:44.456 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:37:44.456 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:37:44.456 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.456 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/818080150' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:37:44.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:44.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:44.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: pgmap v1377: 99 pgs: 4 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 active+undersized+remapped, 91 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 13 KiB/s rd, 38 KiB/s wr, 504 op/s; 19757/200103 objects misplaced (9.873%); 12 MiB/s, 186 objects/s recovering 2024-08-29T15:37:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:37:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:37:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:37:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:37:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:37:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:37:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/818080150' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:37:44.601 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:44.641 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:44.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:44.742 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:44.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:44.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:44.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:44.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:44.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:44.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:45.017 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:45.042 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:37:45.051 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:45.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:45.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:45.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:45.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:45.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:45.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:45 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:37:45.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/818080150' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:37:45.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:45 smithi027 ceph-mon[98970]: osdmap e378: 8 total, 8 up, 8 in 2024-08-29T15:37:45.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:45 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:37:45.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/818080150' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:37:45.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:45 smithi027 ceph-mon[93921]: osdmap e378: 8 total, 8 up, 8 in 2024-08-29T15:37:45.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:45.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:45.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:45.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:45.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:45.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:45 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:37:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/818080150' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:37:45.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:45 smithi171 ceph-mon[90031]: osdmap e378: 8 total, 8 up, 8 in 2024-08-29T15:37:45.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:45.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:45.601 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:45.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:45.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:45.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:45.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:45.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:45.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:45.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:45.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:45.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:45.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:46.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:46.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:46.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:46.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:46.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:46.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:46.283 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:46 smithi027 ceph-mon[98970]: pgmap v1378: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 17490/193149 objects misplaced (9.055%); 24 MiB/s, 390 objects/s recovering 2024-08-29T15:37:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:46 smithi027 ceph-mon[93921]: pgmap v1378: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 17490/193149 objects misplaced (9.055%); 24 MiB/s, 390 objects/s recovering 2024-08-29T15:37:46.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:46.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:46.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:46 smithi171 ceph-mon[90031]: pgmap v1378: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 17490/193149 objects misplaced (9.055%); 24 MiB/s, 390 objects/s recovering 2024-08-29T15:37:46.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:46.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:46.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:46.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:46.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:46.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:46.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:46.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:46.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:46.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:46.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:47.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:47.117 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:47.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:47.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:47.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:47.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:47.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:47.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:47.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:47.396 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:47.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:47.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:47.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:47.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:47.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:47.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:47.682 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:47.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:47.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:47.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:47.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:47.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:47.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:47.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:47.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:48.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:37:48.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:48.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:48.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:48.247 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:37:48.247 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:37:48.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:48.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:48.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:48.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:48.482 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:48.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:48.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:48 smithi171 ceph-mon[90031]: pgmap v1380: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 15939/187896 objects misplaced (8.483%); 30 MiB/s, 480 objects/s recovering 2024-08-29T15:37:48.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:48 smithi027 ceph-mon[98970]: pgmap v1380: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 15939/187896 objects misplaced (8.483%); 30 MiB/s, 480 objects/s recovering 2024-08-29T15:37:48.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:48 smithi027 ceph-mon[93921]: pgmap v1380: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 15939/187896 objects misplaced (8.483%); 30 MiB/s, 480 objects/s recovering 2024-08-29T15:37:48.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:48.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:48.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:48.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:48.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:48.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:48.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:48.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:48.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:48.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:49.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:49.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:49.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:49.070 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:37:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:37:49] "GET /metrics HTTP/1.1" 200 36624 "" "Prometheus/2.51.0" 2024-08-29T15:37:49.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:49.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:49.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:49.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:49.319 INFO:teuthology.orchestra.run.smithi027.stdout:moved 11 / 297 (3.7037%) 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 5.66651 -> 5.01093 (expected baseline 5.69951) 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.5 with 30 -> 28 pgs (0.808081 -> 0.754209 * mean) 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.4 with 45 -> 41 pgs (1.21212 -> 1.10438 * mean) 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:oload 1000 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 3 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0220 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.2196 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 0.8964 -> 1.0000 2024-08-29T15:37:49.320 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.7621 -> 0.8106 2024-08-29T15:37:49.321 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 weight 0.8224 -> 0.8648 2024-08-29T15:37:49.321 INFO:teuthology.orchestra.run.smithi027.stderr:no change 2024-08-29T15:37:49.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:49.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:49.455 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:49.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:49.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:49.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:49.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:37:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:37:49.316+0000 7f7dea97a640 -1 mgr.server reply reply (3) No such process no change 2024-08-29T15:37:49.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:49.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:49.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:49.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:49.732 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:49.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:37:49.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:49.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:49.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:49.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:49.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:49.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:49.979 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:50.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:50.077 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:50.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:50.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:50.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:50.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:50.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:50 smithi027 ceph-mon[98970]: pgmap v1381: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 14964/183225 objects misplaced (8.167%); 28 MiB/s, 455 objects/s recovering 2024-08-29T15:37:50.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:50 smithi027 ceph-mon[98970]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:50 smithi027 ceph-mon[93921]: pgmap v1381: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 14964/183225 objects misplaced (8.167%); 28 MiB/s, 455 objects/s recovering 2024-08-29T15:37:50.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:50 smithi027 ceph-mon[93921]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:50.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:50.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:50.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:50.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:50.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:50.484 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-29T15:37:50.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd test-reweight-by-utilization 110 1.0 2024-08-29T15:37:50.510 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:50 smithi171 ceph-mon[90031]: pgmap v1381: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 14964/183225 objects misplaced (8.167%); 28 MiB/s, 455 objects/s recovering 2024-08-29T15:37:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:50 smithi171 ceph-mon[90031]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:37:50.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:50.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:50.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:50.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:50.660 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:50.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:50.762 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:50.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:50.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:50.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:50.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:50.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:50.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:51.041 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:51.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:51.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:51.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:51.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:51.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:51.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:51.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:51.371 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:51.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:51.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:51.507 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:51.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:51.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:51.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:51.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:51.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:51.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:51.797 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:51.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:51.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:51.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:51.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:51.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:51.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:51.952 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:37:52.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:52.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:52.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:52.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:52.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:52.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:52.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:52.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:52.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:52.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:52.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:52.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:52.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:52.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:52 smithi171 ceph-mon[90031]: pgmap v1382: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 13287/179166 objects misplaced (7.416%); 34 MiB/s, 541 objects/s recovering 2024-08-29T15:37:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:52 smithi027 ceph-mon[93921]: pgmap v1382: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 13287/179166 objects misplaced (7.416%); 34 MiB/s, 541 objects/s recovering 2024-08-29T15:37:52.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:52 smithi027 ceph-mon[98970]: pgmap v1382: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 13287/179166 objects misplaced (7.416%); 34 MiB/s, 541 objects/s recovering 2024-08-29T15:37:52.607 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:52.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:52.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:52.747 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:52.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:52.780 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:52.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:52.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:52.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:52.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:53.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:53.065 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:53.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:53.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:53.168 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:53.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:53.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:53.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:53.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:53.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:53.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:53.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:53.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:53.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:53.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:53.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:53.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:53.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:53.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:53.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:53.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:53.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:53.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:53.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:53.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:54.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:54.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:54.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:54.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:54.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:54.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:54.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:54.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:54.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:54 smithi171 ceph-mon[90031]: pgmap v1383: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 12633/176154 objects misplaced (7.172%); 32 MiB/s, 509 objects/s recovering 2024-08-29T15:37:54.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:54.568 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:54.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:54.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:54 smithi027 ceph-mon[93921]: pgmap v1383: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 12633/176154 objects misplaced (7.172%); 32 MiB/s, 509 objects/s recovering 2024-08-29T15:37:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:54 smithi027 ceph-mon[98970]: pgmap v1383: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 12633/176154 objects misplaced (7.172%); 32 MiB/s, 509 objects/s recovering 2024-08-29T15:37:54.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:54.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:54.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:54.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:54.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:54.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:54.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:55.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:55.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:55.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:55.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:55.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:55.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:55.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:55.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:55.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:55.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:55.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:55.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:55.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3641194361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:55.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3641194361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:55.621 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:37:55.621 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:37:55.621 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:37:55.622 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:37:55.622 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:37:55.622 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:37:55.622 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:37:55.622 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:37:55.622 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:37:55.623 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:37:55.623 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:37:55.623 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:37:55.623 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:37:55.623 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:37:55.623 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:37:55.623 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:37:55.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:55.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:55.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:55.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:55.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3641194361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:37:55.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:55.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:55.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:55.896 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:55.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:55.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:55.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:56.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:56.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.043 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:56.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:56.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.167 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:56.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:56.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:56.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:56.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:56.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:56.435 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:56.537 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:56 smithi027 ceph-mon[93921]: pgmap v1384: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10477/169167 objects misplaced (6.193%); 33 MiB/s, 520 objects/s recovering 2024-08-29T15:37:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:56 smithi027 ceph-mon[93921]: osdmap e379: 8 total, 8 up, 8 in 2024-08-29T15:37:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:56 smithi027 ceph-mon[98970]: pgmap v1384: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10477/169167 objects misplaced (6.193%); 33 MiB/s, 520 objects/s recovering 2024-08-29T15:37:56.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:56 smithi027 ceph-mon[98970]: osdmap e379: 8 total, 8 up, 8 in 2024-08-29T15:37:56.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:56.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:56.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:56.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:56 smithi171 ceph-mon[90031]: pgmap v1384: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 10477/169167 objects misplaced (6.193%); 33 MiB/s, 520 objects/s recovering 2024-08-29T15:37:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:56 smithi171 ceph-mon[90031]: osdmap e379: 8 total, 8 up, 8 in 2024-08-29T15:37:56.815 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:56.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:56.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:56.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:56.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:56.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:56.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:57.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:57.111 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:57.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:57.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:57.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:57.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:57.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:57.389 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:57.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:57.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:57.524 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:57.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:57 smithi027 ceph-mon[93921]: osdmap e380: 8 total, 8 up, 8 in 2024-08-29T15:37:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:57 smithi027 ceph-mon[98970]: osdmap e380: 8 total, 8 up, 8 in 2024-08-29T15:37:57.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:57.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:57.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:57.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:57.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:57 smithi171 ceph-mon[90031]: osdmap e380: 8 total, 8 up, 8 in 2024-08-29T15:37:57.800 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:57.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:57.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:57.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:57.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:58.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:58.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:58.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:58.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:58.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:58.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:58.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:58.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:58.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:58.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:58.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:58.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:37:58.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:58.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:58.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:37:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:58 smithi027 ceph-mon[93921]: pgmap v1387: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 8677/164364 objects misplaced (5.279%); 40 MiB/s, 635 objects/s recovering 2024-08-29T15:37:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:58 smithi027 ceph-mon[98970]: pgmap v1387: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 8677/164364 objects misplaced (5.279%); 40 MiB/s, 635 objects/s recovering 2024-08-29T15:37:58.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:58.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:58.640 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:58.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:58.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:37:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:58 smithi171 ceph-mon[90031]: pgmap v1387: 99 pgs: 1 active+remapped+backfill_wait, 4 active+remapped+backfilling, 94 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 8677/164364 objects misplaced (5.279%); 40 MiB/s, 635 objects/s recovering 2024-08-29T15:37:58.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:58.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:37:58.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:37:58.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:58.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:37:59.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:37:59.059 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:59.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:37:59.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:59.197 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:59.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:59.299 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:37:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:37:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:37:59] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-29T15:37:59.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:59.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:59.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:37:59.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:59.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:59.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:59.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:59.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:37:59.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:37:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:59.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:37:59.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:37:59.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:59.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:37:59.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:37:59.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:37:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:37:59.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:37:59.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:00.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:37:59 smithi171 ceph-mon[90031]: pgmap v1388: 99 pgs: 5 active+remapped+backfilling, 94 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 7304/160626 objects misplaced (4.547%); 37 MiB/s, 596 objects/s recovering 2024-08-29T15:38:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:00.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:37:59 smithi027 ceph-mon[93921]: pgmap v1388: 99 pgs: 5 active+remapped+backfilling, 94 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 7304/160626 objects misplaced (4.547%); 37 MiB/s, 596 objects/s recovering 2024-08-29T15:38:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:37:59 smithi027 ceph-mon[98970]: pgmap v1388: 99 pgs: 5 active+remapped+backfilling, 94 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 7304/160626 objects misplaced (4.547%); 37 MiB/s, 596 objects/s recovering 2024-08-29T15:38:00.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:00.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:00.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:00.269 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:00.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:00.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:00.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:00.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:00.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:00.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:00.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:00.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:00.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:00.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:00.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:00.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:00.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:00.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:00.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:01.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:01.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:01.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:01.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:01.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:01.376 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:01.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:01.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:01.517 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:01.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:01.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:01.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:01.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:01.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:01.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:01.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:01.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:01.986 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:01.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:01 smithi027 ceph-mon[93921]: pgmap v1389: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 6648/156420 objects misplaced (4.250%); 38 MiB/s, 609 objects/s recovering 2024-08-29T15:38:02.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:01 smithi027 ceph-mon[98970]: pgmap v1389: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 6648/156420 objects misplaced (4.250%); 38 MiB/s, 609 objects/s recovering 2024-08-29T15:38:02.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:02.122 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:02.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:02.224 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:02.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:02.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:02.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:01 smithi171 ceph-mon[90031]: pgmap v1389: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 6648/156420 objects misplaced (4.250%); 38 MiB/s, 609 objects/s recovering 2024-08-29T15:38:02.351 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:02.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:02.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:02.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:02.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:02.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:02.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:02.625 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:02.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:02.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:02.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:02.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:02.812 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:02.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:02.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:02.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:02.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:02.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:02.975 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:38:03.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:03.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:03.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:03.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:03.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:03.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:03.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3945480619' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:02 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:03.315 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3945480619' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:03.315 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:02 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:03.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3945480619' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:03.316 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:02 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:03.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:03.349 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:03.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:03.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:03.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:03.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:03.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:03.625 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:03.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:03.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:03.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:03.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:03.880 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:38:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:38:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:38:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:38:03.880 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:38:03.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:03.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:03.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:03.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:04.006 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:04.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:04.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:04.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:04.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:04.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:04.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:03 smithi171 ceph-mon[90031]: pgmap v1390: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6035/153246 objects misplaced (3.938%); 29 MiB/s, 463 objects/s recovering 2024-08-29T15:38:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:03 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:03 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:38:04.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:03 smithi171 ceph-mon[90031]: osdmap e381: 8 total, 8 up, 8 in 2024-08-29T15:38:04.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[98970]: pgmap v1390: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6035/153246 objects misplaced (3.938%); 29 MiB/s, 463 objects/s recovering 2024-08-29T15:38:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:38:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[98970]: osdmap e381: 8 total, 8 up, 8 in 2024-08-29T15:38:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[93921]: pgmap v1390: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 6035/153246 objects misplaced (3.938%); 29 MiB/s, 463 objects/s recovering 2024-08-29T15:38:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:38:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:03 smithi027 ceph-mon[93921]: osdmap e381: 8 total, 8 up, 8 in 2024-08-29T15:38:04.324 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:04.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:04.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:04.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:04.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:04.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:04.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:04.605 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:04.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:04.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:04.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:04.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:04.748 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:04.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:04.849 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:04.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:04.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:04.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:05.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:05.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:05.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:05.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:05.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:05.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:05.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:05.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:05.365 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:05.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:05.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:05.466 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:05.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:05.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:05.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:05.601 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:05.647 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:05.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:05.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:38:05.732 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:05.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:05.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:05.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:05.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:05.912 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:05.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:05.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:06.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:06.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:06.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:06.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:06.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:06.206 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:05 smithi027 ceph-mon[93921]: pgmap v1392: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 4796/146664 objects misplaced (3.270%); 29 MiB/s, 460 objects/s recovering 2024-08-29T15:38:06.206 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:06 smithi027 ceph-mon[98970]: pgmap v1392: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 4796/146664 objects misplaced (3.270%); 29 MiB/s, 460 objects/s recovering 2024-08-29T15:38:06.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:06.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:06.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:05 smithi171 ceph-mon[90031]: pgmap v1392: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 4796/146664 objects misplaced (3.270%); 29 MiB/s, 460 objects/s recovering 2024-08-29T15:38:06.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:06.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:06.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:06.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:06.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:38:06.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:06.534 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:06.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:06.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:06.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:06.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:06.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:06.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:06.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:06.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:06.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:06.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:06.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:06.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:06.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:38:07.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:07.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:07.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:07.141 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:07.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:07.205 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:07.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:07.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:07.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:07.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:07.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:07.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:07.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:07.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:07.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:07.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:07.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:07.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:07.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:07.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:07.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:07.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:07.889 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:07.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:07.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:08.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:08.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:08.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:08.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:08.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:08.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:08.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:08 smithi171 ceph-mon[90031]: pgmap v1393: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 4183/141468 objects misplaced (2.957%); 22 MiB/s, 358 objects/s recovering 2024-08-29T15:38:08.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:08.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:08.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:08.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:08 smithi027 ceph-mon[93921]: pgmap v1393: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 4183/141468 objects misplaced (2.957%); 22 MiB/s, 358 objects/s recovering 2024-08-29T15:38:08.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:08 smithi027 ceph-mon[98970]: pgmap v1393: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 4183/141468 objects misplaced (2.957%); 22 MiB/s, 358 objects/s recovering 2024-08-29T15:38:08.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:08.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:08.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:08.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:08.601 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:08.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:08.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:08.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:08.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:08.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:08.890 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:08.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:08.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:09.037 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:09.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:09.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:09.163 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:38:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:38:09] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-29T15:38:09.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:09.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:09.172 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:09.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:09.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:09.318 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:09.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:09.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:09.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:09.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:09.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:moved 15 / 297 (5.05051%) 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 5.66651 -> 5.77576 (expected baseline 5.69951) 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.5 with 30 -> 26 pgs (0.808081 -> 0.700337 * mean) 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.4 with 45 -> 39 pgs (1.21212 -> 1.05051 * mean) 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 1 2024-08-29T15:38:09.563 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:38:09.564 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0187 2024-08-29T15:38:09.564 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0206 2024-08-29T15:38:09.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 weight 0.7645 -> 0.6621 2024-08-29T15:38:09.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 0.8964 -> 1.0000 2024-08-29T15:38:09.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 weight 0.8577 -> 0.7774 2024-08-29T15:38:09.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 weight 1.0000 -> 0.9081 2024-08-29T15:38:09.564 INFO:teuthology.orchestra.run.smithi027.stderr:no change 2024-08-29T15:38:09.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:09.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:09.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:09.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:09.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:09.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:09.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:09.739 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:38:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:38:09.560+0000 7f7dea97a640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-08-29T15:38:09.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:09.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:09.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:09.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:10.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:10.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:10.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:10.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:10.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:10.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:10.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:10.263 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:10 smithi027 ceph-mon[93921]: pgmap v1394: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 2897/136980 objects misplaced (2.115%); 22 MiB/s, 357 objects/s recovering 2024-08-29T15:38:10.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:10 smithi027 ceph-mon[98970]: pgmap v1394: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 2897/136980 objects misplaced (2.115%); 22 MiB/s, 357 objects/s recovering 2024-08-29T15:38:10.267 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:10 smithi171 ceph-mon[90031]: pgmap v1394: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 2897/136980 objects misplaced (2.115%); 22 MiB/s, 357 objects/s recovering 2024-08-29T15:38:10.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:10.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:10.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:10.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:10.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:10.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148195 7.5 7.29cfb6a5 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 405.18720243400003, 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.001068407, 2024-08-29T15:38:10.456 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148195 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191640+0000", 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2989999999999999e-06 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191646+0000", 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8690000000000001e-06 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191664+0000", 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8189000000000002e-05 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191673+0000", 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7229999999999993e-06 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.198665+0000", 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069919730000000003 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.198675+0000", 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8740000000000007e-06 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192690+0000", 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99401576199999997 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192706+0000", 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5718000000000001e-05 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148242 7.5 7.35df8025 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:25.313183+0000", 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 405.06565673799997, 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87956382, 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148242 2024-08-29T15:38:10.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313183+0000", 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313183+0000", 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313185+0000", 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.186e-06 2024-08-29T15:38:10.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313189+0000", 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2339999999999996e-06 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313201+0000", 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1674e-05 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313208+0000", 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3390000000000004e-06 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313262+0000", 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4545e-05 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.313268+0000", 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6069999999999998e-06 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192738+0000", 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87947040700000001 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192747+0000", 2024-08-29T15:38:10.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8279999999999992e-06 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191681 7.5 7:a9d21a90:::benchmark_data_smithi027_271924_object75340:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.43239256200002, 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.12813253, 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:10.468 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.469 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.469 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.469 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191681 2024-08-29T15:38:10.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.469 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:38:10.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:38:10.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946449+0000", 2024-08-29T15:38:10.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6509999999999999e-06 2024-08-29T15:38:10.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946455+0000", 2024-08-29T15:38:10.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8769999999999998e-06 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946472+0000", 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7169000000000001e-05 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946481+0000", 2024-08-29T15:38:10.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3289999999999994e-06 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946564+0000", 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3615000000000005e-05 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op must wait for map", 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946571+0000", 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0849999999999999e-06 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.952704+0000", 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061325340000000002 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.952713+0000", 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2709999999999998e-06 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042516+0000", 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.089802597 2024-08-29T15:38:10.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042641+0000", 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012506600000000001 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074432+0000", 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.031791257000000003 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074580+0000", 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014807899999999999 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191685 7.5 7:a0ab1ff6:::benchmark_data_smithi027_271924_object75344:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.43233808700001, 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1510580690000001, 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.477 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191685 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946503+0000", 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4199999999999996e-07 2024-08-29T15:38:10.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946505+0000", 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2589999999999999e-06 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946512+0000", 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9999999999999999e-06 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946517+0000", 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.079e-06 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946614+0000", 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.692e-05 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting_for_map not empty", 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946619+0000", 2024-08-29T15:38:10.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1200000000000001e-06 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.953271+0000", 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066521210000000004 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.953278+0000", 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.204e-06 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042965+0000", 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.089686486 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043078+0000", 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113163 2024-08-29T15:38:10.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097459+0000", 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.054381391000000001 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097560+0000", 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100784 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.484 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191701 7.5 7:a40d5d11:::benchmark_data_smithi027_271924_object75360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.39729535100003, 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.099438114, 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191701 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981546+0000", 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.291e-06 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981551+0000", 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9779999999999996e-06 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981563+0000", 2024-08-29T15:38:10.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1787000000000001e-05 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981570+0000", 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4240000000000004e-06 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981641+0000", 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1018000000000003e-05 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981649+0000", 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9379999999999992e-06 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043134+0000", 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0614847059999999 2024-08-29T15:38:10.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043265+0000", 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013072100000000001 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043851+0000", 2024-08-29T15:38:10.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058628399999999996 2024-08-29T15:38:10.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.067960+0000", 2024-08-29T15:38:10.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024109189 2024-08-29T15:38:10.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.069771+0000", 2024-08-29T15:38:10.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018104740000000001 2024-08-29T15:38:10.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080779+0000", 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011008132 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080848+0000", 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9249000000000001e-05 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080983+0000", 2024-08-29T15:38:10.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013492300000000001 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191702 7.5 7:a63127fd:::benchmark_data_smithi027_271924_object75361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.39725297699999, 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.099668844, 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.494 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191702 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981588+0000", 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7399999999999998e-07 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981590+0000", 2024-08-29T15:38:10.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.266e-06 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981597+0000", 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.579e-06 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981602+0000", 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3329999999999996e-06 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981671+0000", 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9581999999999999e-05 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981677+0000", 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9000000000000003e-06 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065439+0000", 2024-08-29T15:38:10.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0837620489999999 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065523+0000", 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3906000000000006e-05 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065611+0000", 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7347000000000002e-05 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072205+0000", 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065941050000000003 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072219+0000", 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3817e-05 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081102+0000", 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088836209999999995 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081154+0000", 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1907000000000003e-05 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081256+0000", 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101958 2024-08-29T15:38:10.501 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191704 7.5 7:adaa8151:::benchmark_data_smithi027_271924_object75363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.39188742099998, 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0945452710000001, 2024-08-29T15:38:10.502 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191704 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986955+0000", 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.781e-06 2024-08-29T15:38:10.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986961+0000", 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2349999999999996e-06 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986976+0000", 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5676e-05 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986984+0000", 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8369999999999997e-06 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.001951+0000", 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014966225 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.001960+0000", 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8379999999999997e-06 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.069808+0000", 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.067847722 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.070348+0000", 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00053970699999999995 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.070512+0000", 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016436399999999999 2024-08-29T15:38:10.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074579+0000", 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004066315 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080650+0000", 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060713909999999998 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081349+0000", 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00069884000000000005 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081398+0000", 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9524999999999998e-05 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081498+0000", 2024-08-29T15:38:10.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9815000000000006e-05 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191709 7.5 7:aeaa9bf4:::benchmark_data_smithi027_271924_object75368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.39038449399999, 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0993208800000001, 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.510 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191709 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:38:10.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988457+0000", 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4270000000000001e-06 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988462+0000", 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0159999999999999e-06 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988477+0000", 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5116999999999999e-05 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988484+0000", 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1799999999999999e-06 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.003492+0000", 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015007802000000001 2024-08-29T15:38:10.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.003518+0000", 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5970999999999999e-05 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072246+0000", 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.068727446 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072383+0000", 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013752099999999999 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072536+0000", 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015290200000000001 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077323+0000", 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047870919999999997 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081054+0000", 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037308939999999998 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087498+0000", 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006444015 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087668+0000", 2024-08-29T15:38:10.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016991900000000001 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087777+0000", 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108578 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191717 7.5 7:a9f87d2e:::benchmark_data_smithi027_271924_object75376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:38:10.517 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.372051742, 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0898794949999999, 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191717 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:38:10.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006790+0000", 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9970000000000001e-06 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006797+0000", 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4570000000000002e-06 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006830+0000", 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2867999999999998e-05 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006840+0000", 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.01e-05 2024-08-29T15:38:10.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006951+0000", 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011073600000000001 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006963+0000", 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1977999999999999e-05 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074609+0000", 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0676467890000001 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074705+0000", 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5200999999999999e-05 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074792+0000", 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6955999999999999e-05 2024-08-29T15:38:10.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080664+0000", 2024-08-29T15:38:10.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058721310000000001 2024-08-29T15:38:10.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081300+0000", 2024-08-29T15:38:10.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00063660400000000003 2024-08-29T15:38:10.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096470+0000", 2024-08-29T15:38:10.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015169472 2024-08-29T15:38:10.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096534+0000", 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3911999999999994e-05 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096668+0000", 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134294 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.526 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191718 7.5 7:a50fea8b:::benchmark_data_smithi027_271924_object75377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.37197228700001, 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.090114451, 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191718 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006869+0000", 2024-08-29T15:38:10.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2099999999999996e-07 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006872+0000", 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2729999999999998e-06 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006882+0000", 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8779999999999993e-06 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006887+0000", 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6409999999999999e-06 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006985+0000", 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7449999999999997e-05 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006994+0000", 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5799999999999998e-06 2024-08-29T15:38:10.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077356+0000", 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0703613439999999 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077468+0000", 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112322 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077596+0000", 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012779999999999999 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087397+0000", 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098012389999999998 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087443+0000", 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6204e-05 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096811+0000", 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093673639999999996 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096888+0000", 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7793000000000005e-05 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096982+0000", 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3942000000000003e-05 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.534 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191761 7.5 7:a52c750c:::benchmark_data_smithi027_271924_object75420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.29060222800001, 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0210746239999999, 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191761 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:38:10.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:38:10.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088240+0000", 2024-08-29T15:38:10.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6300000000000001e-06 2024-08-29T15:38:10.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088247+0000", 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9990000000000002e-06 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088261+0000", 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4416000000000001e-05 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088269+0000", 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1629999999999996e-06 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088333+0000", 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.347e-05 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088342+0000", 2024-08-29T15:38:10.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.363e-06 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081533+0000", 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99319060000000003 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081648+0000", 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115339 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081776+0000", 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012817300000000001 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087457+0000", 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056813039999999999 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087842+0000", 2024-08-29T15:38:10.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038465299999999999 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109101+0000", 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021259324 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109209+0000", 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107187 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109313+0000", 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104003 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191763 7.5 7:a9d168ed:::benchmark_data_smithi027_271924_object75422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.287296481, 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.018041102, 2024-08-29T15:38:10.543 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191763 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091547+0000", 2024-08-29T15:38:10.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8590000000000001e-06 2024-08-29T15:38:10.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091556+0000", 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4599999999999992e-06 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091580+0000", 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4314000000000001e-05 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091592+0000", 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1768e-05 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091686+0000", 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4029000000000004e-05 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091696+0000", 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7410000000000004e-06 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084389+0000", 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99269340699999997 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084526+0000", 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136444 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084664+0000", 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013856400000000001 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087792+0000", 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003127286 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087905+0000", 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113612 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109403+0000", 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021497480999999999 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109491+0000", 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7936000000000006e-05 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109585+0000", 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4201000000000002e-05 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191826 7.5 7:a4f0d2f2:::benchmark_data_smithi027_271924_object75485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.224887+0000", 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.15395339600002, 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89589589999999997, 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.553 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191826 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224887+0000", 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224887+0000", 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224888+0000", 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.468e-06 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224894+0000", 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9989999999999997e-06 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224907+0000", 2024-08-29T15:38:10.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3117e-05 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224916+0000", 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9409999999999996e-06 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224984+0000", 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7309000000000002e-05 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.224996+0000", 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2308e-05 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087963+0000", 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86296706599999995 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.088088+0000", 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012504700000000001 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.088213+0000", 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012473599999999999 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096738+0000", 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085248749999999995 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097073+0000", 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033500400000000002 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120619+0000", 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023546629999999999 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120679+0000", 2024-08-29T15:38:10.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9697000000000001e-05 2024-08-29T15:38:10.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120783+0000", 2024-08-29T15:38:10.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010370300000000001 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191832 7.5 7:abd490dc:::benchmark_data_smithi027_271924_object75491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.253103+0000", 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.12573701500003, 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86786427600000005, 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.561 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191832 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253103+0000", 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253103+0000", 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253105+0000", 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1739999999999999e-06 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253113+0000", 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.464e-06 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253131+0000", 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8467000000000001e-05 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253142+0000", 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0973e-05 2024-08-29T15:38:10.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253227+0000", 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4938999999999994e-05 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.253242+0000", 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4722999999999999e-05 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.090751+0000", 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83750952899999997 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.090874+0000", 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000122059 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.091012+0000", 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013845799999999999 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097056+0000", 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060442359999999997 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097136+0000", 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9408999999999998e-05 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120833+0000", 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023696993 2024-08-29T15:38:10.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120879+0000", 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5948000000000001e-05 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.120967+0000", 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8904000000000001e-05 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191850 7.5 7:a96a40a5:::benchmark_data_smithi027_271924_object75509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.290683+0000", 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.088157368, 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83046312, 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191850 2024-08-29T15:38:10.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290683+0000", 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290683+0000", 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290685+0000", 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7630000000000001e-06 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290692+0000", 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5789999999999998e-06 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290710+0000", 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7948000000000002e-05 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290718+0000", 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8080000000000005e-06 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290781+0000", 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2927000000000002e-05 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.290791+0000", 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.006e-05 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.093701+0000", 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80291038400000003 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.093810+0000", 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108713 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.093953+0000", 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014307600000000001 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097143+0000", 2024-08-29T15:38:10.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031897739999999998 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097599+0000", 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045595800000000001 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.121007+0000", 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023407661999999999 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.121054+0000", 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7503999999999998e-05 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.121146+0000", 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1964000000000002e-05 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.576 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191855 7.5 7:aa7d08ea:::benchmark_data_smithi027_271924_object75514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.302772+0000", 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.076068217, 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.82153611599999998, 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191855 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.577 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302772+0000", 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302772+0000", 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302774+0000", 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0420000000000001e-06 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302781+0000", 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9319999999999999e-06 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302801+0000", 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9596e-05 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302810+0000", 2024-08-29T15:38:10.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8900000000000002e-06 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302895+0000", 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4055999999999996e-05 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.302908+0000", 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2989999999999999e-05 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097630+0000", 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79472256399999996 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097730+0000", 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9567000000000004e-05 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097857+0000", 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127324 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.106251+0000", 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083942419999999997 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109062+0000", 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028109110000000001 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124144+0000", 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015081695000000001 2024-08-29T15:38:10.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124201+0000", 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7362999999999999e-05 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124308+0000", 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106944 2024-08-29T15:38:10.584 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191856 7.5 7:af2f8f1a:::benchmark_data_smithi027_271924_object75515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.304572+0000", 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.07426786100001, 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.81997056599999996, 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.585 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191856 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304572+0000", 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304572+0000", 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304574+0000", 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5400000000000001e-06 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304585+0000", 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0951999999999999e-05 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304598+0000", 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3587999999999999e-05 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304606+0000", 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9899999999999997e-06 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304669+0000", 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2188999999999997e-05 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.304679+0000", 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0149999999999999e-05 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.100714+0000", 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79603570099999998 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.100834+0000", 2024-08-29T15:38:10.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119759 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.100967+0000", 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013277100000000001 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109323+0000", 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083562849999999998 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109365+0000", 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1319999999999997e-05 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124347+0000", 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014982812 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124444+0000", 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6793000000000007e-05 2024-08-29T15:38:10.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124543+0000", 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8715999999999995e-05 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191865 7.5 7:a304caa0:::benchmark_data_smithi027_271924_object75524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.374259+0000", 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 321.00458151800001, 2024-08-29T15:38:10.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.75050312799999996, 2024-08-29T15:38:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:10.593 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191865 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374259+0000", 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374259+0000", 2024-08-29T15:38:10.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374261+0000", 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2680000000000001e-06 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374269+0000", 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4139999999999994e-06 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374290+0000", 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0143e-05 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374300+0000", 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0037000000000001e-05 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374379+0000", 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9752000000000005e-05 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.374392+0000", 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2704999999999999e-05 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.103523+0000", 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72913066900000001 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.103653+0000", 2024-08-29T15:38:10.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000130576 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.103799+0000", 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014593900000000001 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109328+0000", 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055292700000000002 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.113609+0000", 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042803959999999997 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124621+0000", 2024-08-29T15:38:10.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011012088999999999 2024-08-29T15:38:10.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124673+0000", 2024-08-29T15:38:10.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1977000000000001e-05 2024-08-29T15:38:10.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.124762+0000", 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8893000000000004e-05 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185381 8.a 8:593dd5d4:::benchmark_data_smithi027_306999_object58602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:38:10.601 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 34.001060246999998, 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8689627519999998, 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:10.602 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185381 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377781+0000", 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377785+0000", 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5480000000000002e-06 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377794+0000", 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6290000000000005e-06 2024-08-29T15:38:10.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377804+0000", 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3079999999999997e-06 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.387009+0000", 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092053380000000004 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.387019+0000", 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8940000000000005e-06 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088587+0000", 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70156798499999995 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088600+0000", 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2778e-05 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238539+0000", 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1499392150000001 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238659+0000", 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012044699999999999 2024-08-29T15:38:10.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238741+0000", 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1568000000000002e-05 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246057+0000", 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007315963 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246083+0000", 2024-08-29T15:38:10.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5687999999999999e-05 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246298+0000", 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021570700000000001 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246644+0000", 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034599799999999998 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246681+0000", 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6872999999999997e-05 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246743+0000", 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1496999999999998e-05 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185404 8.a 8:510e3083:::benchmark_data_smithi027_306999_object58625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:38:10.610 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 33.955378533999998, 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.823589948, 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185404 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:10.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423463+0000", 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4500000000000001e-06 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423469+0000", 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1870000000000002e-06 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423483+0000", 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3193000000000001e-05 2024-08-29T15:38:10.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:10.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423489+0000", 2024-08-29T15:38:10.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0220000000000001e-06 2024-08-29T15:38:10.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423549+0000", 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0466e-05 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423556+0000", 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9029999999999998e-06 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088626+0000", 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.66507021300000002 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088634+0000", 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9379999999999992e-06 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242305+0000", 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1536706910000001 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242434+0000", 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129484 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242523+0000", 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8343000000000004e-05 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246560+0000", 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040375419999999999 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246569+0000", 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6570000000000003e-06 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246605+0000", 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6331000000000002e-05 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246965+0000", 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00036038900000000002 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246996+0000", 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0054000000000001e-05 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:10.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:10.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:10.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.247052+0000", 2024-08-29T15:38:10.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6085000000000003e-05 2024-08-29T15:38:10.689 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.689 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.689 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.689 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:10.689 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:10.689 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:38:10.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:10.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:10.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:10.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:10.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:10.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:10.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:10.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:10.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:10.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:11.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:11.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:11.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:11.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:11.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:11.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:11.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:11.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:11.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:11 smithi027 ceph-mon[98970]: from='client.46625 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:38:11.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:11 smithi027 ceph-mon[98970]: osdmap e382: 8 total, 8 up, 8 in 2024-08-29T15:38:11.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:11 smithi027 ceph-mon[93921]: from='client.46625 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:38:11.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:11 smithi027 ceph-mon[93921]: osdmap e382: 8 total, 8 up, 8 in 2024-08-29T15:38:11.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:11.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:11.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:11.474 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:11.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:11.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:11 smithi171 ceph-mon[90031]: from='client.46625 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:38:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:11 smithi171 ceph-mon[90031]: osdmap e382: 8 total, 8 up, 8 in 2024-08-29T15:38:11.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:11.645 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:11.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:11.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:11.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:11.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:11.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:11.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:38:11.882 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:11.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:11.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:12.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:12.063 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:12.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:12.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:12.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:12.216 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:12 smithi027 ceph-mon[93921]: pgmap v1396: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 2808/132717 objects misplaced (2.116%); 19 MiB/s, 310 objects/s recovering 2024-08-29T15:38:12.216 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:12 smithi027 ceph-mon[93921]: osdmap e383: 8 total, 8 up, 8 in 2024-08-29T15:38:12.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:12 smithi027 ceph-mon[98970]: pgmap v1396: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 2808/132717 objects misplaced (2.116%); 19 MiB/s, 310 objects/s recovering 2024-08-29T15:38:12.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:12 smithi027 ceph-mon[98970]: osdmap e383: 8 total, 8 up, 8 in 2024-08-29T15:38:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:12.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:12.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:12.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:12.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:12.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:12.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:12.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:12.510 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:12.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:12 smithi171 ceph-mon[90031]: pgmap v1396: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 2808/132717 objects misplaced (2.116%); 19 MiB/s, 310 objects/s recovering 2024-08-29T15:38:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:12 smithi171 ceph-mon[90031]: osdmap e383: 8 total, 8 up, 8 in 2024-08-29T15:38:12.613 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:12.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:12.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:12.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:12.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:12.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:12.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:12.923 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:12.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:13.025 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:13.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:13.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:13.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:13.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:13.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:13.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:13.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:13.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:13.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:13.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:13.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:13.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:13.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:13.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:13.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:13.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:13.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:13.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:13.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:13.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:13.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:13.902 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:13.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:14.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:14.042 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:14.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:14.067 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-29T15:38:14.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 1000 1.0 2024-08-29T15:38:14.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:14.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:14.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:14.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:14.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:14.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:14 smithi027 ceph-mon[93921]: pgmap v1398: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 2526/130383 objects misplaced (1.937%); 14 MiB/s, 228 objects/s recovering 2024-08-29T15:38:14.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:14.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:14 smithi027 ceph-mon[98970]: pgmap v1398: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 2526/130383 objects misplaced (1.937%); 14 MiB/s, 228 objects/s recovering 2024-08-29T15:38:14.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:14.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:14.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:14.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:14.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:14.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:14.495 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:14.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:14 smithi171 ceph-mon[90031]: pgmap v1398: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 2526/130383 objects misplaced (1.937%); 14 MiB/s, 228 objects/s recovering 2024-08-29T15:38:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:14.596 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:14.627 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:14.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:14.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:14.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:14.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:14.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:14.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:14.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:14.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:15.010 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:15.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:15.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:15.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.174 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:15.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:15.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:15.306 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:15.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:15.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:15.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:15.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:15.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:15.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:15.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:15.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:15.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:15.864 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:15.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:15.966 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:15.977 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:15.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:15.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:15.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:16.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:16.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:16.139 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:16.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:16.240 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:16.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:16.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:16.294 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:16 smithi027 ceph-mon[93921]: pgmap v1399: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 1428/124569 objects misplaced (1.146%); 18 MiB/s, 295 objects/s recovering 2024-08-29T15:38:16.294 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:16 smithi027 ceph-mon[98970]: pgmap v1399: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 1428/124569 objects misplaced (1.146%); 18 MiB/s, 295 objects/s recovering 2024-08-29T15:38:16.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:16.431 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:16.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:16 smithi171 ceph-mon[90031]: pgmap v1399: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 1428/124569 objects misplaced (1.146%); 18 MiB/s, 295 objects/s recovering 2024-08-29T15:38:16.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:16.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:16.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:16.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:16.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:16.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:16.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:16.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:16.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:16.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:16.904 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:16.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:16.989 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:16.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:16.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:17.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:17.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:17.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:17.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:17.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:17.256 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:17.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:17.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:17.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:17.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:17.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:17.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:17.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:17.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:17.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:17.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:17.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:17.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:17.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:17.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:17.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:17.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:17.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:17.950 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:17.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:17.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:18.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:18.075 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:38:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:18.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:18.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:18.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:18.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:18.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:18.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:18 smithi027 ceph-mon[98970]: pgmap v1400: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1346/119490 objects misplaced (1.126%); 10 MiB/s, 159 objects/s recovering 2024-08-29T15:38:18.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:18 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/893406594' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:38:18.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:18 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:38:18.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:18 smithi027 ceph-mon[93921]: pgmap v1400: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1346/119490 objects misplaced (1.126%); 10 MiB/s, 159 objects/s recovering 2024-08-29T15:38:18.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:18 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/893406594' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:38:18.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:18 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:38:18.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:18.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:18.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:18.489 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:18 smithi171 ceph-mon[90031]: pgmap v1400: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1346/119490 objects misplaced (1.126%); 10 MiB/s, 159 objects/s recovering 2024-08-29T15:38:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:18 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/893406594' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:38:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:18 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:38:18.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:18.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:18.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:18.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:18.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:18.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:18.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:18.766 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:18.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:18.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:18.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:18.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:18.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:18.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:19.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:19.046 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:38:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:38:19] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-08-29T15:38:19.066 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:19.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:19.201 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:19.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:19 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:19 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:38:19.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:19 smithi027 ceph-mon[98970]: osdmap e384: 8 total, 8 up, 8 in 2024-08-29T15:38:19.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:19 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:19.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:19 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:38:19.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:19 smithi027 ceph-mon[93921]: osdmap e384: 8 total, 8 up, 8 in 2024-08-29T15:38:19.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:19.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:19.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:19.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:19 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:19 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:38:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:19 smithi171 ceph-mon[90031]: osdmap e384: 8 total, 8 up, 8 in 2024-08-29T15:38:19.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:19.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:19.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:19.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:19.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:19.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:19.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:19.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:19.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:20.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:20.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:20.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:20.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:20.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:20.235 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:20.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:20.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:20.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:20.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:20.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:20.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:20.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:20.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:20.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:20 smithi171 ceph-mon[90031]: pgmap v1402: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 831/114402 objects misplaced (0.726%); 14 MiB/s, 218 objects/s recovering 2024-08-29T15:38:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:20 smithi171 ceph-mon[90031]: osdmap e385: 8 total, 8 up, 8 in 2024-08-29T15:38:20.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:20 smithi027 ceph-mon[98970]: pgmap v1402: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 831/114402 objects misplaced (0.726%); 14 MiB/s, 218 objects/s recovering 2024-08-29T15:38:20.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:20 smithi027 ceph-mon[98970]: osdmap e385: 8 total, 8 up, 8 in 2024-08-29T15:38:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:20 smithi027 ceph-mon[93921]: pgmap v1402: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 97 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 831/114402 objects misplaced (0.726%); 14 MiB/s, 218 objects/s recovering 2024-08-29T15:38:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:20 smithi027 ceph-mon[93921]: osdmap e385: 8 total, 8 up, 8 in 2024-08-29T15:38:20.586 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:20.608 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:20.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:20.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:20.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:20.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:20.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:20.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:20.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:20.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:20.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:20.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:21.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:21.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:21.033 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:38:21.033 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:38:21.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.033 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218135 8.c 8:366edb88:::benchmark_data_smithi027_306999_object91356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2024-08-29T15:38:21.033 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.024152+0000", 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0077517000000000003, 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077839459999999999, 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:21.034 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218135 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024152+0000", 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024152+0000", 2024-08-29T15:38:21.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024154+0000", 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.421e-06 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024158+0000", 2024-08-29T15:38:21.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8639999999999998e-06 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024171+0000", 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3067999999999999e-05 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024179+0000", 2024-08-29T15:38:21.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1920000000000005e-06 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024262+0000", 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3005999999999998e-05 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024370+0000", 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108138 2024-08-29T15:38:21.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.024479+0000", 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109059 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46475.0:218133 8.a e386/374 8:58d49990:::benchmark_data_smithi027_306999_object91354:head v 386'13690, mlcod=386'13686)", 2024-08-29T15:38:21.039 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.026846+0000", 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0050584200000000001, 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051608230000000001, 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.026846+0000", 2024-08-29T15:38:21.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.026846+0000", 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.026847+0000", 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.209e-06 2024-08-29T15:38:21.041 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.026857+0000", 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8300000000000008e-06 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.026871+0000", 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4481000000000001e-05 2024-08-29T15:38:21.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.026880+0000", 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9099999999999994e-06 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.026988+0000", 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107604 2024-08-29T15:38:21.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.027011+0000", 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3558999999999999e-05 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46475.0:218138 8.a e386/374 8:5da5dfbd:::benchmark_data_smithi027_306999_object91359:head v 386'13691, mlcod=386'13687)", 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.028954+0000", 2024-08-29T15:38:21.044 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0029497619999999999, 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031011770000000001, 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.028954+0000", 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.028954+0000", 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.028955+0000", 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.018e-06 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:21.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.028962+0000", 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2759999999999997e-06 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.028974+0000", 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2376e-05 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.028982+0000", 2024-08-29T15:38:21.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9480000000000008e-06 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.030423+0000", 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014407809999999999 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.030444+0000", 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1166999999999999e-05 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:21.048 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:21.049 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:21.049 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:21.049 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:38:21.049 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 3 2024-08-29T15:38:21.049 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:38:21.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:21.156 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:21.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:21.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:21 smithi027 ceph-mon[93921]: osdmap e386: 8 total, 8 up, 8 in 2024-08-29T15:38:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:21 smithi027 ceph-mon[98970]: osdmap e386: 8 total, 8 up, 8 in 2024-08-29T15:38:21.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:21.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:21.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:21 smithi171 ceph-mon[90031]: osdmap e386: 8 total, 8 up, 8 in 2024-08-29T15:38:21.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.533 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:21.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:21.634 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:21.656 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:21.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:21.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:21.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:21.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:21.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:21.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:22.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:22.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:22.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:22.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:22.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:22.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:22.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:22.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:22.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:22.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:22.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:22.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:22.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:22.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:22.467 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:22 smithi027 ceph-mon[98970]: pgmap v1405: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 200/109872 objects misplaced (0.182%); 11 MiB/s, 170 objects/s recovering 2024-08-29T15:38:22.467 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:22 smithi027 ceph-mon[98970]: osdmap e387: 8 total, 8 up, 8 in 2024-08-29T15:38:22.468 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:22 smithi027 ceph-mon[93921]: pgmap v1405: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 200/109872 objects misplaced (0.182%); 11 MiB/s, 170 objects/s recovering 2024-08-29T15:38:22.468 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:22 smithi027 ceph-mon[93921]: osdmap e387: 8 total, 8 up, 8 in 2024-08-29T15:38:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:22 smithi171 ceph-mon[90031]: pgmap v1405: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 200/109872 objects misplaced (0.182%); 11 MiB/s, 170 objects/s recovering 2024-08-29T15:38:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:22 smithi171 ceph-mon[90031]: osdmap e387: 8 total, 8 up, 8 in 2024-08-29T15:38:22.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:22.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:22.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:22.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:22.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:22.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:22.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:22.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:22.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:22.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:22.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:22.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:23.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:23.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:23.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:23.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:23.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:23.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:23.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:23.431 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:23.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:23 smithi171 ceph-mon[90031]: osdmap e388: 8 total, 8 up, 8 in 2024-08-29T15:38:23.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:23.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:23.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:23 smithi027 ceph-mon[98970]: osdmap e388: 8 total, 8 up, 8 in 2024-08-29T15:38:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:23 smithi027 ceph-mon[93921]: osdmap e388: 8 total, 8 up, 8 in 2024-08-29T15:38:23.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:23.700 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:23.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:23.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:23.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:23.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:23.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:23.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:23.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:24.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:24.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:24.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:24.236 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:24.269 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:24.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:24.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:24.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:24.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:24.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:24.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:24 smithi171 ceph-mon[90031]: pgmap v1408: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 200/107715 objects misplaced (0.186%); 8.7 MiB/s, 138 objects/s recovering 2024-08-29T15:38:24.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:24.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:24.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:24 smithi027 ceph-mon[98970]: pgmap v1408: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 200/107715 objects misplaced (0.186%); 8.7 MiB/s, 138 objects/s recovering 2024-08-29T15:38:24.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:24 smithi027 ceph-mon[93921]: pgmap v1408: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 97 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 200/107715 objects misplaced (0.186%); 8.7 MiB/s, 138 objects/s recovering 2024-08-29T15:38:24.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:24.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:24.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:24.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:24.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:24.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:24.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:24.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:24.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:24.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:24.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:25.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:25.124 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:25.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:25.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:25.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:25.237 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:38:25.237 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:38:25.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:25.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:25.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:38:25.283 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:38:25.322 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:38:25.322 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:38:25.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:25.370 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:38:25.370 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:38:25.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:25.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:25.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:25.413 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:38:25.414 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:38:25.457 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:38:25.457 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:38:25.497 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:38:25.497 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:38:25.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:25.528 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:38:25.528 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:38:25.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:25.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:25.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:25.685 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:25.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:25.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:25.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:25.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:25.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:25.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:38:25.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:25.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:25.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:25.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:26.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:26.132 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:26.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:26.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:26.133 INFO:teuthology.orchestra.run.smithi027.stdout:moved 8 / 297 (2.6936%) 2024-08-29T15:38:26.133 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:38:26.133 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 5.66651 -> 4.75493 (expected baseline 5.69951) 2024-08-29T15:38:26.133 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.5 with 30 -> 29 pgs (0.808081 -> 0.781145 * mean) 2024-08-29T15:38:26.133 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.4 with 45 -> 41 pgs (1.21212 -> 1.10438 * mean) 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout:oload 1000 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 1 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0158 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.1584 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 weight 0.8964 -> 1.0000 2024-08-29T15:38:26.134 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 weight 0.8224 -> 0.8441 2024-08-29T15:38:26.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:26.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:26.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:26.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:26.362 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:26.384 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:26.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:26.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:26.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:26.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:26.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:26 smithi171 ceph-mon[90031]: pgmap v1409: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 200/102912 objects misplaced (0.194%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:38:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3309742421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:26 smithi171 ceph-mon[90031]: from='client.36753 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:38:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','2':'55321'}"} : dispatch 2024-08-29T15:38:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[98970]: pgmap v1409: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 200/102912 objects misplaced (0.194%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:38:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3309742421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[98970]: from='client.36753 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:38:26.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','2':'55321'}"} : dispatch 2024-08-29T15:38:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[93921]: pgmap v1409: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 200/102912 objects misplaced (0.194%); 5.9 MiB/s, 95 objects/s recovering 2024-08-29T15:38:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3309742421' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[93921]: from='client.36753 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:38:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','2':'55321'}"} : dispatch 2024-08-29T15:38:26.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:26.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:26.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:26.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:26.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:26.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:26.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:26.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:26.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:26.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:26.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:26.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:27.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:27.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:27.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:27.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:27.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:27.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:27.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:27.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:27.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:27 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'65536','2':'55321'}"}]': finished 2024-08-29T15:38:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:27 smithi027 ceph-mon[93921]: osdmap e389: 8 total, 8 up, 8 in 2024-08-29T15:38:27.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:27 smithi027 ceph-mon[93921]: osdmap e390: 8 total, 8 up, 8 in 2024-08-29T15:38:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:27 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'65536','2':'55321'}"}]': finished 2024-08-29T15:38:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:27 smithi027 ceph-mon[98970]: osdmap e389: 8 total, 8 up, 8 in 2024-08-29T15:38:27.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:27 smithi027 ceph-mon[98970]: osdmap e390: 8 total, 8 up, 8 in 2024-08-29T15:38:27.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:27.375 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:27.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:27.477 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:27.500 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:27 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'65536','2':'55321'}"}]': finished 2024-08-29T15:38:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:27 smithi171 ceph-mon[90031]: osdmap e389: 8 total, 8 up, 8 in 2024-08-29T15:38:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:27 smithi171 ceph-mon[90031]: osdmap e390: 8 total, 8 up, 8 in 2024-08-29T15:38:27.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:27.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:27.728 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:27.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:27.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:27.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:27.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:27.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:27.908 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:27.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:28.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:28.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:28.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:28.128 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:28.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:28.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:28.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:28.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:28.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:28.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:28.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:28 smithi027 ceph-mon[93921]: pgmap v1411: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2.1 MiB/s, 33 objects/s recovering 2024-08-29T15:38:28.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:28 smithi027 ceph-mon[93921]: osdmap e391: 8 total, 8 up, 8 in 2024-08-29T15:38:28.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:28 smithi027 ceph-mon[98970]: pgmap v1411: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2.1 MiB/s, 33 objects/s recovering 2024-08-29T15:38:28.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:28 smithi027 ceph-mon[98970]: osdmap e391: 8 total, 8 up, 8 in 2024-08-29T15:38:28.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:28.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.511 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:28.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:28 smithi171 ceph-mon[90031]: pgmap v1411: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2.1 MiB/s, 33 objects/s recovering 2024-08-29T15:38:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:28 smithi171 ceph-mon[90031]: osdmap e391: 8 total, 8 up, 8 in 2024-08-29T15:38:28.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:28.656 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:28.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:38:28.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:28.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:28.892 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:28.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:28.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:29.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:29.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:29.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:29.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:29.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:29.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:29.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:29.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:29 smithi027 ceph-mon[93921]: 8.c starting backfill to osd.0 from (0'0,0'0] MAX to 388'14211 2024-08-29T15:38:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:29 smithi027 ceph-mon[93921]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 388'13802 2024-08-29T15:38:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:29 smithi027 ceph-mon[98970]: 8.c starting backfill to osd.0 from (0'0,0'0] MAX to 388'14211 2024-08-29T15:38:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:29 smithi027 ceph-mon[98970]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 388'13802 2024-08-29T15:38:29.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:29.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:38:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:38:29] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-29T15:38:29.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:29.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:29.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:29.428 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:29.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:29.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:29.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:29 smithi171 ceph-mon[90031]: 8.c starting backfill to osd.0 from (0'0,0'0] MAX to 388'14211 2024-08-29T15:38:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:29 smithi171 ceph-mon[90031]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 388'13802 2024-08-29T15:38:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:29.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:29.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:29.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:29.623 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:29.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:29.767 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:29.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:29.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:29.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:38:29.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:29.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:29.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:30.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:30.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:30.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:30.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:30.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:30.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:30.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:30.283 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:30 smithi027 ceph-mon[98970]: pgmap v1414: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2.1 MiB/s, 33 objects/s recovering 2024-08-29T15:38:30.284 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:30 smithi027 ceph-mon[93921]: pgmap v1414: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2.1 MiB/s, 33 objects/s recovering 2024-08-29T15:38:30.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:30.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:30.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:30.493 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:30 smithi171 ceph-mon[90031]: pgmap v1414: 99 pgs: 99 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2.1 MiB/s, 33 objects/s recovering 2024-08-29T15:38:30.522 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:30.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:30.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:30.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:30.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:30.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:30.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:30.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:30.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:30.820 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:38:30.820 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:38:30.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:30.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:30.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:31.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:31.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:31.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:31.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:31.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:31.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:31.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:31.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:31.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:31.355 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:31.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:31.457 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:31.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:31.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:31.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:31.631 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:31.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:31.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:31.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:31.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:31.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:31.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:31.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:31.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:31.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:32.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:32.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:38:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:32.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:32.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:32.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:32.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:32.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:32.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:32.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:32 smithi027 ceph-mon[93921]: pgmap v1415: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3844/92748 objects misplaced (4.145%); 2.4 MiB/s, 39 objects/s recovering 2024-08-29T15:38:32.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:32 smithi027 ceph-mon[98970]: pgmap v1415: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3844/92748 objects misplaced (4.145%); 2.4 MiB/s, 39 objects/s recovering 2024-08-29T15:38:32.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:32.367 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:38:32.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 0 1 2024-08-29T15:38:32.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:32.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:32.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:32.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:32 smithi171 ceph-mon[90031]: pgmap v1415: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3844/92748 objects misplaced (4.145%); 2.4 MiB/s, 39 objects/s recovering 2024-08-29T15:38:32.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:32.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:32.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:32.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:32.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:32.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:32.734 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:32.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:32.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:32.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:32.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:32.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:32.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:32.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:33.055 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:33.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:33.127 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:33.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:33.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:33.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:33.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:33.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:33.370 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:33.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:33.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:33.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:33.527 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:33.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:33.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:33.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:33.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:33.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:33.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:33.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:33.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:33.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:33.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:33.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:34.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:34.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:34.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:34.206 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:34.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:34.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:34.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:34.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:34.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:34.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:34.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:34.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:34 smithi171 ceph-mon[90031]: pgmap v1416: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 3844/90564 objects misplaced (4.245%); 339 KiB/s, 5 objects/s recovering 2024-08-29T15:38:34.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:34 smithi027 ceph-mon[98970]: pgmap v1416: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 3844/90564 objects misplaced (4.245%); 339 KiB/s, 5 objects/s recovering 2024-08-29T15:38:34.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:34 smithi027 ceph-mon[93921]: pgmap v1416: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 3844/90564 objects misplaced (4.245%); 339 KiB/s, 5 objects/s recovering 2024-08-29T15:38:34.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:34.645 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:34.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:34.722 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:34.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:34.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:34.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:34.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:34.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:34.931 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:34.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:35.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:35.075 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:35.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:35.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:35.205 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:38:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:35.206 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:35.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:35.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2833927472' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:35.211 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:35 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:35.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2833927472' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:35.211 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:35 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:35.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:35.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:35.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:35.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:35.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:35.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:35.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:35.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2833927472' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:35.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:35 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:35.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:35.629 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:35.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:35.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:35.771 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:35.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:35.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:35.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:35.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:35.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:35.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:36.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:36.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:36.137 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:36.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:36.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:36.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:36.316 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:36.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:36.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:36.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:36.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:36.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[93921]: pgmap v1417: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 2959/83670 objects misplaced (3.537%); 3.0 MiB/s, 48 objects/s recovering 2024-08-29T15:38:36.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:38:36.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:38:36.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:38:36.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[93921]: osdmap e392: 8 total, 8 up, 8 in 2024-08-29T15:38:36.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[98970]: pgmap v1417: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 2959/83670 objects misplaced (3.537%); 3.0 MiB/s, 48 objects/s recovering 2024-08-29T15:38:36.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:38:36.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:38:36.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:38:36.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:36 smithi027 ceph-mon[98970]: osdmap e392: 8 total, 8 up, 8 in 2024-08-29T15:38:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:36 smithi171 ceph-mon[90031]: pgmap v1417: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 2959/83670 objects misplaced (3.537%); 3.0 MiB/s, 48 objects/s recovering 2024-08-29T15:38:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:36 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:38:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:36 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:38:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:36 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:38:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:36 smithi171 ceph-mon[90031]: osdmap e392: 8 total, 8 up, 8 in 2024-08-29T15:38:36.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:36.611 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:36.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:36.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:36.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:36.741 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:36.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:36.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:36.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:36.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:36.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:36.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:37.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:37.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:37.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:37.245 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:37.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:37.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:37.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:37.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:37.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:37.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:37.558 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:37.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:37.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:37.695 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:37.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:37.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:37.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:37.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:37.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:37.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:38.075 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:38.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:38.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:38.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:38.246 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:38.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:38.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:38.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:38.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:38.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:38.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:38 smithi171 ceph-mon[90031]: pgmap v1419: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2959/78507 objects misplaced (3.769%); 2.7 MiB/s, 43 objects/s recovering 2024-08-29T15:38:38.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:38.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:38 smithi027 ceph-mon[98970]: pgmap v1419: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2959/78507 objects misplaced (3.769%); 2.7 MiB/s, 43 objects/s recovering 2024-08-29T15:38:38.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:38 smithi027 ceph-mon[93921]: pgmap v1419: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 2959/78507 objects misplaced (3.769%); 2.7 MiB/s, 43 objects/s recovering 2024-08-29T15:38:38.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:38.659 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:38.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:38.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:38.761 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:38.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:38.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:38.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:38.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:38.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:38.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:38.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:39.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:39.076 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:39.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:39.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:39.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:39.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:39.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:39.278 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:38:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:38:39] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-29T15:38:39.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:39.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:39.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:39.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:39.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:39.483 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:38:39.483 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:38:39.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:39.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:39.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:39.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:38:39.520 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:38:39.554 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:38:39.554 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:38:39.595 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:38:39.595 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:38:39.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:39.631 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:38:39.631 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:38:39.637 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:39.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:39.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:39.664 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:38:39.664 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:38:39.699 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:38:39.699 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:38:39.733 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:38:39.733 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:38:39.739 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:39.769 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:39.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:39.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:39.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:39.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:39.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:39.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:40.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:40.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:40.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:40.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:40.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:40.309 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:40.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:38:40.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:40.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:40.467 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:40 smithi027 ceph-mon[93921]: pgmap v1420: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 2959/76002 objects misplaced (3.893%); 2.4 MiB/s, 38 objects/s recovering 2024-08-29T15:38:40.467 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/922332123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:40.468 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:40 smithi027 ceph-mon[98970]: pgmap v1420: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 2959/76002 objects misplaced (3.893%); 2.4 MiB/s, 38 objects/s recovering 2024-08-29T15:38:40.468 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/922332123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:40 smithi171 ceph-mon[90031]: pgmap v1420: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 2959/76002 objects misplaced (3.893%); 2.4 MiB/s, 38 objects/s recovering 2024-08-29T15:38:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/922332123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:40.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:40.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:40.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:40.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:40.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:40.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:40.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:40.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:40.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:40.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:40.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:41.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:41.107 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:38:41.107 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:38:41.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:38:41.107 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:38:41.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.108 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110025 7.2 7:4fa9aa4d:::benchmark_data_smithi027_271924_object110024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:38:41.108 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.573623+0000", 2024-08-29T15:38:41.108 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 486.529436503, 2024-08-29T15:38:41.108 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29751161300000001, 2024-08-29T15:38:41.108 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.108 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110025 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.109 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573623+0000", 2024-08-29T15:38:41.110 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.110 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573623+0000", 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573624+0000", 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8599999999999998e-07 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573699+0000", 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5284000000000006e-05 2024-08-29T15:38:41.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573708+0000", 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8189999999999998e-06 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.573714+0000", 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5350000000000001e-06 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600325+0000", 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0266109 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600826+0000", 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00050131999999999998 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:38:41.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600960+0000", 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133497 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614386+0000", 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013426301999999999 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614495+0000", 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109065 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.870954+0000", 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25645894800000002 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.871081+0000", 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126985 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.871135+0000", 2024-08-29T15:38:41.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3471999999999999e-05 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110051 7.a 7:5df6ce9d:::benchmark_data_smithi027_271924_object110050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.600656+0000", 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 486.50240367399999, 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35938326700000001, 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.116 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110051 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600656+0000", 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600656+0000", 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600657+0000", 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1459999999999999e-06 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600683+0000", 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6390999999999999e-05 2024-08-29T15:38:41.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600694+0000", 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0438999999999999e-05 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.600701+0000", 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4240000000000004e-06 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614640+0000", 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013938259 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614772+0000", 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013251300000000001 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614917+0000", 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000145067 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622870+0000", 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079524539999999994 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626735+0000", 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038654359999999999 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.959899+0000", 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.333164397 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.959984+0000", 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4740000000000005e-05 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.960039+0000", 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5000999999999997e-05 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.122 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110064 7.a 7:52f01eaa:::benchmark_data_smithi027_271924_object110063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.614498+0000", 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 486.48856169999999, 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.359004622, 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.123 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110064 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614498+0000", 2024-08-29T15:38:41.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614498+0000", 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614499+0000", 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.266e-06 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614519+0000", 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0137999999999999e-05 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614531+0000", 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1411e-05 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.614538+0000", 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9879999999999998e-06 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618649+0000", 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041114539999999996 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618765+0000", 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115796 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618923+0000", 2024-08-29T15:38:41.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015772599999999999 2024-08-29T15:38:41.129 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.129 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.129 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.129 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622901+0000", 2024-08-29T15:38:41.129 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039787269999999996 2024-08-29T15:38:41.129 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626740+0000", 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038387320000000001 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973426+0000", 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34668595400000002 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.130 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973470+0000", 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4051999999999998e-05 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973503+0000", 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2378000000000001e-05 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.131 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110066 7.2 7:420ab7a3:::benchmark_data_smithi027_271924_object110065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.615195+0000", 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 486.48786488000002, 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35819247199999998, 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.132 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110066 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615195+0000", 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615195+0000", 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615196+0000", 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8299999999999995e-07 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.134 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615214+0000", 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8365000000000001e-05 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615225+0000", 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0729e-05 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.615231+0000", 2024-08-29T15:38:41.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7039999999999997e-06 2024-08-29T15:38:41.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618426+0000", 2024-08-29T15:38:41.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031946800000000001 2024-08-29T15:38:41.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.619142+0000", 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00071609799999999997 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.619460+0000", 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031777800000000001 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.629806+0000", 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010345896 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.881698+0000", 2024-08-29T15:38:41.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25189198099999999 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973276+0000", 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091578520999999996 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973339+0000", 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2898000000000007e-05 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973387+0000", 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7939000000000003e-05 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110068 7.2 7:4efeab30:::benchmark_data_smithi027_271924_object110067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.618779+0000", 2024-08-29T15:38:41.141 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 486.484280248, 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35482978100000001, 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110068 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618779+0000", 2024-08-29T15:38:41.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618779+0000", 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618781+0000", 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.217e-06 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618802+0000", 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0936000000000001e-05 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618813+0000", 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1375999999999999e-05 2024-08-29T15:38:41.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.618842+0000", 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.923e-05 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.623102+0000", 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042601360000000003 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.623259+0000", 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015687200000000001 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.623393+0000", 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013349799999999999 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.629814+0000", 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064215679999999999 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.881890+0000", 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25207546600000003 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973534+0000", 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091644728999999994 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973577+0000", 2024-08-29T15:38:41.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2110999999999997e-05 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973609+0000", 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2641999999999997e-05 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.148 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110074 7.2 7:47f6c0a9:::benchmark_data_smithi027_271924_object110073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.622690+0000", 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 486.48037000400001, 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35112648899999999, 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110074 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622690+0000", 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622690+0000", 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622691+0000", 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6500000000000001e-06 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622730+0000", 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9138999999999999e-05 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622742+0000", 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1659e-05 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.622756+0000", 2024-08-29T15:38:41.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4297000000000001e-05 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626764+0000", 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040075370000000003 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626931+0000", 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000166976 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:38:41.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.627109+0000", 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017861299999999999 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.629980+0000", 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028709130000000001 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.882057+0000", 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25207687000000001 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973744+0000", 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091687025000000005 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973785+0000", 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1072000000000002e-05 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.973816+0000", 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0738000000000003e-05 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:110076 7.3 7:c7ddcbee:::benchmark_data_smithi027_271924_object110075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e303)", 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:30:34.626845+0000", 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 486.47621511199998, 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33320111000000002, 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.155 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 110076 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626845+0000", 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626845+0000", 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626846+0000", 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.624e-06 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626872+0000", 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5880999999999998e-05 2024-08-29T15:38:41.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626886+0000", 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4426999999999999e-05 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626896+0000", 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1859999999999998e-06 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.626947+0000", 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1385999999999999e-05 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.627035+0000", 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7575000000000001e-05 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.627123+0000", 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8832000000000006e-05 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.630447+0000", 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033235719999999999 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.630990+0000", 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00054312800000000004 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.959956+0000", 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32896541400000001 2024-08-29T15:38:41.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.960007+0000", 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1001000000000002e-05 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:30:34.960046+0000", 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9084000000000001e-05 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:166125 7.2 7:41360a75:::benchmark_data_smithi027_271924_object49784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:01.176981+0000", 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.92607814799999, 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21275208700000001, 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166125 2024-08-29T15:38:41.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.176981+0000", 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.176981+0000", 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.176983+0000", 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.254e-06 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.176987+0000", 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2540000000000003e-06 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.176998+0000", 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1435e-05 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177006+0000", 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7170000000000008e-06 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177083+0000", 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7336000000000006e-05 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177247+0000", 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016390799999999999 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.177322+0000", 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4598000000000006e-05 2024-08-29T15:38:41.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.190323+0000", 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013000683000000001 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.194370+0000", 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040472260000000001 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.389626+0000", 2024-08-29T15:38:41.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19525631299999999 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.389692+0000", 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6279999999999996e-05 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:01.389734+0000", 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1083e-05 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185356 8.c 8:3e65ed30:::benchmark_data_smithi027_306999_object58577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 64.755439941000006, 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.954196238, 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185356 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.169 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347621+0000", 2024-08-29T15:38:41.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4959999999999999e-06 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347626+0000", 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7110000000000003e-06 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347636+0000", 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7690000000000008e-06 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347643+0000", 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9439999999999999e-06 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347708+0000", 2024-08-29T15:38:41.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5822999999999996e-05 2024-08-29T15:38:41.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347924+0000", 2024-08-29T15:38:41.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021531600000000001 2024-08-29T15:38:41.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:41.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:41.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:38:41.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.348044+0000", 2024-08-29T15:38:41.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120091 2024-08-29T15:38:41.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359802+0000", 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011758006 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359853+0000", 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1329999999999998e-05 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390576+0000", 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030722635000000002 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390589+0000", 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2792000000000001e-05 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085219+0000", 2024-08-29T15:38:41.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69463000399999997 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085233+0000", 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3964e-05 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.301754+0000", 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2165213420000001 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.301816+0000", 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2014999999999995e-05 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185365 8.c 8:34b703f9:::benchmark_data_smithi027_306999_object58586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:38:41.179 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 64.749197026999994, 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.948337904, 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185365 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353864+0000", 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.136e-06 2024-08-29T15:38:41.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353867+0000", 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4340000000000001e-06 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353876+0000", 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8980000000000002e-06 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353882+0000", 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9800000000000003e-06 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359882+0000", 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059996040000000004 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359992+0000", 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110373 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.360098+0000", 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010622499999999999 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390623+0000", 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030524273000000001 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390631+0000", 2024-08-29T15:38:41.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9530000000000005e-06 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085266+0000", 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69463474700000005 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085276+0000", 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3610000000000007e-06 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302132+0000", 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2168561900000001 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302200+0000", 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8730000000000001e-05 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.186 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185370 8.c 8:321ac8e5:::benchmark_data_smithi027_306999_object58591:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 64.743562866000005, 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.942814085, 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185370 2024-08-29T15:38:41.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359498+0000", 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7150000000000001e-06 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359504+0000", 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9279999999999998e-06 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359518+0000", 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3708e-05 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359526+0000", 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5100000000000001e-06 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.367931+0000", 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008405615 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.368066+0000", 2024-08-29T15:38:41.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013519199999999999 2024-08-29T15:38:41.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:38:41.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.368197+0000", 2024-08-29T15:38:41.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013018200000000001 2024-08-29T15:38:41.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390661+0000", 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022464378 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390669+0000", 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1960000000000008e-06 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085313+0000", 2024-08-29T15:38:41.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69464367800000004 2024-08-29T15:38:41.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085321+0000", 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0919999999999998e-06 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302245+0000", 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2169240729999999 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302311+0000", 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5817999999999999e-05 2024-08-29T15:38:41.194 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185373 8.c 8:3539b1da:::benchmark_data_smithi027_306999_object58594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 64.739847440000005, 2024-08-29T15:38:41.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9786454089999999, 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185373 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363214+0000", 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4559999999999999e-06 2024-08-29T15:38:41.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363221+0000", 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0700000000000001e-06 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363234+0000", 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3397999999999999e-05 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363244+0000", 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7629999999999995e-06 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390698+0000", 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027454427 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390706+0000", 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2250000000000003e-06 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085347+0000", 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69464127399999998 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085355+0000", 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3180000000000007e-06 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302347+0000", 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.216992074 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302464+0000", 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116505 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.303107+0000", 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064362000000000002 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329229+0000", 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.026121961999999999 2024-08-29T15:38:41.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329277+0000", 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7749e-05 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338695+0000", 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094183740000000002 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338857+0000", 2024-08-29T15:38:41.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016147700000000001 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.339098+0000", 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024068099999999999 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341858+0000", 2024-08-29T15:38:41.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002760036 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185396 8.c 8:37b7f43f:::benchmark_data_smithi027_306999_object58617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:38:41.205 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 64.694452287000004, 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9334865830000001, 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185396 2024-08-29T15:38:41.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408609+0000", 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.114e-06 2024-08-29T15:38:41.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408615+0000", 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9900000000000002e-06 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408631+0000", 2024-08-29T15:38:41.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5426000000000001e-05 2024-08-29T15:38:41.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408639+0000", 2024-08-29T15:38:41.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3499999999999997e-06 2024-08-29T15:38:41.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408716+0000", 2024-08-29T15:38:41.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6490999999999997e-05 2024-08-29T15:38:41.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408726+0000", 2024-08-29T15:38:41.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0570999999999999e-05 2024-08-29T15:38:41.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085379+0000", 2024-08-29T15:38:41.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.67665228200000005 2024-08-29T15:38:41.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085386+0000", 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8730000000000007e-06 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319619+0000", 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2342329700000001 2024-08-29T15:38:41.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319745+0000", 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125611 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:38:41.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319880+0000", 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134673 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329291+0000", 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094111300000000002 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329493+0000", 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020221700000000001 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338735+0000", 2024-08-29T15:38:41.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092423220000000007 2024-08-29T15:38:41.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341916+0000", 2024-08-29T15:38:41.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031804369999999999 2024-08-29T15:38:41.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341975+0000", 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9617000000000002e-05 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342094+0000", 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118509 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185406 8.c 8:388927f1:::benchmark_data_smithi027_306999_object58627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 64.674797991000005, 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9141273249999999, 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.220 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185406 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1699999999999997e-07 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428267+0000", 2024-08-29T15:38:41.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1339999999999997e-06 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428278+0000", 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0927999999999999e-05 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428283+0000", 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1510000000000003e-06 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428329+0000", 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6029000000000003e-05 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428334+0000", 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3979999999999998e-06 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085411+0000", 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.657076836 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085419+0000", 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.644e-06 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.323864+0000", 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2384456589999999 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.323988+0000", 2024-08-29T15:38:41.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012354099999999999 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.324118+0000", 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129987 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329297+0000", 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051789770000000004 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338774+0000", 2024-08-29T15:38:41.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094772769999999992 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338812+0000", 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7799000000000002e-05 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342228+0000", 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034166130000000002 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342282+0000", 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3149999999999998e-05 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342389+0000", 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107385 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.229 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210331 8.c 8:3aae235d:::benchmark_data_smithi027_306999_object83552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 31.079661354999999, 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.042995135, 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210331 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.230 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023400+0000", 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.875e-06 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023406+0000", 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5149999999999997e-06 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023422+0000", 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6733999999999999e-05 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023430+0000", 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7279999999999995e-06 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024594+0000", 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011642289999999999 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024726+0000", 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013213600000000001 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024866+0000", 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013915999999999999 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.033244+0000", 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008378155 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.034487+0000", 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012431300000000001 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.035958+0000", 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001471539 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.040150+0000", 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041912520000000003 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.040161+0000", 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1744e-05 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066310+0000", 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.02614841 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066393+0000", 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3528000000000004e-05 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210341 8.c 8:3b7d0476:::benchmark_data_smithi027_306999_object83562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 31.064284817000001, 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0358083629999999, 2024-08-29T15:38:41.237 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.238 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.238 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.238 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.238 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.238 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.239 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210341 2024-08-29T15:38:41.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.239 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038776+0000", 2024-08-29T15:38:41.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.409e-06 2024-08-29T15:38:41.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038781+0000", 2024-08-29T15:38:41.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5680000000000001e-06 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038791+0000", 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0149000000000001e-05 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038799+0000", 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8630000000000008e-06 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.041098+0000", 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022990480000000002 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.041106+0000", 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0080000000000002e-06 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066885+0000", 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0257787739999999 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066990+0000", 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010541500000000001 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.067139+0000", 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014869899999999999 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074258+0000", 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007119383 2024-08-29T15:38:41.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074420+0000", 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016190599999999999 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074489+0000", 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8998000000000006e-05 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074542+0000", 2024-08-29T15:38:41.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2697e-05 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074583+0000", 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1446000000000003e-05 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210361 8.c 8:36e2a08e:::benchmark_data_smithi027_306999_object83582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:38:41.246 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 31.042451693, 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.026884127, 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210361 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060609+0000", 2024-08-29T15:38:41.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5700000000000002e-07 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060613+0000", 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0419999999999996e-06 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060624+0000", 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0821e-05 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060630+0000", 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2890000000000003e-06 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060687+0000", 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7071000000000003e-05 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060695+0000", 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.605e-06 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070807+0000", 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.010112353 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070906+0000", 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8680999999999999e-05 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070983+0000", 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7495000000000003e-05 2024-08-29T15:38:41.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074476+0000", 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003492673 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074595+0000", 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119187 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087278+0000", 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012683589 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087441+0000", 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016204500000000001 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087492+0000", 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1518999999999999e-05 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.253 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210366 8.c 8:3bfc388b:::benchmark_data_smithi027_306999_object83587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 31.034103244000001, 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0187623729999999, 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.254 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210366 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:38:41.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068958+0000", 2024-08-29T15:38:41.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3519999999999999e-06 2024-08-29T15:38:41.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068962+0000", 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7690000000000004e-06 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068975+0000", 2024-08-29T15:38:41.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.217e-05 2024-08-29T15:38:41.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068982+0000", 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.069045+0000", 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2791000000000002e-05 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.069054+0000", 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5640000000000002e-06 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074624+0000", 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.005569739 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074808+0000", 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184026 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074951+0000", 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014334600000000001 2024-08-29T15:38:41.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087555+0000", 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012603130000000001 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087570+0000", 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5834000000000001e-05 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087625+0000", 2024-08-29T15:38:41.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4398999999999999e-05 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087681+0000", 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6036999999999999e-05 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087719+0000", 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7907999999999997e-05 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.262 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210405 8.c 8:3354f347:::benchmark_data_smithi027_306999_object83626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 30.990369907000002, 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98371990300000001, 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.263 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210405 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112691+0000", 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5740000000000001e-06 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112697+0000", 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5999999999999997e-06 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112714+0000", 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6898e-05 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112721+0000", 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6669999999999996e-06 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112782+0000", 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0148000000000001e-05 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112790+0000", 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1089999999999998e-06 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083315+0000", 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97052495400000005 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083455+0000", 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139925 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083591+0000", 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013614500000000001 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.091643+0000", 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080527510000000004 2024-08-29T15:38:41.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.095910+0000", 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042662530000000002 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.095950+0000", 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0290000000000002e-05 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096361+0000", 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041099000000000002 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096410+0000", 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8599e-05 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.270 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210417 8.c 8:351de282:::benchmark_data_smithi027_306999_object83638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 30.975794914000002, 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97409499899999996, 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210417 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:38:41.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:38:41.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127266+0000", 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.291e-06 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127271+0000", 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3689999999999998e-06 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127281+0000", 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9320000000000008e-06 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127289+0000", 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5730000000000002e-06 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127353+0000", 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4462999999999996e-05 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127362+0000", 2024-08-29T15:38:41.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1330000000000005e-06 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087778+0000", 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96041514900000002 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087926+0000", 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014825999999999999 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:38:41.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.088065+0000", 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013917199999999999 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096459+0000", 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008393546 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101179+0000", 2024-08-29T15:38:41.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047199649999999996 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101267+0000", 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8187000000000001e-05 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101319+0000", 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2432999999999998e-05 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:38:41.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101360+0000", 2024-08-29T15:38:41.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0525999999999998e-05 2024-08-29T15:38:41.278 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.278 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.278 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.278 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:38:41.278 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:38:41.278 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:38:41.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:41.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:41.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:41.417 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:41.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:41.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:41.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:41.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:41.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:41.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:41.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:41.730 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:41.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:41.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:41.832 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:41.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:41.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:41.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:41.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:41.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:41.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:41.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:42.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:42.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:42.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:42.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:42.214 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:42.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:42.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:42.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:42.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:42.394 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:42.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:42.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:42.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:42 smithi171 ceph-mon[90031]: pgmap v1421: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 2139/68769 objects misplaced (3.110%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:38:42.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:42.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:42.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:42 smithi027 ceph-mon[93921]: pgmap v1421: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 2139/68769 objects misplaced (3.110%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:38:42.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:42 smithi027 ceph-mon[98970]: pgmap v1421: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 2139/68769 objects misplaced (3.110%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:38:42.635 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:42.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:42.670 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:42.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:42.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:42.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:38:42.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:42.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:42.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:42.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:42.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:42.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:43.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:43.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:38:43.125 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:43.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:43.129 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:38:43.217 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.0 to 1 (10000) 2024-08-29T15:38:43.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:43.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:43.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:43.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:43.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:43.410 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:43.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:43.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/305648667' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:38:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:38:43.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:43.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:43.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/305648667' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:38:43.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:38:43.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/305648667' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:38:43.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:38:43.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:43.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:43.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:43.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:43.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:43.819 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:43.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:43.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:43.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:43.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:43.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:44.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:44.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:44.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:44.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:44.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:44.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:44.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:44.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:44.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:44.388 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:44.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:44.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:44 smithi171 ceph-mon[90031]: pgmap v1422: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 2139/65562 objects misplaced (3.263%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:38:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:44 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:38:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:44 smithi171 ceph-mon[90031]: osdmap e393: 8 total, 8 up, 8 in 2024-08-29T15:38:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:44.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:44.530 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:44.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[93921]: pgmap v1422: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 2139/65562 objects misplaced (3.263%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[93921]: osdmap e393: 8 total, 8 up, 8 in 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[98970]: pgmap v1422: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 2139/65562 objects misplaced (3.263%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[98970]: osdmap e393: 8 total, 8 up, 8 in 2024-08-29T15:38:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:44.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:44.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:44.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:44.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:44.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:44.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:44.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:44.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:44.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:44.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:44.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:45.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:45.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:45.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:45.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:45.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:45.264 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:45.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:45.281 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:38:45.281 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:38:45.281 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:38:45.281 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:38:45.307 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:38:45.308 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:38:45.365 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:45.404 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:45.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:45.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:38:45.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:45.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:45.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:45.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:45.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:45.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:45.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:45.784 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:45.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:45.828 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:45.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:45.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:45.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:45.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:46.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:38:46.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:46.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:46.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:46.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:46.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:46.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:46.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:46.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:46.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:46.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:46 smithi171 ceph-mon[90031]: pgmap v1424: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 1403/58692 objects misplaced (2.390%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:38:46.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:46.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:46.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:46.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:46.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:46 smithi027 ceph-mon[98970]: pgmap v1424: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 1403/58692 objects misplaced (2.390%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:38:46.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:46 smithi027 ceph-mon[93921]: pgmap v1424: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 1403/58692 objects misplaced (2.390%); 5.1 MiB/s, 81 objects/s recovering 2024-08-29T15:38:46.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:46.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:46.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:46.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:46.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:46.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:46.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:46.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:46.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:47.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:47.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:47.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:47.106 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:38:47.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:47.192 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:47.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:47.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:47.272 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:38:47.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:47.316 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:47.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:47.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:47.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:47.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 1 1 2024-08-29T15:38:47.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:47.450 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:47.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:47.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:47 smithi171 ceph-mon[90031]: osdmap e394: 8 total, 8 up, 8 in 2024-08-29T15:38:47.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3425803416' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:38:47.511 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:47 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:38:47.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:47 smithi027 ceph-mon[98970]: osdmap e394: 8 total, 8 up, 8 in 2024-08-29T15:38:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3425803416' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:38:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:47 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:38:47.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:47 smithi027 ceph-mon[93921]: osdmap e394: 8 total, 8 up, 8 in 2024-08-29T15:38:47.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3425803416' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:38:47.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:47 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:38:47.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:47.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:47.635 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:47.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:47.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:47.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:47.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:47.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:47.887 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:47.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:47.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:48.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:48.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:48.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:48.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:48.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:48.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:48.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:48.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:48.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:48.322 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:48.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:48.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:48.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[98970]: pgmap v1426: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 1403/53667 objects misplaced (2.614%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:38:48.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:48.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:38:48.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[98970]: osdmap e395: 8 total, 8 up, 8 in 2024-08-29T15:38:48.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[93921]: pgmap v1426: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 1403/53667 objects misplaced (2.614%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:38:48.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:48.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:38:48.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:48 smithi027 ceph-mon[93921]: osdmap e395: 8 total, 8 up, 8 in 2024-08-29T15:38:48.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:48.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:48.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:48 smithi171 ceph-mon[90031]: pgmap v1426: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 1403/53667 objects misplaced (2.614%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:38:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:48 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:38:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:48 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:38:48.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:48 smithi171 ceph-mon[90031]: osdmap e395: 8 total, 8 up, 8 in 2024-08-29T15:38:48.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:48.637 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:48.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:48.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:48.739 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:48.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:48.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:48.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:48.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:48.918 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:48.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:48.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:49.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:49.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:49.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:49.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:49.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:49.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:49.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:49.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:38:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:38:49] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-29T15:38:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:49.347 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:49.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:49.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:49.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:49.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:49.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:49.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:49.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:49.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:49.672 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:38:49.672 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:38:49.672 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:38:49.672 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:38:49.672 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:38:49.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:49.760 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:49.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:49.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:49.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:49.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:50.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:50.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:50.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:50.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:50.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:38:50.225 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:50.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:50.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:50.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:50.359 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:50.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:50.380 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:50 smithi171 ceph-mon[90031]: pgmap v1428: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1403/51729 objects misplaced (2.712%); 4.0 MiB/s, 63 objects/s recovering 2024-08-29T15:38:50.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:50.518 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:50.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:50.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:50.572 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:50 smithi027 ceph-mon[93921]: pgmap v1428: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1403/51729 objects misplaced (2.712%); 4.0 MiB/s, 63 objects/s recovering 2024-08-29T15:38:50.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:50 smithi027 ceph-mon[98970]: pgmap v1428: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 1.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1403/51729 objects misplaced (2.712%); 4.0 MiB/s, 63 objects/s recovering 2024-08-29T15:38:50.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:50.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:50.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:50.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:50.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:50.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:50.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:50.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:50.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:50.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:50.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:50.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:51.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:51.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:51.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:51.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:51.106 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:38:51.106 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:38:51.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:38:51.157 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:38:51.157 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:38:51.180 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:51.188 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:38:51.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:51.195 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:38:51.195 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:38:51.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:51.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:51.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:51.237 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:38:51.238 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:38:51.272 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:38:51.272 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:38:51.317 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:38:51.317 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:38:51.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/586284697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:51.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/586284697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:51.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:51.345 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:38:51.345 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:38:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:51.351 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:51.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:51.374 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:38:51.374 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:38:51.389 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/586284697' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:38:51.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:51.488 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:51.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:51.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:51.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:51.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:51.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:51.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:51.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:51.764 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:51.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:51.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:51.913 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:51.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:52.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:52.060 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:52.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:52.122 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:38:52.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:52.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.202 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:52.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:52.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:52.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:52.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:52.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:52.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:52.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:52 smithi171 ceph-mon[90031]: pgmap v1429: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 961 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 299/45984 objects misplaced (0.650%); 11 MiB/s, 172 objects/s recovering 2024-08-29T15:38:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:52 smithi171 ceph-mon[90031]: osdmap e396: 8 total, 8 up, 8 in 2024-08-29T15:38:52.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:52 smithi027 ceph-mon[93921]: pgmap v1429: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 961 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 299/45984 objects misplaced (0.650%); 11 MiB/s, 172 objects/s recovering 2024-08-29T15:38:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:52 smithi027 ceph-mon[93921]: osdmap e396: 8 total, 8 up, 8 in 2024-08-29T15:38:52.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:52 smithi027 ceph-mon[98970]: pgmap v1429: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 961 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 299/45984 objects misplaced (0.650%); 11 MiB/s, 172 objects/s recovering 2024-08-29T15:38:52.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:52 smithi027 ceph-mon[98970]: osdmap e396: 8 total, 8 up, 8 in 2024-08-29T15:38:52.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:52.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:52.702 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:52.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:52.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:52.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:52.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:52.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:52.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:52.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:53.052 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:53.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:53.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:53.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:53.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:53.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:53.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:53.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:53.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:53.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:53.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:53.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:53.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:53.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:53.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:53.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:53.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:53.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:53.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:53.771 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:53.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:53.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:53.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:53.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:53.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:54.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:54.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:54.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:54.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:54.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:54.223 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:54 smithi171 ceph-mon[90031]: pgmap v1431: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 901 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 299/43101 objects misplaced (0.694%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:38:54.223 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:54 smithi171 ceph-mon[90031]: osdmap e397: 8 total, 8 up, 8 in 2024-08-29T15:38:54.223 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:54.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:54.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:54.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:54.424 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:54.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:54.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:54.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:54 smithi027 ceph-mon[98970]: pgmap v1431: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 901 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 299/43101 objects misplaced (0.694%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:38:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:54 smithi027 ceph-mon[98970]: osdmap e397: 8 total, 8 up, 8 in 2024-08-29T15:38:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:54.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:54 smithi027 ceph-mon[93921]: pgmap v1431: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 901 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 299/43101 objects misplaced (0.694%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:38:54.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:54 smithi027 ceph-mon[93921]: osdmap e397: 8 total, 8 up, 8 in 2024-08-29T15:38:54.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:54.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:54.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:54.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:54.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:54.709 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:38:54.709 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:38:54.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:54.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:54.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:54.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:54.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:54.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:54.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:54.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:55.023 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:55.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:55.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:55.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:55.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:55.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:55.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:55.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:55.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:55.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:55.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:55.351 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:55.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:38:55.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:55.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:55.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:55.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:55.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:55.598 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:55.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:55.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:55.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:55.720 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:38:55.737 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:55.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:55.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:55.839 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:38:55.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:55.867 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:55.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:38:55.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:55.999 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:56.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:56.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:56.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:56.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:56.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:56.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:56.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:56.428 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:56 smithi171 ceph-mon[90031]: pgmap v1433: 99 pgs: 99 active+clean; 787 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 10 MiB/s, 160 objects/s recovering 2024-08-29T15:38:56.519 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:56.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:56 smithi027 ceph-mon[98970]: pgmap v1433: 99 pgs: 99 active+clean; 787 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 10 MiB/s, 160 objects/s recovering 2024-08-29T15:38:56.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:56 smithi027 ceph-mon[93921]: pgmap v1433: 99 pgs: 99 active+clean; 787 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 10 MiB/s, 160 objects/s recovering 2024-08-29T15:38:56.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:56.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:56.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:56.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:56.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:56.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:56.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:57.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:57.104 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:57.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:57.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:57.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:57.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:57.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:57.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:57.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:57.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:57.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:57.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:38:57.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:57.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:57.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:57.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:57.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:57.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:57.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:57.702 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:38:57.702 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:38:57.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:57.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:57.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:57.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:57.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:57.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:58.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:38:58.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:58.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:38:58.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:38:58.201 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:58.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:58.231 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:58.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:58.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:58 smithi171 ceph-mon[90031]: pgmap v1434: 99 pgs: 99 active+clean; 670 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 9.6 MiB/s, 153 objects/s recovering 2024-08-29T15:38:58.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:58.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:58.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:58.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:58.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:58.484 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:58 smithi027 ceph-mon[98970]: pgmap v1434: 99 pgs: 99 active+clean; 670 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 9.6 MiB/s, 153 objects/s recovering 2024-08-29T15:38:58.485 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:58 smithi027 ceph-mon[93921]: pgmap v1434: 99 pgs: 99 active+clean; 670 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 9.6 MiB/s, 153 objects/s recovering 2024-08-29T15:38:58.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:58.492 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:58.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:58.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:38:58.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:38:58.628 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:58.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:38:58.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:58.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:58.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:58.862 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:38:58.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:58.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:58.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:38:58.945 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:38:59.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:59.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:59.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:59.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:59.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:59.207 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:38:59] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-08-29T15:38:59.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:59.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:59.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:38:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:59.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:59.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:38:59.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:59.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:59.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:59.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:59.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:38:59.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:59.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:59.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:38:59.753 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:38:59.772 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:38:59.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:38:59.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:59.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:38:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[93921]: pgmap v1435: 99 pgs: 99 active+clean; 605 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 2.2 MiB/s, 35 objects/s recovering 2024-08-29T15:38:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4263857571' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:59.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:38:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:38:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[98970]: pgmap v1435: 99 pgs: 99 active+clean; 605 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 2.2 MiB/s, 35 objects/s recovering 2024-08-29T15:38:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4263857571' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:38:59 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:38:59.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:38:59.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:38:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:38:59.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:59 smithi171 ceph-mon[90031]: pgmap v1435: 99 pgs: 99 active+clean; 605 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 2.2 MiB/s, 35 objects/s recovering 2024-08-29T15:39:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4263857571' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:38:59 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:00.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:00.101 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:00.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:00.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:00.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:00.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:00.332 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:00.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:00.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:00.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:00.482 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:00.517 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:00.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:00.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:00.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:00.660 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:00.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:00.746 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.1 to 1 (10000) 2024-08-29T15:39:00.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:00.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:00.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:00.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:00.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:00.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:00.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:00 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:00 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:00 smithi171 ceph-mon[90031]: osdmap e398: 8 total, 8 up, 8 in 2024-08-29T15:39:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3854880067' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:39:01.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:01.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[98970]: osdmap e398: 8 total, 8 up, 8 in 2024-08-29T15:39:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3854880067' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:39:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[93921]: osdmap e398: 8 total, 8 up, 8 in 2024-08-29T15:39:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3854880067' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:39:01.159 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:01.181 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:01.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:01.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:01.305 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:01.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:01.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:01.427 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:39:01.427 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:39:01.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:01.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:01.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:01.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:01.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:01.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:01.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:01.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:01.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:01.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:01.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:01.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3854880067' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:39:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:01 smithi171 ceph-mon[90031]: osdmap e399: 8 total, 8 up, 8 in 2024-08-29T15:39:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:01 smithi171 ceph-mon[90031]: pgmap v1438: 99 pgs: 99 active+clean; 505 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2.2 MiB/s, 35 objects/s recovering 2024-08-29T15:39:02.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:01 smithi171 ceph-mon[90031]: osdmap e400: 8 total, 8 up, 8 in 2024-08-29T15:39:02.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3854880067' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:39:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[98970]: osdmap e399: 8 total, 8 up, 8 in 2024-08-29T15:39:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[98970]: pgmap v1438: 99 pgs: 99 active+clean; 505 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2.2 MiB/s, 35 objects/s recovering 2024-08-29T15:39:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[98970]: osdmap e400: 8 total, 8 up, 8 in 2024-08-29T15:39:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3854880067' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:39:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[93921]: osdmap e399: 8 total, 8 up, 8 in 2024-08-29T15:39:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[93921]: pgmap v1438: 99 pgs: 99 active+clean; 505 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 2.2 MiB/s, 35 objects/s recovering 2024-08-29T15:39:02.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:01 smithi027 ceph-mon[93921]: osdmap e400: 8 total, 8 up, 8 in 2024-08-29T15:39:02.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:39:02.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.081 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:02.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:02.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:02.206 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:02.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:02.277 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:39:02.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:02.331 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:02.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:02.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:02.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:02.583 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:02.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:02.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:02.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:02.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:02.833 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:02.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:02.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:02.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:02.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:03.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:03.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:03.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:03.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:03.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:39:03.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:03.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:03.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:03.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:03.338 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:03.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:03.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:03.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:03.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:03.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:03.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:03.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:03.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:03.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:03.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 2 1 2024-08-29T15:39:03.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:03.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:03.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:03.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[93921]: 8.3 starting backfill to osd.1 from (350'4327,381'13095] MIN to 398'15446 2024-08-29T15:39:03.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[93921]: 8.b starting backfill to osd.1 from (351'5702,378'12370] MIN to 398'15276 2024-08-29T15:39:03.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[93921]: osdmap e401: 8 total, 8 up, 8 in 2024-08-29T15:39:03.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[93921]: pgmap v1441: 99 pgs: 1 peering, 98 active+clean; 434 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-08-29T15:39:03.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:03.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:03.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[98970]: 8.3 starting backfill to osd.1 from (350'4327,381'13095] MIN to 398'15446 2024-08-29T15:39:03.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[98970]: 8.b starting backfill to osd.1 from (351'5702,378'12370] MIN to 398'15276 2024-08-29T15:39:03.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[98970]: osdmap e401: 8 total, 8 up, 8 in 2024-08-29T15:39:03.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:03 smithi027 ceph-mon[98970]: pgmap v1441: 99 pgs: 1 peering, 98 active+clean; 434 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-08-29T15:39:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:03.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:03.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:03.951 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:03.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:03.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:03 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:03.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:03 smithi171 ceph-mon[90031]: 8.3 starting backfill to osd.1 from (350'4327,381'13095] MIN to 398'15446 2024-08-29T15:39:03.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:03 smithi171 ceph-mon[90031]: 8.b starting backfill to osd.1 from (351'5702,378'12370] MIN to 398'15276 2024-08-29T15:39:03.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:03 smithi171 ceph-mon[90031]: osdmap e401: 8 total, 8 up, 8 in 2024-08-29T15:39:03.969 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:03 smithi171 ceph-mon[90031]: pgmap v1441: 99 pgs: 1 peering, 98 active+clean; 434 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-08-29T15:39:03.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:03.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:04.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:04.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:04.105 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:04.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:04.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:04.257 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:39:04.257 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:39:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:04.300 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:04.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:04.349 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:04.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:04.420 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:39:04.420 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:39:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:04.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:04.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:04.467 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:04.476 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:39:04.476 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:39:04.511 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:39:04.512 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:39:04.547 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:39:04.547 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:39:04.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:04.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:04.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:04.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:04.583 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:39:04.583 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:39:04.620 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:39:04.621 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:39:04.653 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:39:04.653 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:39:04.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:04.688 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:39:04.688 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:39:04.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:04.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:04.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:04.865 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:04.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:04.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:04.901 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4261379740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:04.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:39:04.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:05.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:05.045 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4261379740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:05.045 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4261379740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:05.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:05.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:05.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:05.244 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:05.272 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:39:05.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:05.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.454 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:05.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:05.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:05.596 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:05.698 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:05.728 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:05.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:05.830 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:05.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:05.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:05.984 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:05.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:05.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:06.086 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:06.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:06.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:06.219 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:06.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:06.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:06.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:06.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:06.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:06.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:06.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:06.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:06.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:06 smithi027 ceph-mon[93921]: pgmap v1442: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 peering, 96 active+clean; 362 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 529/17241 objects misplaced (3.068%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:39:06.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:06.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:06 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:06.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:06 smithi027 ceph-mon[98970]: pgmap v1442: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 peering, 96 active+clean; 362 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 529/17241 objects misplaced (3.068%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:39:06.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:06.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:06 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:06.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:06.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:06.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:06.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:06.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:06.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:06.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:06.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:06.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:06.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:06 smithi171 ceph-mon[90031]: pgmap v1442: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 peering, 96 active+clean; 362 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 529/17241 objects misplaced (3.068%); 1.3 MiB/s, 21 objects/s recovering 2024-08-29T15:39:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:07.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:06 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:07.061 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:07.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:07.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:07.212 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:07.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:07.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:07.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:39:07.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:07.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:07.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:07.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:07.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:07.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:07.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:07.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:07.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:07.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:07.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:07.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:07.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:08.005 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:07 smithi171 ceph-mon[90031]: osdmap e402: 8 total, 8 up, 8 in 2024-08-29T15:39:08.005 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:07 smithi171 ceph-mon[90031]: pgmap v1444: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 290 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 529/13767 objects misplaced (3.843%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:39:08.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:08.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:07 smithi027 ceph-mon[93921]: osdmap e402: 8 total, 8 up, 8 in 2024-08-29T15:39:08.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:07 smithi027 ceph-mon[93921]: pgmap v1444: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 290 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 529/13767 objects misplaced (3.843%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:39:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:07 smithi027 ceph-mon[98970]: osdmap e402: 8 total, 8 up, 8 in 2024-08-29T15:39:08.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:07 smithi027 ceph-mon[98970]: pgmap v1444: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 290 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 529/13767 objects misplaced (3.843%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:39:08.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:08.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:08.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:08.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:08.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:08.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:08.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:08.466 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:08.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:08.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:08.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:08.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:08.692 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:08.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:08.782 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:39:08.815 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:08 smithi027 ceph-mon[93921]: osdmap e403: 8 total, 8 up, 8 in 2024-08-29T15:39:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2827510195' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:39:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:08 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:39:08.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:08 smithi027 ceph-mon[98970]: osdmap e403: 8 total, 8 up, 8 in 2024-08-29T15:39:08.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2827510195' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:39:08.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:08 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:39:08.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:08.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:08.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:08.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:09.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:09.094 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:08 smithi171 ceph-mon[90031]: osdmap e403: 8 total, 8 up, 8 in 2024-08-29T15:39:09.094 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2827510195' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:39:09.094 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:08 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:39:09.122 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:09.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:09.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:09.256 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:39:09] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-08-29T15:39:09.267 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:09.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:09.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:09.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:09.416 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:09.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:09.451 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:39:09.451 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:39:09.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.451 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:251171 8.d e404/344 8:b0d4b011:::benchmark_data_smithi027_306999_object124392:head v 404'15739, mlcod=404'15736)", 2024-08-29T15:39:09.451 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:39:09.400731+0000", 2024-08-29T15:39:09.451 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0031040669999999999, 2024-08-29T15:39:09.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031353349999999999, 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.400731+0000", 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:09.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.400731+0000", 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.400730+0000", 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.400738+0000", 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3319999999999992e-06 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.400754+0000", 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6246999999999999e-05 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.400763+0000", 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1119999999999996e-06 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.400983+0000", 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00022057000000000001 2024-08-29T15:39:09.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.401033+0000", 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9838e-05 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:251177 8.5 e404/344 8:ad087a41:::benchmark_data_smithi027_306999_object124398:head v 404'15797, mlcod=404'15795)", 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:39:09.403239+0000", 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00059652499999999998, 2024-08-29T15:39:09.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00068741099999999997, 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403239+0000", 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403239+0000", 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:09.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403237+0000", 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403246+0000", 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1109999999999997e-06 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:09.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403266+0000", 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9358999999999999e-05 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403274+0000", 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8820000000000001e-06 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403445+0000", 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017129800000000001 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403464+0000", 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8862999999999999e-05 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:251174 8.c e404/396 8:39e70e15:::benchmark_data_smithi027_306999_object124395:head v 404'16032, mlcod=404'16030)", 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:39:09.403295+0000", 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00054045299999999998, 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00068356099999999996, 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:09.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403295+0000", 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403295+0000", 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403294+0000", 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.460 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403299+0000", 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3919999999999996e-06 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403305+0000", 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5400000000000001e-06 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403309+0000", 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9820000000000002e-06 2024-08-29T15:39:09.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403408+0000", 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9226000000000002e-05 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:09.403428+0000", 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9885999999999999e-05 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:09.462 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:39:09.463 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 3 2024-08-29T15:39:09.463 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:39:09.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:09.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:09.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:09.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:09.706 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:09.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:09.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:09.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:09.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:09.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:09.951 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:09.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:09.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:10.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:10.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:39:10.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[98970]: osdmap e404: 8 total, 8 up, 8 in 2024-08-29T15:39:10.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[98970]: pgmap v1447: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 248 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 529/11781 objects misplaced (4.490%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:39:10.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:10.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:39:10.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[93921]: osdmap e404: 8 total, 8 up, 8 in 2024-08-29T15:39:10.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:09 smithi027 ceph-mon[93921]: pgmap v1447: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 248 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 529/11781 objects misplaced (4.490%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:39:10.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:10.101 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:09 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:10.101 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:39:10.101 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:09 smithi171 ceph-mon[90031]: osdmap e404: 8 total, 8 up, 8 in 2024-08-29T15:39:10.101 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:09 smithi171 ceph-mon[90031]: pgmap v1447: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 248 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 529/11781 objects misplaced (4.490%); 1.1 MiB/s, 18 objects/s recovering 2024-08-29T15:39:10.130 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:10.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:10.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:10.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:39:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:10.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:10.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:10.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:10.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:10.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:10.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:10.550 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:10.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:10.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:10.583 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:39:10.652 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:10.669 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:10.699 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:10.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:10.794 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.2 to 1 (10000) 2024-08-29T15:39:10.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:10.853 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:10.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:10.910 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/917796681' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:39:10.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:10.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.000 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:11.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:11.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/917796681' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:39:11.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/917796681' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:39:11.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:11.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:11.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:11.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.282 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:11.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:11.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:11.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:11.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:11.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:11.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:11.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:11.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:11.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:11.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:11.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:11.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:11.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:11.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:12.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/917796681' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:39:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[98970]: osdmap e405: 8 total, 8 up, 8 in 2024-08-29T15:39:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[98970]: pgmap v1449: 99 pgs: 99 active+clean; 165 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-29T15:39:12.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[98970]: osdmap e406: 8 total, 8 up, 8 in 2024-08-29T15:39:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/917796681' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:39:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[93921]: osdmap e405: 8 total, 8 up, 8 in 2024-08-29T15:39:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[93921]: pgmap v1449: 99 pgs: 99 active+clean; 165 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-29T15:39:12.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:11 smithi027 ceph-mon[93921]: osdmap e406: 8 total, 8 up, 8 in 2024-08-29T15:39:12.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:12.078 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/917796681' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:39:12.079 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:11 smithi171 ceph-mon[90031]: osdmap e405: 8 total, 8 up, 8 in 2024-08-29T15:39:12.079 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:11 smithi171 ceph-mon[90031]: pgmap v1449: 99 pgs: 99 active+clean; 165 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-29T15:39:12.079 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:11 smithi171 ceph-mon[90031]: osdmap e406: 8 total, 8 up, 8 in 2024-08-29T15:39:12.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:12.150 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Removed 126776 objects 2024-08-29T15:39:12.150 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Clean up completed and total clean up time :159.395 2024-08-29T15:39:12.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:12.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:12.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:12.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:12.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:12.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 3 1 2024-08-29T15:39:12.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:12.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:39:12.590 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:12.598 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:39:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:39:12.599 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:39:12.600 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:39:12.600 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:39:12.614 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:12.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:12.699 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-08-29T15:39:12.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-08-29T15:39:12.716 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:12.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:12.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:12.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:12.861 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:12.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:12.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:12.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:13.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:13.108 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:13.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:13.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:13.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:39:13.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:13.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:13.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:13.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:13.477 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:39:13.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:13.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:13.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:13.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:13.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:13.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:13.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:13.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:13.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:13.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:13.932 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:13 smithi171 ceph-mon[90031]: osdmap e407: 8 total, 8 up, 8 in 2024-08-29T15:39:13.932 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:13 smithi171 ceph-mon[90031]: pgmap v1452: 99 pgs: 99 active+clean; 107 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-29T15:39:13.932 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:13 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:13.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:13.982 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:13.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:13.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:13 smithi027 ceph-mon[93921]: osdmap e407: 8 total, 8 up, 8 in 2024-08-29T15:39:14.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:13 smithi027 ceph-mon[93921]: pgmap v1452: 99 pgs: 99 active+clean; 107 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-29T15:39:14.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:13 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:13 smithi027 ceph-mon[98970]: osdmap e407: 8 total, 8 up, 8 in 2024-08-29T15:39:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:13 smithi027 ceph-mon[98970]: pgmap v1452: 99 pgs: 99 active+clean; 107 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-08-29T15:39:14.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:13 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:14.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:14.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:14.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:14.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:14.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:14.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:14.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:14.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:14.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:14.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:14.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:14.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:14.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:14.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:14.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:14.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:14.858 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:14.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:14.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:14.972 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:14 smithi171 ceph-mon[90031]: osdmap e408: 8 total, 8 up, 8 in 2024-08-29T15:39:14.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:14.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:14 smithi027 ceph-mon[93921]: osdmap e408: 8 total, 8 up, 8 in 2024-08-29T15:39:15.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:14 smithi027 ceph-mon[98970]: osdmap e408: 8 total, 8 up, 8 in 2024-08-29T15:39:15.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:15.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:15.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:15.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:15.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:15.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:15.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:15.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:15.241 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:39:15.241 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:39:15.241 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:39:15.241 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:39:15.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.241 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:181363 7.c 7:374819ae:::benchmark_data_smithi027_271924_object65022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:31.121438+0000", 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 404.11693981500002, 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12429384, 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 181363 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.242 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121438+0000", 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121438+0000", 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121437+0000", 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121444+0000", 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.613e-06 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121455+0000", 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1379999999999999e-05 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.121460+0000", 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1959999999999999e-06 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.127088+0000", 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056279779999999996 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.127714+0000", 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00062543899999999999 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.196681+0000", 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.068967428999999997 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.196801+0000", 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012019 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.196954+0000", 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000152941 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.209476+0000", 2024-08-29T15:39:15.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012521947 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.209485+0000", 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.0669999999999998e-06 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.243529+0000", 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034044267000000003 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.243589+0000", 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9919e-05 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:31.245732+0000", 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021428319999999999 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.248 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:183981 7.c 7:37b24d37:::benchmark_data_smithi027_271924_object67640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:35.790310+0000", 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 399.44806776500002, 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.142854971, 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 183981 2024-08-29T15:39:15.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790310+0000", 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790310+0000", 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790309+0000", 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790316+0000", 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1199999999999996e-06 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790331+0000", 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.449e-05 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.790338+0000", 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4070000000000004e-06 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.807843+0000", 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01750521 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.807891+0000", 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7818999999999997e-05 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.854100+0000", 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.046209159 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.854159+0000", 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9163999999999998e-05 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.854266+0000", 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106943 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.864760+0000", 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010493593000000001 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.867054+0000", 2024-08-29T15:39:15.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022938609999999999 2024-08-29T15:39:15.255 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.255 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.255 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.931263+0000", 2024-08-29T15:39:15.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.064209242 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.931297+0000", 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3673000000000001e-05 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:35.933165+0000", 2024-08-29T15:39:15.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001868682 2024-08-29T15:39:15.257 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.257 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.257 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:185906 7.c 7:392ad6ee:::benchmark_data_smithi027_271924_object69565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.000130+0000", 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 396.23824838399997, 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12084690200000001, 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 185906 2024-08-29T15:39:15.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000130+0000", 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000130+0000", 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000128+0000", 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000135+0000", 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5490000000000003e-06 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000148+0000", 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3553999999999999e-05 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.000155+0000", 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8970000000000002e-06 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.007287+0000", 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0071321889999999997 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.007830+0000", 2024-08-29T15:39:15.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00054281100000000003 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.062993+0000", 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.055163104999999997 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.063115+0000", 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000121966 2024-08-29T15:39:15.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.063280+0000", 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000164699 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.079375+0000", 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016094799999999999 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.091591+0000", 2024-08-29T15:39:15.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012216401999999999 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.118746+0000", 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027154729999999998 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.118845+0000", 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9225e-05 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.120977+0000", 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002131473 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:186004 7.c 7:3c022e11:::benchmark_data_smithi027_271924_object69663:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:39:15.265 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.201273+0000", 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 396.03710554999998, 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13242031000000001, 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 186004 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201273+0000", 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201273+0000", 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201271+0000", 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201278+0000", 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1969999999999999e-06 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201293+0000", 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4674e-05 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.201300+0000", 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8210000000000004e-06 2024-08-29T15:39:15.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.210716+0000", 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0094159840000000005 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.210774+0000", 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8548999999999998e-05 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.291659+0000", 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.080884510000000007 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.291736+0000", 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7298000000000004e-05 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.291830+0000", 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4399000000000002e-05 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.270 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.299628+0000", 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00779745 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.299638+0000", 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0093e-05 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.331393+0000", 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031755115 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.331453+0000", 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9846999999999998e-05 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.333693+0000", 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022401209999999999 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.273 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:186059 7.c 7:3f2c4a83:::benchmark_data_smithi027_271924_object69718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:39:15.273 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.308963+0000", 2024-08-29T15:39:15.273 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 395.92941531999998, 2024-08-29T15:39:15.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.120818908, 2024-08-29T15:39:15.273 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 186059 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308963+0000", 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308963+0000", 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308962+0000", 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308965+0000", 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0070000000000002e-06 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308972+0000", 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9040000000000003e-06 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.308977+0000", 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0950000000000003e-06 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.314312+0000", 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0053341250000000003 2024-08-29T15:39:15.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.314833+0000", 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00052143199999999995 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.357405+0000", 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042572461999999998 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.357472+0000", 2024-08-29T15:39:15.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6929999999999998e-05 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.357559+0000", 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6762999999999996e-05 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363038+0000", 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0054784350000000002 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363810+0000", 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00077244800000000001 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.427824+0000", 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.064014038999999995 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.427901+0000", 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7435999999999995e-05 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.429782+0000", 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001880271 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:186072 7.c 7:3e3c8b97:::benchmark_data_smithi027_271924_object69731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:39.337065+0000", 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 395.90131333800002, 2024-08-29T15:39:15.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12659251599999999, 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 186072 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337065+0000", 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337065+0000", 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337064+0000", 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337067+0000", 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3470000000000001e-06 2024-08-29T15:39:15.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337072+0000", 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.797e-06 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.337076+0000", 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9319999999999998e-06 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363071+0000", 2024-08-29T15:39:15.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025995829000000002 2024-08-29T15:39:15.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.363792+0000", 2024-08-29T15:39:15.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000720238 2024-08-29T15:39:15.285 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.435485+0000", 2024-08-29T15:39:15.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.071693713000000006 2024-08-29T15:39:15.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.286 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.286 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.435577+0000", 2024-08-29T15:39:15.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2023999999999998e-05 2024-08-29T15:39:15.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.287 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.287 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.435713+0000", 2024-08-29T15:39:15.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000135227 2024-08-29T15:39:15.287 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.287 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.288 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.447400+0000", 2024-08-29T15:39:15.288 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011686999999999999 2024-08-29T15:39:15.288 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.288 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.288 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.461301+0000", 2024-08-29T15:39:15.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013901939 2024-08-29T15:39:15.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.461311+0000", 2024-08-29T15:39:15.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2390000000000008e-06 2024-08-29T15:39:15.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.461395+0000", 2024-08-29T15:39:15.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3772999999999999e-05 2024-08-29T15:39:15.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.291 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:39.463657+0000", 2024-08-29T15:39:15.291 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002262834 2024-08-29T15:39:15.291 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.292 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.292 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.292 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.292 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.292 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193430 7.c 7:3582ae89:::benchmark_data_smithi027_271924_object77089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:39:15.293 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.570981+0000", 2024-08-29T15:39:15.293 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 382.66739735800002, 2024-08-29T15:39:15.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12667557800000001, 2024-08-29T15:39:15.293 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.293 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.294 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.294 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.294 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.294 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193430 2024-08-29T15:39:15.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.294 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570981+0000", 2024-08-29T15:39:15.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.296 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.296 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570981+0000", 2024-08-29T15:39:15.296 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.296 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.296 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.297 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.297 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570979+0000", 2024-08-29T15:39:15.297 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.297 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.297 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.297 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.570986+0000", 2024-08-29T15:39:15.298 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2980000000000003e-06 2024-08-29T15:39:15.298 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.298 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.298 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.298 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.571004+0000", 2024-08-29T15:39:15.299 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7306e-05 2024-08-29T15:39:15.299 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.299 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.299 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.299 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.571012+0000", 2024-08-29T15:39:15.299 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4449999999999996e-06 2024-08-29T15:39:15.300 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.300 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.300 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.300 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.576859+0000", 2024-08-29T15:39:15.300 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0058465210000000004 2024-08-29T15:39:15.301 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.301 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.301 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.301 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.577402+0000", 2024-08-29T15:39:15.301 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00054362699999999998 2024-08-29T15:39:15.301 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.302 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.302 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.302 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.630804+0000", 2024-08-29T15:39:15.302 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.053401692000000001 2024-08-29T15:39:15.302 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.302 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.303 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.303 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.630900+0000", 2024-08-29T15:39:15.303 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5832e-05 2024-08-29T15:39:15.303 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.303 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.303 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.304 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.631043+0000", 2024-08-29T15:39:15.304 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014345800000000001 2024-08-29T15:39:15.304 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.304 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.304 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.304 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.642639+0000", 2024-08-29T15:39:15.305 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011596245999999999 2024-08-29T15:39:15.305 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.305 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.305 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.305 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.642694+0000", 2024-08-29T15:39:15.306 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4666999999999997e-05 2024-08-29T15:39:15.306 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.306 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.306 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.306 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.695939+0000", 2024-08-29T15:39:15.306 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.053244646 2024-08-29T15:39:15.307 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.307 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.307 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.307 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.695995+0000", 2024-08-29T15:39:15.307 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6042999999999999e-05 2024-08-29T15:39:15.307 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.308 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.308 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.308 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.697656+0000", 2024-08-29T15:39:15.308 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016616160000000001 2024-08-29T15:39:15.308 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.309 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.309 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.309 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.309 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.309 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193477 7.c 7:3b7390b6:::benchmark_data_smithi027_271924_object77136:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:39:15.309 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.654544+0000", 2024-08-29T15:39:15.309 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 382.58383410499999, 2024-08-29T15:39:15.310 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.121148282, 2024-08-29T15:39:15.310 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.310 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.310 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.310 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.311 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.311 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193477 2024-08-29T15:39:15.311 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.311 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.311 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.311 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.312 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654544+0000", 2024-08-29T15:39:15.312 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.312 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.312 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.312 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.312 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654544+0000", 2024-08-29T15:39:15.313 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.313 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.313 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.313 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.313 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654543+0000", 2024-08-29T15:39:15.313 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:39:15.314 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.314 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.314 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.314 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654548+0000", 2024-08-29T15:39:15.314 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7149999999999997e-06 2024-08-29T15:39:15.314 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.315 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.315 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.315 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654558+0000", 2024-08-29T15:39:15.315 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6169999999999996e-06 2024-08-29T15:39:15.315 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.315 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.316 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.316 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654563+0000", 2024-08-29T15:39:15.316 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0019999999999997e-06 2024-08-29T15:39:15.316 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.316 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.316 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.317 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.654596+0000", 2024-08-29T15:39:15.317 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3031000000000003e-05 2024-08-29T15:39:15.317 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.317 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.317 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.317 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.655082+0000", 2024-08-29T15:39:15.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00048625900000000001 2024-08-29T15:39:15.318 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.318 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.318 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.318 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.707741+0000", 2024-08-29T15:39:15.318 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.052658917999999999 2024-08-29T15:39:15.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.319 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.319 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.319 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.707808+0000", 2024-08-29T15:39:15.319 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7167000000000003e-05 2024-08-29T15:39:15.319 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.320 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.320 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.320 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.707901+0000", 2024-08-29T15:39:15.320 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2989e-05 2024-08-29T15:39:15.320 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.321 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720410+0000", 2024-08-29T15:39:15.321 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012508594 2024-08-29T15:39:15.321 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.321 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.321 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.322 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.738610+0000", 2024-08-29T15:39:15.322 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018200213 2024-08-29T15:39:15.322 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.322 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.322 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.773726+0000", 2024-08-29T15:39:15.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035116742999999999 2024-08-29T15:39:15.323 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.323 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.323 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.323 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.773782+0000", 2024-08-29T15:39:15.323 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5612000000000002e-05 2024-08-29T15:39:15.324 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.324 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.324 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.324 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775692+0000", 2024-08-29T15:39:15.324 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001910246 2024-08-29T15:39:15.324 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.324 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193483 7.c 7:32868c1b:::benchmark_data_smithi027_271924_object77142:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.656968+0000", 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 382.58141021599999, 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.122521351, 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.325 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193483 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656968+0000", 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656968+0000", 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.326 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656967+0000", 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656972+0000", 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.417e-06 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656985+0000", 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3152999999999999e-05 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.327 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.656994+0000", 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.867e-06 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.657562+0000", 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000567302 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.658150+0000", 2024-08-29T15:39:15.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00058876200000000005 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720458+0000", 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062307686000000001 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720562+0000", 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000104076 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.720715+0000", 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015248700000000001 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.732477+0000", 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011761984 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.738685+0000", 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0062084130000000003 2024-08-29T15:39:15.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.777488+0000", 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038803294000000002 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.777537+0000", 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8822000000000001e-05 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.779489+0000", 2024-08-29T15:39:15.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00195211 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193484 7.c 7:3eb7034a:::benchmark_data_smithi027_271924_object77143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.659073+0000", 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 382.57930486399999, 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12237255900000001, 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.332 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193484 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659073+0000", 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659073+0000", 2024-08-29T15:39:15.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659072+0000", 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659081+0000", 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4440000000000008e-06 2024-08-29T15:39:15.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659094+0000", 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3171e-05 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659101+0000", 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3669999999999999e-06 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659152+0000", 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0470999999999998e-05 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.659730+0000", 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00057846700000000004 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.726851+0000", 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.067120432999999993 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.726917+0000", 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6421999999999995e-05 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.727028+0000", 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111351 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.737340+0000", 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010311694 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.738725+0000", 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001384758 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.779539+0000", 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040814069000000001 2024-08-29T15:39:15.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:15.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.779586+0000", 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6828e-05 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.781446+0000", 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018601900000000001 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.341 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193536 7.c 7:3cbbe8ac:::benchmark_data_smithi027_271924_object77195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.775463+0000", 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 382.46291564400002, 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.13050738200000001, 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193536 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775463+0000", 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775463+0000", 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775461+0000", 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775467+0000", 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7910000000000001e-06 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775478+0000", 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1139e-05 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.775484+0000", 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.127e-06 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.796483+0000", 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020999522999999999 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.797785+0000", 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013018859999999999 2024-08-29T15:39:15.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.837846+0000", 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.040060428000000002 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.837925+0000", 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9503000000000001e-05 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.838042+0000", 2024-08-29T15:39:15.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011702299999999999 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.848706+0000", 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010664237 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.864670+0000", 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015963537999999999 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.904123+0000", 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039452698000000001 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.904159+0000", 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6752999999999999e-05 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.905970+0000", 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0018105020000000001 2024-08-29T15:39:15.348 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:193545 7.c 7:3ea5e9f6:::benchmark_data_smithi027_271924_object77204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:32:52.783647+0000", 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 382.454730934, 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.124103535, 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.349 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 193545 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783647+0000", 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783647+0000", 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783646+0000", 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783654+0000", 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.055e-06 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783669+0000", 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5077e-05 2024-08-29T15:39:15.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.783676+0000", 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6040000000000004e-06 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.797824+0000", 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014147730000000001 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.802988+0000", 2024-08-29T15:39:15.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005164302 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.845319+0000", 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042330378000000002 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.845395+0000", 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6686000000000004e-05 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.845534+0000", 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013818500000000001 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.864676+0000", 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019142306000000001 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.864727+0000", 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.126e-05 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.906028+0000", 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041300986999999997 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.906081+0000", 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2518000000000001e-05 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:32:52.907751+0000", 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016700230000000001 2024-08-29T15:39:15.355 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:206192 7.c 7:3fe59ff6:::benchmark_data_smithi027_271924_object89851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:13.748591+0000", 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 361.48978718799998, 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12247730699999999, 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.356 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 206192 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748591+0000", 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748591+0000", 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.357 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748589+0000", 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748597+0000", 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1589999999999993e-06 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748611+0000", 2024-08-29T15:39:15.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4183e-05 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748628+0000", 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6101999999999999e-05 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748657+0000", 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9575999999999999e-05 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.748720+0000", 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2524e-05 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.839050+0000", 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.090330220000000003 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.839156+0000", 2024-08-29T15:39:15.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106018 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.839317+0000", 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016095700000000001 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.862968+0000", 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.023650786 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.361 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.862980+0000", 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2702e-05 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.869517+0000", 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.006536549 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.869557+0000", 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9705e-05 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:13.871068+0000", 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001511667 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209048 7.c 7:3c105219:::benchmark_data_smithi027_271924_object92707:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.323326+0000", 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.91505224600002, 2024-08-29T15:39:15.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.151098872, 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209048 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323326+0000", 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323326+0000", 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323325+0000", 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323329+0000", 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6449999999999998e-06 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323337+0000", 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3329999999999997e-06 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.323343+0000", 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.733e-06 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.328539+0000", 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051963679999999998 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.328600+0000", 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1122000000000002e-05 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.357721+0000", 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029121030999999999 2024-08-29T15:39:15.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.357828+0000", 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010610799999999999 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.357977+0000", 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014897700000000001 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.368487+0000", 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010509977 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.368496+0000", 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6970000000000005e-06 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.472833+0000", 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.104336963 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.472883+0000", 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9663999999999997e-05 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.474425+0000", 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015419279999999999 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209068 7.c 7:30640b78:::benchmark_data_smithi027_271924_object92727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.370 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.344896+0000", 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.89348205200002, 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.130916859, 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209068 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344896+0000", 2024-08-29T15:39:15.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344896+0000", 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344895+0000", 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344900+0000", 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2320000000000001e-06 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344914+0000", 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4229e-05 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.344921+0000", 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7499999999999997e-06 2024-08-29T15:39:15.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.345970+0000", 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0010485729999999999 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.346631+0000", 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00066114899999999998 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.364287+0000", 2024-08-29T15:39:15.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017656505999999999 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.364390+0000", 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010242399999999999 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.364535+0000", 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0001449 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.374351+0000", 2024-08-29T15:39:15.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0098159440000000001 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.374372+0000", 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1821e-05 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.474493+0000", 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.100120244 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.474548+0000", 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4885000000000001e-05 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.475813+0000", 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001265399 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209071 7.c 7:3a2a109c:::benchmark_data_smithi027_271924_object92730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.377 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.351037+0000", 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.88734160400003, 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15185913200000001, 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209071 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351037+0000", 2024-08-29T15:39:15.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351037+0000", 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351034+0000", 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351041+0000", 2024-08-29T15:39:15.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0230000000000004e-06 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351057+0000", 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5682999999999999e-05 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351065+0000", 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7800000000000001e-06 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.351216+0000", 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015084399999999999 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.355370+0000", 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004154189 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.462040+0000", 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.10666985399999999 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.462107+0000", 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7359000000000003e-05 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.462197+0000", 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9567000000000005e-05 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.475879+0000", 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013681858 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.475890+0000", 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1669e-05 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501444+0000", 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025554132 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501518+0000", 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4066999999999994e-05 2024-08-29T15:39:15.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.502896+0000", 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001377234 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209088 7.c 7:30b45f45:::benchmark_data_smithi027_271924_object92747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.407525+0000", 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.83085314800002, 2024-08-29T15:39:15.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.17467528700000001, 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209088 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407525+0000", 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407525+0000", 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407523+0000", 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407532+0000", 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8449999999999992e-06 2024-08-29T15:39:15.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407551+0000", 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9578000000000001e-05 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407560+0000", 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3950000000000001e-06 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407619+0000", 2024-08-29T15:39:15.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8794e-05 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.407695+0000", 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5962000000000002e-05 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.507558+0000", 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.099863482000000003 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.507663+0000", 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010514999999999999 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.507823+0000", 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015936600000000001 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.518533+0000", 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010709984000000001 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.518596+0000", 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3349999999999995e-05 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.580676+0000", 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.062079955999999999 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.580732+0000", 2024-08-29T15:39:15.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6558000000000003e-05 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.582200+0000", 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014678110000000001 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209145 7.c 7:3ba8fe40:::benchmark_data_smithi027_271924_object92804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.501713+0000", 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.736665426, 2024-08-29T15:39:15.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12807848699999999, 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209145 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501713+0000", 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501713+0000", 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501711+0000", 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501718+0000", 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0380000000000002e-06 2024-08-29T15:39:15.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501730+0000", 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2211999999999999e-05 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.501736+0000", 2024-08-29T15:39:15.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9660000000000001e-06 2024-08-29T15:39:15.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.504637+0000", 2024-08-29T15:39:15.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002900812 2024-08-29T15:39:15.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.505279+0000", 2024-08-29T15:39:15.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00064199000000000005 2024-08-29T15:39:15.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.576672+0000", 2024-08-29T15:39:15.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.071392707 2024-08-29T15:39:15.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.397 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.576750+0000", 2024-08-29T15:39:15.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8045000000000002e-05 2024-08-29T15:39:15.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.576841+0000", 2024-08-29T15:39:15.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1044000000000004e-05 2024-08-29T15:39:15.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.589018+0000", 2024-08-29T15:39:15.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012176910000000001 2024-08-29T15:39:15.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.589075+0000", 2024-08-29T15:39:15.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7438999999999997e-05 2024-08-29T15:39:15.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.628573+0000", 2024-08-29T15:39:15.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039498097000000003 2024-08-29T15:39:15.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.628615+0000", 2024-08-29T15:39:15.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1423e-05 2024-08-29T15:39:15.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.629791+0000", 2024-08-29T15:39:15.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0011766649999999999 2024-08-29T15:39:15.403 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.403 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.403 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.404 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:209157 7.c 7:36d544a5:::benchmark_data_smithi027_271924_object92816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.404 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:18.517313+0000", 2024-08-29T15:39:15.404 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 356.72106529299998, 2024-08-29T15:39:15.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12589757600000001, 2024-08-29T15:39:15.404 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.404 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.405 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.405 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.405 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.405 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 209157 2024-08-29T15:39:15.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.406 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517313+0000", 2024-08-29T15:39:15.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517313+0000", 2024-08-29T15:39:15.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517311+0000", 2024-08-29T15:39:15.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:39:15.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517318+0000", 2024-08-29T15:39:15.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6669999999999999e-06 2024-08-29T15:39:15.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517335+0000", 2024-08-29T15:39:15.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7294000000000001e-05 2024-08-29T15:39:15.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.517343+0000", 2024-08-29T15:39:15.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9719999999999992e-06 2024-08-29T15:39:15.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.518677+0000", 2024-08-29T15:39:15.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0013335179999999999 2024-08-29T15:39:15.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for degraded object", 2024-08-29T15:39:15.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.519253+0000", 2024-08-29T15:39:15.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00057647500000000001 2024-08-29T15:39:15.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.584808+0000", 2024-08-29T15:39:15.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.065554772999999997 2024-08-29T15:39:15.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.584931+0000", 2024-08-29T15:39:15.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000123538 2024-08-29T15:39:15.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.585071+0000", 2024-08-29T15:39:15.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000139469 2024-08-29T15:39:15.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.595031+0000", 2024-08-29T15:39:15.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.009960666 2024-08-29T15:39:15.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.595041+0000", 2024-08-29T15:39:15.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.431e-06 2024-08-29T15:39:15.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.642176+0000", 2024-08-29T15:39:15.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.047135336 2024-08-29T15:39:15.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.642206+0000", 2024-08-29T15:39:15.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9872000000000001e-05 2024-08-29T15:39:15.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:18.643210+0000", 2024-08-29T15:39:15.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00100435 2024-08-29T15:39:15.419 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.419 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.420 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.420 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.36273.0:212171 7.c 7:338e7bd7:::benchmark_data_smithi027_271924_object95830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e333)", 2024-08-29T15:39:15.420 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:33:23.410992+0000", 2024-08-29T15:39:15.421 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 351.82738665199997, 2024-08-29T15:39:15.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.12099120100000001, 2024-08-29T15:39:15.421 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:39:15.421 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:39:15.421 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:39:15.421 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.36273", 2024-08-29T15:39:15.422 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:39:15.422 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 212171 2024-08-29T15:39:15.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.422 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:39:15.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:39:15.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410992+0000", 2024-08-29T15:39:15.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.424 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:39:15.424 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410992+0000", 2024-08-29T15:39:15.424 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:39:15.424 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.424 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:39:15.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410990+0000", 2024-08-29T15:39:15.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:39:15.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:39:15.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.410996+0000", 2024-08-29T15:39:15.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3119999999999999e-06 2024-08-29T15:39:15.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:39:15.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.411008+0000", 2024-08-29T15:39:15.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1187e-05 2024-08-29T15:39:15.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:39:15.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.411013+0000", 2024-08-29T15:39:15.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8420000000000004e-06 2024-08-29T15:39:15.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:39:15.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.419818+0000", 2024-08-29T15:39:15.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0088056160000000005 2024-08-29T15:39:15.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:39:15.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.419915+0000", 2024-08-29T15:39:15.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7058000000000005e-05 2024-08-29T15:39:15.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:39:15.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.420032+0000", 2024-08-29T15:39:15.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117299 2024-08-29T15:39:15.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:39:15.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.426619+0000", 2024-08-29T15:39:15.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0065867820000000002 2024-08-29T15:39:15.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.427738+0000", 2024-08-29T15:39:15.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001119117 2024-08-29T15:39:15.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:39:15.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.530575+0000", 2024-08-29T15:39:15.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.102836512 2024-08-29T15:39:15.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:39:15.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.530628+0000", 2024-08-29T15:39:15.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2670000000000002e-05 2024-08-29T15:39:15.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:39:15.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:39:15.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:39:15.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:33:23.531983+0000", 2024-08-29T15:39:15.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001355141 2024-08-29T15:39:15.435 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.435 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.435 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.435 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:39:15.435 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:39:15.435 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:39:15.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:15.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:15.529 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:15.534 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:15.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:15.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:15.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:15.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:15.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:15.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:15.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:15.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:15.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:15.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:15.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:15.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:15.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:15.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:15 smithi171 ceph-mon[90031]: osdmap e409: 8 total, 8 up, 8 in 2024-08-29T15:39:16.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:15 smithi171 ceph-mon[90031]: pgmap v1455: 99 pgs: 1 peering, 98 active+clean; 38 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 145 B/s, 0 objects/s recovering 2024-08-29T15:39:16.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:15 smithi027 ceph-mon[98970]: osdmap e409: 8 total, 8 up, 8 in 2024-08-29T15:39:16.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:15 smithi027 ceph-mon[98970]: pgmap v1455: 99 pgs: 1 peering, 98 active+clean; 38 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 145 B/s, 0 objects/s recovering 2024-08-29T15:39:16.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:15 smithi027 ceph-mon[93921]: osdmap e409: 8 total, 8 up, 8 in 2024-08-29T15:39:16.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:15 smithi027 ceph-mon[93921]: pgmap v1455: 99 pgs: 1 peering, 98 active+clean; 38 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 145 B/s, 0 objects/s recovering 2024-08-29T15:39:16.075 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:16.177 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:16.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:16.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:16.299 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:16.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:16.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:39:16.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:16.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:16.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:16.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:16.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:16.610 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:39:16.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:16.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:16.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:16.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:16.819 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:16.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:16.949 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:16.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:16.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:17.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:17.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:17.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:17.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:17.201 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:17.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:17.303 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:17.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:17.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:17.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:17.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:17.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:17.556 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:17.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:17.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:17.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:17.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:17.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:17.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:17.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:17.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:17.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:17.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:18.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:18.060 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:17 smithi027 ceph-mon[93921]: pgmap v1456: 99 pgs: 1 peering, 98 active+clean; 7.3 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 113 B/s, 0 objects/s recovering 2024-08-29T15:39:18.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:17 smithi027 ceph-mon[98970]: pgmap v1456: 99 pgs: 1 peering, 98 active+clean; 7.3 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 113 B/s, 0 objects/s recovering 2024-08-29T15:39:18.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:18.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:18.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:18.249 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:17 smithi171 ceph-mon[90031]: pgmap v1456: 99 pgs: 1 peering, 98 active+clean; 7.3 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 113 B/s, 0 objects/s recovering 2024-08-29T15:39:18.249 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:39:18.250 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:39:18.250 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:39:18.250 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:39:18.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:18.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:18.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:18.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:18.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:18.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:18.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:18.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:18.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:18.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:18.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:18.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:18.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:18.850 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:39:18.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:18.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:18.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:18.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:19.049 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:39:19.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:19.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:19.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:19.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:19.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:19.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:39:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:39:19] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.51.0" 2024-08-29T15:39:19.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:19.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:19.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:19.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:19.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:19.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:19.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:19.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:19.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:19.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:19.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:19.933 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:19.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:19.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:19.991 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.3 to 1 (10000) 2024-08-29T15:39:19.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:19 smithi027 ceph-mon[93921]: pgmap v1457: 99 pgs: 1 peering, 98 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 94 B/s, 0 objects/s recovering 2024-08-29T15:39:19.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/571431013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:39:20.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:20.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:20.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:20.162 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:20 smithi171 ceph-mon[90031]: pgmap v1457: 99 pgs: 1 peering, 98 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 94 B/s, 0 objects/s recovering 2024-08-29T15:39:20.162 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:20 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/571431013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:39:20.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:20.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:20.280 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:20 smithi027 ceph-mon[98970]: pgmap v1457: 99 pgs: 1 peering, 98 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 94 B/s, 0 objects/s recovering 2024-08-29T15:39:20.280 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:20 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/571431013' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:39:20.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:20.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:20.429 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:20.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:20.508 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:20.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:20.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:20.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:20.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:20.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:20.761 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:39:20.761 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:39:20.774 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:39:20.774 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:39:20.775 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:39:20.775 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:39:20.775 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:39:20.780 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:39:20.781 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:39:20.798 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:39:20.798 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:39:20.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:20.814 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:39:20.814 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:39:20.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:20.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:20.831 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:39:20.831 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:39:20.847 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:39:20.847 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:39:20.864 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:39:20.864 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:39:20.881 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:39:20.881 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:39:20.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:20.951 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:20.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:21.053 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:21.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:21.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:21.197 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:21.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:21.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/571431013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:39:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: osdmap e410: 8 total, 8 up, 8 in 2024-08-29T15:39:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:39:21.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:39:21.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.269 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3059588693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:21.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/571431013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:39:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: osdmap e410: 8 total, 8 up, 8 in 2024-08-29T15:39:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3059588693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/571431013' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: osdmap e410: 8 total, 8 up, 8 in 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:39:21.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:39:21.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:21.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3059588693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:21.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:21.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:21.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:21.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:39:21.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:21.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:21.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:21.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:21.641 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:39:21.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:21.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:21.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:21.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.816 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:21.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:21.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:21.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:21.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:22 smithi171 ceph-mon[90031]: pgmap v1459: 99 pgs: 99 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 133 op/s 2024-08-29T15:39:22.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:22 smithi171 ceph-mon[90031]: osdmap e411: 8 total, 8 up, 8 in 2024-08-29T15:39:22.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:22 smithi171 ceph-mon[90031]: osdmap e412: 8 total, 8 up, 8 in 2024-08-29T15:39:22.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:22.064 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:22.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:22.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.190 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:22.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:22.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:22.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:22.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:22 smithi027 ceph-mon[98970]: pgmap v1459: 99 pgs: 99 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 133 op/s 2024-08-29T15:39:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:22 smithi027 ceph-mon[98970]: osdmap e411: 8 total, 8 up, 8 in 2024-08-29T15:39:22.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:22 smithi027 ceph-mon[98970]: osdmap e412: 8 total, 8 up, 8 in 2024-08-29T15:39:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:22 smithi027 ceph-mon[93921]: pgmap v1459: 99 pgs: 99 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 133 op/s 2024-08-29T15:39:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:22 smithi027 ceph-mon[93921]: osdmap e411: 8 total, 8 up, 8 in 2024-08-29T15:39:22.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:22 smithi027 ceph-mon[93921]: osdmap e412: 8 total, 8 up, 8 in 2024-08-29T15:39:22.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:22.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:22.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:22.561 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:22.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:22.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:22.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:22.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:22.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 4 1 2024-08-29T15:39:22.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:22.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:22.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:22.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:22.936 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:22.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:22.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:23.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:23.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:23.150 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:23 smithi171 ceph-mon[90031]: 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 406'15624 2024-08-29T15:39:23.151 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:23 smithi171 ceph-mon[90031]: osdmap e413: 8 total, 8 up, 8 in 2024-08-29T15:39:23.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:23.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:23.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:23.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:23.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:23.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:23 smithi027 ceph-mon[98970]: 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 406'15624 2024-08-29T15:39:23.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:23 smithi027 ceph-mon[98970]: osdmap e413: 8 total, 8 up, 8 in 2024-08-29T15:39:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:23 smithi027 ceph-mon[93921]: 8.b starting backfill to osd.3 from (0'0,0'0] MAX to 406'15624 2024-08-29T15:39:23.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:23 smithi027 ceph-mon[93921]: osdmap e413: 8 total, 8 up, 8 in 2024-08-29T15:39:23.392 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:39:23.392 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:39:23.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:23.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:23.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.560 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:23.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:23.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:23.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:23.785 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:23.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:23.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:23.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:23.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:23.931 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:23.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:24.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:24.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:24.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:24.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:39:24.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:39:24.123 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_3' removed 2024-08-29T15:39:24.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:24.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:24.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:24.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:24.383 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[98970]: pgmap v1463: 99 pgs: 1 remapped+peering, 98 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:24.383 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[98970]: osdmap e414: 8 total, 8 up, 8 in 2024-08-29T15:39:24.384 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/426514868' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:39:24.384 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:39:24.384 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[93921]: pgmap v1463: 99 pgs: 1 remapped+peering, 98 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:24.384 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[93921]: osdmap e414: 8 total, 8 up, 8 in 2024-08-29T15:39:24.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/426514868' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:39:24.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:24 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:39:24.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:24.431 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:24 smithi171 ceph-mon[90031]: pgmap v1463: 99 pgs: 1 remapped+peering, 98 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:24.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:24 smithi171 ceph-mon[90031]: osdmap e414: 8 total, 8 up, 8 in 2024-08-29T15:39:24.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/426514868' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:39:24.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:24 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:39:24.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:24.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:24.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:24.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:24.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:24.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:24.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:24.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:24.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:24.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:24.987 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-29T15:39:24.988 INFO:tasks.radosbench:Beginning radosbench... 2024-08-29T15:39:24.989 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-08-29T15:39:24.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-08-29T15:39:25.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:25.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:25.063 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:25.063 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:25.064 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:25.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:25.180 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:25.275 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:25 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:39:25.275 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:25 smithi027 ceph-mon[93921]: osdmap e415: 8 total, 8 up, 8 in 2024-08-29T15:39:25.275 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:25 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:39:25.275 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:25 smithi027 ceph-mon[98970]: osdmap e415: 8 total, 8 up, 8 in 2024-08-29T15:39:25.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:25.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:25.406 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:25.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:25 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:39:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:25 smithi171 ceph-mon[90031]: osdmap e415: 8 total, 8 up, 8 in 2024-08-29T15:39:25.530 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:25.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:25.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:25.653 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:25.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:25.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.798 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:25.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:25.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:25.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:25.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:26.024 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:26.046 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:26.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:26.141 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:26 smithi027 ceph-mon[93921]: pgmap v1466: 83 pgs: 1 remapped+peering, 82 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:26.141 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:26 smithi027 ceph-mon[98970]: pgmap v1466: 83 pgs: 1 remapped+peering, 82 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:26.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:26.170 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:26.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:26.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:26.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:26.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:26.420 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:26.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:26 smithi171 ceph-mon[90031]: pgmap v1466: 83 pgs: 1 remapped+peering, 82 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:26.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:26.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:26.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:26.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:26.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:26.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:26.773 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:26.797 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:26.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:26.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:26.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:26.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:27.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:27.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:27.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:27.174 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:27.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:27.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:27.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:27.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:27.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:27.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:27.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:27.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:27.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:27.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:27.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:27.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:27.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:27.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:28.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:28.058 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:28.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:28.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:28.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:28.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:28.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:28.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:28.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:28.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:28 smithi027 ceph-mon[98970]: pgmap v1467: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:28 smithi027 ceph-mon[93921]: pgmap v1467: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:28.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:28.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:28 smithi171 ceph-mon[90031]: pgmap v1467: 83 pgs: 83 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:28.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:28.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:28.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:28.688 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:28.789 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:28.812 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:28.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:28.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:28.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:28.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:28.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:29.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:29.065 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:29.142 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:39:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:39:29] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.51.0" 2024-08-29T15:39:29.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:29.189 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:29.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:29.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:29.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:29.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:29.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:29.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:29.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:29.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:29.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:29.656 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:39:29.656 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:39:29.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:29.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:29.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:29.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:29.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:29.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:29.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:30.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:30.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:30.149 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:39:30.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:30.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:30.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:30 smithi027 ceph-mon[93921]: pgmap v1468: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:30.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3970196926' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:30.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:30 smithi027 ceph-mon[98970]: pgmap v1468: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3970196926' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:30.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:30.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:39:30.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:30.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:30.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:30.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:30.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:30 smithi171 ceph-mon[90031]: pgmap v1468: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3970196926' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:30.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:30.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:30.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:30.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:30.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:30.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:30.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:30.932 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:30.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:30.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:31.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:31.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:31.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:31.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:31.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:31.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:31.329 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:31.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:31.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:39:31.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:39:31.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:31.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[98970]: osdmap e416: 8 total, 8 up, 8 in 2024-08-29T15:39:31.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-08-29T15:39:31.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:39:31.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:39:31.379 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:31.379 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[93921]: osdmap e416: 8 total, 8 up, 8 in 2024-08-29T15:39:31.379 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-08-29T15:39:31.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:31.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:31.495 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:39:31.495 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:39:31.498 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:39:31.499 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:39:31.517 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:39:31.517 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:39:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:31 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:39:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:31 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:39:31.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:31.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:31 smithi171 ceph-mon[90031]: osdmap e416: 8 total, 8 up, 8 in 2024-08-29T15:39:31.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-08-29T15:39:31.533 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:39:31.533 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:39:31.550 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:39:31.550 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:39:31.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:31.567 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:39:31.567 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:39:31.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:31.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:31.583 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:39:31.584 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:39:31.600 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:39:31.601 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:39:31.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:31.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:31.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:31.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:31.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:31.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:31.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:31.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:31.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:32.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:32.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:32.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:32.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:32.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[98970]: pgmap v1470: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-08-29T15:39:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[98970]: osdmap e417: 8 total, 8 up, 8 in 2024-08-29T15:39:32.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1044285479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:32.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[93921]: pgmap v1470: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-08-29T15:39:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[93921]: osdmap e417: 8 total, 8 up, 8 in 2024-08-29T15:39:32.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1044285479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:32.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:32.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:32.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:32.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:32 smithi171 ceph-mon[90031]: pgmap v1470: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:32 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-08-29T15:39:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:32 smithi171 ceph-mon[90031]: osdmap e417: 8 total, 8 up, 8 in 2024-08-29T15:39:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1044285479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:32.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:32.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:32.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:32.670 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:32.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:32.709 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:32.716 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:32.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:32.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:32.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:32.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:32.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:32.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:33.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:33.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:33.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:33.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:39:33.179 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.4 to 1 (10000) 2024-08-29T15:39:33.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:33.226 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:33.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:33.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:33.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:33 smithi027 ceph-mon[93921]: osdmap e418: 8 total, 8 up, 8 in 2024-08-29T15:39:33.384 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1973379108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:39:33.384 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:33 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:39:33.385 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:33 smithi027 ceph-mon[98970]: osdmap e418: 8 total, 8 up, 8 in 2024-08-29T15:39:33.385 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1973379108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:39:33.385 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:33 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:39:33.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:33.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:33.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:33.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:33.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:33 smithi171 ceph-mon[90031]: osdmap e418: 8 total, 8 up, 8 in 2024-08-29T15:39:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1973379108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:39:33.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:33 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:39:33.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:33.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:33.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:33.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:33.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:33.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:33.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:33.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:33.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:33.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:33.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:33.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:33.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:34.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:34.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:34.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:34.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:34.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:34.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:34.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:34.415 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:34.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:39:34.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:34 smithi027 ceph-mon[93921]: pgmap v1473: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:34.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:34 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:39:34.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:34 smithi027 ceph-mon[93921]: osdmap e419: 8 total, 8 up, 8 in 2024-08-29T15:39:34.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:34 smithi027 ceph-mon[98970]: pgmap v1473: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:34.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:34 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:39:34.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:34 smithi027 ceph-mon[98970]: osdmap e419: 8 total, 8 up, 8 in 2024-08-29T15:39:34.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:34 smithi171 ceph-mon[90031]: pgmap v1473: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:34 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:39:34.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:34 smithi171 ceph-mon[90031]: osdmap e419: 8 total, 8 up, 8 in 2024-08-29T15:39:34.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:34.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:34.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:34.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:34.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:34.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:34.788 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:34.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:34.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:34.912 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:34.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:34.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:35.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:35.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:35.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:35.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:35.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:35.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:35.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:35.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:35.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 7 1 2024-08-29T15:39:35.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:35.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:35.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:35.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:35.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:35.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:35.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:35.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:35.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:35.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:35.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:35.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:35.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:35.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:35.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:36.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:36.057 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:36.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:36.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:36.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:36.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_4' created 2024-08-29T15:39:36.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:36 smithi027 ceph-mon[93921]: pgmap v1475: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:36.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1845898649' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-29T15:39:36.297 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:36 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-29T15:39:36.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:36 smithi027 ceph-mon[98970]: pgmap v1475: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:36.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1845898649' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-29T15:39:36.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:36 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-29T15:39:36.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:36.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:36.413 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:36.443 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:36.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:36 smithi171 ceph-mon[90031]: pgmap v1475: 83 pgs: 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1845898649' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-29T15:39:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:36 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-29T15:39:36.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:36.564 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:36.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:36.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:36.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:36.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:36.799 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:36.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:36.917 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:36.917 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:36.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:36.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-08-29T15:39:36.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:36.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:36.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:37.054 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:37.080 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:37.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:37.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:37.308 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:37.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:37.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:37.435 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:37.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:37.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:37 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-29T15:39:37.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:37 smithi027 ceph-mon[98970]: osdmap e420: 8 total, 8 up, 8 in 2024-08-29T15:39:37.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:37 smithi027 ceph-mon[98970]: osdmap e421: 8 total, 8 up, 8 in 2024-08-29T15:39:37.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:37 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-29T15:39:37.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:37 smithi027 ceph-mon[93921]: osdmap e420: 8 total, 8 up, 8 in 2024-08-29T15:39:37.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:37 smithi027 ceph-mon[93921]: osdmap e421: 8 total, 8 up, 8 in 2024-08-29T15:39:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:37 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-29T15:39:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:37 smithi171 ceph-mon[90031]: osdmap e420: 8 total, 8 up, 8 in 2024-08-29T15:39:37.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:37 smithi171 ceph-mon[90031]: osdmap e421: 8 total, 8 up, 8 in 2024-08-29T15:39:37.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:37.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:37.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:37.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:37.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:37.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:37.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:37.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:37.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:37.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:37.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:38.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:38.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:38.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:38.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:38.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:38.316 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:38.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:38.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:38.458 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:38 smithi171 ceph-mon[90031]: pgmap v1477: 99 pgs: 5 creating+peering, 11 unknown, 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:38.458 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:38 smithi171 ceph-mon[90031]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:39:38.458 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:38 smithi171 ceph-mon[90031]: osdmap e422: 8 total, 8 up, 8 in 2024-08-29T15:39:38.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:38.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:38.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:38 smithi027 ceph-mon[93921]: pgmap v1477: 99 pgs: 5 creating+peering, 11 unknown, 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:38.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:38 smithi027 ceph-mon[93921]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:39:38.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:38 smithi027 ceph-mon[93921]: osdmap e422: 8 total, 8 up, 8 in 2024-08-29T15:39:38.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:38 smithi027 ceph-mon[98970]: pgmap v1477: 99 pgs: 5 creating+peering, 11 unknown, 83 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:38.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:38 smithi027 ceph-mon[98970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:39:38.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:38 smithi027 ceph-mon[98970]: osdmap e422: 8 total, 8 up, 8 in 2024-08-29T15:39:38.572 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:38.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:38.698 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:38.720 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:38.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:38.822 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:38.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:38.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:38.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:38.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:38.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:39.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:39.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:39.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:39.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:39.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:39:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:39:39] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.51.0" 2024-08-29T15:39:39.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:39.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:39.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:39.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:39.467 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:39.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:39.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:39.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:39.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:39.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:39.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:39.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:39.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:39.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:39.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:39.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:39.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:40.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:40.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:40.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:40.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:40.216 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:40.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:40.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:40.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:40.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:40.445 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:40 smithi027 ceph-mon[93921]: pgmap v1480: 99 pgs: 8 creating+peering, 5 unknown, 86 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:40.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:40 smithi027 ceph-mon[98970]: pgmap v1480: 99 pgs: 8 creating+peering, 5 unknown, 86 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:40.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:40.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:40 smithi171 ceph-mon[90031]: pgmap v1480: 99 pgs: 8 creating+peering, 5 unknown, 86 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:40.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:40.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:40.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:40.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:40.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:40.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:40.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:40.918 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:39:40.918 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:39:40.918 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:39:40.918 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:39:40.918 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:39:40.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:40.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:40.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:40.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:41.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:41.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:41.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:41.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:41.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:41.323 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:41.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:41.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:41.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:41.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:41.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:41.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:41.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:41.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:41.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:41.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:41.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:41.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:41.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:42.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:42.099 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:42.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:42.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:42.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:42.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:39:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.350 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:42.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:42.451 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:42.468 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:42 smithi027 ceph-mon[93921]: pgmap v1481: 99 pgs: 8 creating+peering, 91 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:42.468 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:42 smithi027 ceph-mon[98970]: pgmap v1481: 99 pgs: 8 creating+peering, 91 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:42.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:42 smithi171 ceph-mon[90031]: pgmap v1481: 99 pgs: 8 creating+peering, 91 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:42.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:42.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:42.704 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:42.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:42.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:42.808 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:39:42.808 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:39:42.826 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:39:42.826 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:39:42.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:42.843 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:39:42.843 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:39:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.855 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:42.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:42.859 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:39:42.860 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:39:42.877 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:39:42.877 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:39:42.894 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:39:42.894 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:39:42.911 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:39:42.911 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:39:42.928 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:39:42.928 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:39:42.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:42.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:42.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:42.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:43.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:43.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:43.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:43.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:43.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:43.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:43.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:43.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:43.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:43.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:43.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1299623033' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:43.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1299623033' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:43.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1299623033' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:43.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:43.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:43.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:43.714 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:43.740 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:43.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:43.793 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:43.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:43.864 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:43.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:43.988 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:43.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:44.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:44.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:44.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:44.210 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.7 to 1 (10000) 2024-08-29T15:39:44.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:44.219 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:39:44.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:44.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:44.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:44.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:44.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:44.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:44.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:44.496 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:44.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:44 smithi171 ceph-mon[90031]: pgmap v1482: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1976472511' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:39:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:44 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:39:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:44.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1563119094' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:39:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[98970]: pgmap v1482: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1976472511' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:39:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:39:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1563119094' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:39:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[93921]: pgmap v1482: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1976472511' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:39:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:39:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:39:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:39:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1563119094' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:39:44.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:44.624 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:44.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:44.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:44.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:44.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:44.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:44.854 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:44.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:44.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:44.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:45.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:45.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:45.132 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:45.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:45.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:45.258 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:45.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:45.384 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:45.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:45.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:39:45.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1563119094' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:39:45.482 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[93921]: osdmap e423: 8 total, 8 up, 8 in 2024-08-29T15:39:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:39:45.482 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1563119094' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:39:45.483 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:45 smithi027 ceph-mon[98970]: osdmap e423: 8 total, 8 up, 8 in 2024-08-29T15:39:45.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:45.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:45.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:45 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:45 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:39:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1563119094' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:39:45.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:45 smithi171 ceph-mon[90031]: osdmap e423: 8 total, 8 up, 8 in 2024-08-29T15:39:45.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:45.635 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:45.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:45.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:45.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:45.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:45.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:45.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:45.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:46.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:46.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:46.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 5 1 2024-08-29T15:39:46.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:46.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:46.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:46.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:46 smithi027 ceph-mon[98970]: pgmap v1484: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:46.245 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:46 smithi027 ceph-mon[93921]: pgmap v1484: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:46.269 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:46.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.399 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:46.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:46.452 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:46.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:46 smithi171 ceph-mon[90031]: pgmap v1484: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:46.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:46.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:46.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:46.673 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:46.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:39:46.752 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:46.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:46.880 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:46.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:46.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:46.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:47.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:47.030 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:47.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:47.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:47.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:47.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:47.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:47.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:47.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:39:47.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:47.530 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:47.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:47.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:47.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:47.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:47.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:47.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:47.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:48.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:48.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:48.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:48.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:48.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:48.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:48.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:48.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:48.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:48.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:48.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:48.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:48 smithi027 ceph-mon[98970]: pgmap v1485: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:48.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:48 smithi027 ceph-mon[93921]: pgmap v1485: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:48.508 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:48 smithi171 ceph-mon[90031]: pgmap v1485: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:48.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:48.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:48.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:48.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:48.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:48.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:48.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:48.915 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:48.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:48.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:49.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:49.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:49.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:49.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:49.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:49.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:49.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:39:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:39:49] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.51.0" 2024-08-29T15:39:49.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:49.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:49.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:49.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:49.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:49.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:49.671 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:49.773 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.797 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:49.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:49.845 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:39:49.845 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:39:49.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:49.920 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:49.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:49.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:50.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:50.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:50.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:50.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:50.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:50.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:50.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:50.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:50 smithi171 ceph-mon[90031]: pgmap v1486: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:50.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:50.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:50.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:50 smithi027 ceph-mon[93921]: pgmap v1486: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:50.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:50 smithi027 ceph-mon[98970]: pgmap v1486: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:50.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:39:50.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:50.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:50.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:50.772 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:50.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:50.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:50.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:50.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:50.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:51.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:51.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:51.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:51.169 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:51.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:51.296 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:51.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:51.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:51.422 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:51.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:51.519 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:51.520 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:51.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:51.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:51.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:51.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:51.671 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:51.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:51.790 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:39:51.800 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:51.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:51.901 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:51.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:51.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:51.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:52.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:52.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:52.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:52.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:52.175 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:52.244 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.5 to 1 (10000) 2024-08-29T15:39:52.246 INFO:teuthology.orchestra.run.smithi027.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-08-29T15:39:52.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:52.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[98970]: pgmap v1487: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/165261136' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:39:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:39:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2459067959' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:39:52.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:39:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[93921]: pgmap v1487: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/165261136' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:39:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:39:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2459067959' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:39:52.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:52 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:39:52.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:52.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:52 smithi171 ceph-mon[90031]: pgmap v1487: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/165261136' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:39:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:52 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:39:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2459067959' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:39:52.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:52 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:39:52.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:52.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:52.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:52.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:52.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:52.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:52.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:52.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:52.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:52.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:53.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:53.060 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:53.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:53.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:53.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:53.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:53.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:53.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:53.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:53 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:39:53.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:53 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:39:53.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:53 smithi027 ceph-mon[98970]: osdmap e424: 8 total, 8 up, 8 in 2024-08-29T15:39:53.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:53 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:39:53.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:53 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:39:53.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:53 smithi027 ceph-mon[93921]: osdmap e424: 8 total, 8 up, 8 in 2024-08-29T15:39:53.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:53.435 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:53.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:53 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:39:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:53 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:39:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:53 smithi171 ceph-mon[90031]: osdmap e424: 8 total, 8 up, 8 in 2024-08-29T15:39:53.537 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:53.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:53.664 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:53.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:53.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:53.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:53.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:53.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:53.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:54.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:54.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:54.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:54.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:54.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:54.296 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:54.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:54.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:54.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:54.470 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:54 smithi027 ceph-mon[98970]: pgmap v1489: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:54.470 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:54 smithi027 ceph-mon[98970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:39:54.470 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:54 smithi027 ceph-mon[98970]: osdmap e425: 8 total, 8 up, 8 in 2024-08-29T15:39:54.470 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:54 smithi027 ceph-mon[93921]: pgmap v1489: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:54.470 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:54 smithi027 ceph-mon[93921]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:39:54.470 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:54 smithi027 ceph-mon[93921]: osdmap e425: 8 total, 8 up, 8 in 2024-08-29T15:39:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:54 smithi171 ceph-mon[90031]: pgmap v1489: 99 pgs: 99 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-29T15:39:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:54 smithi171 ceph-mon[90031]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:39:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:54 smithi171 ceph-mon[90031]: osdmap e425: 8 total, 8 up, 8 in 2024-08-29T15:39:54.549 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:54.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:54.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:54.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:54.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:54.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:54.929 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:54.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:54.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:55.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:55.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:55.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:55.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:55.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:55.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.333 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:55.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:55.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:55.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:55 smithi171 ceph-mon[90031]: osdmap e426: 8 total, 8 up, 8 in 2024-08-29T15:39:55.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:55.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:55 smithi027 ceph-mon[93921]: osdmap e426: 8 total, 8 up, 8 in 2024-08-29T15:39:55.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:55 smithi027 ceph-mon[98970]: osdmap e426: 8 total, 8 up, 8 in 2024-08-29T15:39:55.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:55.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:55.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:55.709 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:55.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:55.835 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:55.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:55.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:55.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:55.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:55.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:56.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:56.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:56.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:56.212 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:56.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:56.336 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:56.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:56 smithi027 ceph-mon[98970]: pgmap v1492: 99 pgs: 1 remapped+peering, 98 active+clean; 4.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 8 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-29T15:39:56.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:56 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:39:56.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:56 smithi027 ceph-mon[98970]: osdmap e427: 8 total, 8 up, 8 in 2024-08-29T15:39:56.392 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:56 smithi027 ceph-mon[93921]: pgmap v1492: 99 pgs: 1 remapped+peering, 98 active+clean; 4.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 8 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-29T15:39:56.392 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:56 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:39:56.392 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:56 smithi027 ceph-mon[93921]: osdmap e427: 8 total, 8 up, 8 in 2024-08-29T15:39:56.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:56.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:56.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:56.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:56.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:56.716 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:56.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:56 smithi171 ceph-mon[90031]: pgmap v1492: 99 pgs: 1 remapped+peering, 98 active+clean; 4.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 8 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-29T15:39:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:56 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:39:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:56 smithi171 ceph-mon[90031]: osdmap e427: 8 total, 8 up, 8 in 2024-08-29T15:39:56.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:56.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:56.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:56.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:56.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:56.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:57.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:57.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:57.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:57.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:57.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:57.287 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:39:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:57 smithi027 ceph-mon[93921]: osdmap e428: 8 total, 8 up, 8 in 2024-08-29T15:39:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2461102617' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:57.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:57 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:57 smithi027 ceph-mon[98970]: osdmap e428: 8 total, 8 up, 8 in 2024-08-29T15:39:57.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2461102617' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:57.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:57 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:57.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:57.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:57.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:57.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:57.472 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:57.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:57.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:57.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:57.682 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:39:57.682 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:39:57.682 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:39:57.683 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:39:57.684 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:39:57.684 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:39:57.684 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:39:57.684 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:39:57.684 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:39:57.698 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:57.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:57.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:57 smithi171 ceph-mon[90031]: osdmap e428: 8 total, 8 up, 8 in 2024-08-29T15:39:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2461102617' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:57 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:39:57.795 DEBUG:teuthology.orchestra.run.smithi027:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-08-29T15:39:57.798 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-29T15:39:57.798 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-29T15:39:57.823 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:57.847 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:57.877 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:hints = 1 2024-08-29T15:39:57.877 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-29T15:39:57.878 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object prefix: benchmark_data_smithi027_341149 2024-08-29T15:39:57.878 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:39:57.878 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 0 0 0 0 0 0 - 0 2024-08-29T15:39:57.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:39:57.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:57.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:57.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:39:58.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:58.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:58.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:58.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:58.254 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:58.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:58.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[93921]: pgmap v1495: 99 pgs: 1 remapped+peering, 98 active+clean; 4.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 119 KiB/s, 1 objects/s recovering 2024-08-29T15:39:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[93921]: osdmap e429: 8 total, 8 up, 8 in 2024-08-29T15:39:58.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/534228315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:58.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[98970]: pgmap v1495: 99 pgs: 1 remapped+peering, 98 active+clean; 4.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 119 KiB/s, 1 objects/s recovering 2024-08-29T15:39:58.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:58.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:58.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[98970]: osdmap e429: 8 total, 8 up, 8 in 2024-08-29T15:39:58.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/534228315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:58.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:58.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:58.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:58.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:39:58.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:58.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:39:58.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:39:58.672 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:58.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:39:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:58 smithi171 ceph-mon[90031]: pgmap v1495: 99 pgs: 1 remapped+peering, 98 active+clean; 4.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 119 KiB/s, 1 objects/s recovering 2024-08-29T15:39:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:58 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:39:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:58 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:39:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:58 smithi171 ceph-mon[90031]: osdmap e429: 8 total, 8 up, 8 in 2024-08-29T15:39:58.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/534228315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:39:58.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:39:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:58.815 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:58.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:39:58.878 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 1 16 3598 3582 223.862 223.875 0.00497585 0.00444921 2024-08-29T15:39:58.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:58.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:58.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:59.035 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:39:59.035 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:39:59.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.035 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:4103 9.9 9:9741e643:::benchmark_data_smithi027_341149_object4102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e429)", 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:39:59.031562+0000", 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0011744240000000001, 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012036670000000001, 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 4103 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:39:59.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031562+0000", 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031562+0000", 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031563+0000", 2024-08-29T15:39:59.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6599999999999994e-07 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031586+0000", 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2342999999999999e-05 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031598+0000", 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1919999999999999e-05 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031605+0000", 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8769999999999993e-06 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031680+0000", 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5078000000000002e-05 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031837+0000", 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015697000000000001 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.031975+0000", 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013709900000000001 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:39:59.032751+0000", 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077663500000000002 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:39:59.040 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2024-08-29T15:39:59.041 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:39:59.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:59.080 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:59.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:59.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:39:59.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:59.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:59.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:39:59.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:39:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:39:59] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.51.0" 2024-08-29T15:39:59.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:59.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:59.455 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:39:59.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:59.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:39:59.595 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:39:59.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:39:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:59.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:39:59.748 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:59.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:59.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:59.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:39:59.878 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 2 16 7030 7014 219.17 214.5 0.00393082 0.00455559 2024-08-29T15:39:59.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:39:59.914 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:39:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:39:59.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:00.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:39:59 smithi171 ceph-mon[90031]: pgmap v1497: 99 pgs: 99 active+clean; 33 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.0 MiB/s wr, 96 op/s; 123 KiB/s, 0 objects/s recovering 2024-08-29T15:40:00.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:00.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:00.063 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:00.063 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:00.063 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:39:59 smithi027 ceph-mon[93921]: pgmap v1497: 99 pgs: 99 active+clean; 33 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.0 MiB/s wr, 96 op/s; 123 KiB/s, 0 objects/s recovering 2024-08-29T15:40:00.064 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:00.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:00.065 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:39:59 smithi027 ceph-mon[98970]: pgmap v1497: 99 pgs: 99 active+clean; 33 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.0 MiB/s wr, 96 op/s; 123 KiB/s, 0 objects/s recovering 2024-08-29T15:40:00.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 6 1 2024-08-29T15:40:00.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:00.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:00.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:00.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:00.331 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:00.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:00.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:00.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:00.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:00.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:00.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:00.600 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:00.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:00.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:00.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:00.735 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:00.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:00.837 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:00.878 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 3 16 10190 10174 211.904 197.5 0.00301559 0.00471101 2024-08-29T15:40:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:00.880 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:00.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:00.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:01.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:01.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:01.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:00 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:40:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:00 smithi171 ceph-mon[90031]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-08-29T15:40:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:00 smithi171 ceph-mon[90031]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-29T15:40:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:00 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:40:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:00 smithi027 ceph-mon[98970]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-08-29T15:40:01.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:00 smithi027 ceph-mon[98970]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-29T15:40:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:00 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:40:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:00 smithi027 ceph-mon[93921]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-08-29T15:40:01.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:00 smithi027 ceph-mon[93921]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-29T15:40:01.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:01.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:01.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:01.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:40:01.250 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:01.277 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:01.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:01.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:01.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:01.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:01.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:01.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:01.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:01.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:01.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:01.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:01.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:40:01.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:01.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:01.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:01.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:01.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:40:01.844 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:01.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:01.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:01.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:01.878 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 4 16 13044 13028 203.506 178.375 0.00408582 0.00490345 2024-08-29T15:40:01.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:01 smithi027 ceph-mon[98970]: pgmap v1498: 99 pgs: 99 active+clean; 154 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 399 op/s; 96 KiB/s, 0 objects/s recovering 2024-08-29T15:40:01.932 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:01 smithi027 ceph-mon[93921]: pgmap v1498: 99 pgs: 99 active+clean; 154 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 399 op/s; 96 KiB/s, 0 objects/s recovering 2024-08-29T15:40:01.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:02.003 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:02.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:01 smithi171 ceph-mon[90031]: pgmap v1498: 99 pgs: 99 active+clean; 154 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 399 op/s; 96 KiB/s, 0 objects/s recovering 2024-08-29T15:40:02.105 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:02.138 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:02.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:02.240 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:02.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:02.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:02.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:02.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:02.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:02.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:02.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:02.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:02.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:02.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:02.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 5 16 15868 15852 198.102 176.5 0.00265916 0.00504169 2024-08-29T15:40:02.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:02.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:02.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:03.007 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:03.021 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:03.044 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:03.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:03.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:03.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:03.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:03.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:03.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:03.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:03.355 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:03.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:03.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:03.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:03.523 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:03.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:03.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:03.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:03.648 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:03.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:03.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:03.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:03.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:03.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 6 16 18245 18229 189.844 148.562 0.00312583 0.00526171 2024-08-29T15:40:03.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:03.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:03.927 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:03.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:04.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:04.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:04.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:04.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:04.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:04.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:04.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:04.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:04 smithi171 ceph-mon[90031]: pgmap v1499: 99 pgs: 99 active+clean; 596 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 75 KiB/s, 0 objects/s recovering 2024-08-29T15:40:04.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:04 smithi027 ceph-mon[98970]: pgmap v1499: 99 pgs: 99 active+clean; 596 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 75 KiB/s, 0 objects/s recovering 2024-08-29T15:40:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:04 smithi027 ceph-mon[93921]: pgmap v1499: 99 pgs: 99 active+clean; 596 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 75 KiB/s, 0 objects/s recovering 2024-08-29T15:40:04.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:04.361 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:04.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:04.466 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:04.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:04.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:04.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:04.640 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:04.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:04.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:04.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:04.774 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:04.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:04.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:04.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 7 16 20314 20298 181.196 129.312 0.0128611 0.00551087 2024-08-29T15:40:04.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:04.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:05.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:05.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:05.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:05.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:05.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:05.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:05.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:05.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:05.443 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:05.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:05.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:05.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:05.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:05.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:05.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:05.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:05.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:05.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:05.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 8 16 22210 22194 173.358 118.5 0.00368164 0.0057632 2024-08-29T15:40:05.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:05.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:05.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:06.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:06 smithi027 ceph-mon[93921]: pgmap v1500: 99 pgs: 99 active+clean; 824 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 66 KiB/s, 0 objects/s recovering 2024-08-29T15:40:06.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:06 smithi027 ceph-mon[98970]: pgmap v1500: 99 pgs: 99 active+clean; 824 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 66 KiB/s, 0 objects/s recovering 2024-08-29T15:40:06.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:06.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:06.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:06.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:06.249 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:06.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:06.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:06.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:06 smithi171 ceph-mon[90031]: pgmap v1500: 99 pgs: 99 active+clean; 824 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 66 KiB/s, 0 objects/s recovering 2024-08-29T15:40:06.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:06.386 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:06.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:06.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:06.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:06.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:06.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:06.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:06.627 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:06.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:06.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:06.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:06.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:06.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:06.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:06.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 9 16 24383 24367 169.186 135.812 0.00766822 0.00590546 2024-08-29T15:40:06.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:06.939 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:06.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:07.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:07.075 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:07.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:07.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:07.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:07.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:07.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:07.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:07.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:07.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:07.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:07.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:07.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:07.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:07.581 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:07.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:07.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:07.715 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:07.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:07.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:07.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:07.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:07.876 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:07.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:07.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 10 16 26210 26194 163.685 114.188 0.00418619 0.00610333 2024-08-29T15:40:07.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:08.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:08.030 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.6 to 1 (10000) 2024-08-29T15:40:08.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:08.149 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:08.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:08.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:08 smithi171 ceph-mon[90031]: pgmap v1501: 99 pgs: 99 active+clean; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 26 KiB/s, 0 objects/s recovering 2024-08-29T15:40:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1788352018' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:40:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:08 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:40:08.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:08.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:08 smithi027 ceph-mon[93921]: pgmap v1501: 99 pgs: 99 active+clean; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 26 KiB/s, 0 objects/s recovering 2024-08-29T15:40:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1788352018' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:40:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:08 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:40:08.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:08 smithi027 ceph-mon[98970]: pgmap v1501: 99 pgs: 99 active+clean; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s; 26 KiB/s, 0 objects/s recovering 2024-08-29T15:40:08.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1788352018' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:40:08.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:08 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:40:08.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:08.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:08.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:08.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:08.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.685 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:08.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:08.786 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:08.819 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:08.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 11 16 28337 28321 160.889 132.938 0.00434173 0.00620919 2024-08-29T15:40:08.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:08.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:08.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:08.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:09.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:09.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:09.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:09.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:09.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:09.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:09.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:09.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:40:09.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:09 smithi027 ceph-mon[93921]: osdmap e430: 8 total, 8 up, 8 in 2024-08-29T15:40:09.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:40:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:40:09] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.51.0" 2024-08-29T15:40:09.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:40:09.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:09 smithi027 ceph-mon[98970]: osdmap e430: 8 total, 8 up, 8 in 2024-08-29T15:40:09.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:09.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:09.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:09.458 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:09.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:09.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:09.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:40:09.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:09 smithi171 ceph-mon[90031]: osdmap e430: 8 total, 8 up, 8 in 2024-08-29T15:40:09.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:09.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:09.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:09.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:09.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:09.740 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:09.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:09.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:09.869 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:09.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:09.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 12 16 29602 29586 154.07 79.0625 0.0062224 0.00648516 2024-08-29T15:40:09.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:10.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:10.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:10.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:10.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:10.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:10.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:10.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:10.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:10.406 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:10 smithi027 ceph-mon[98970]: pgmap v1503: 99 pgs: 99 active+clean; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-08-29T15:40:10.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:10 smithi027 ceph-mon[93921]: pgmap v1503: 99 pgs: 99 active+clean; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-08-29T15:40:10.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:10.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:10 smithi171 ceph-mon[90031]: pgmap v1503: 99 pgs: 99 active+clean; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-08-29T15:40:10.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:10.541 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:10.643 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:10.674 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:10.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:10.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:10.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:10.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:10.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 13 16 31578 31562 151.718 123.5 0.0102369 0.00658603 2024-08-29T15:40:10.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:10.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:10.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:10.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:11.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:11.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:11.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:11.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:11.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:11.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:11.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:11.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:11.359 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:11.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:11.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:11.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:11.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:11.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:11.635 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:11.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:11.737 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:11.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:11.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:11.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:11.879 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 14 16 33585 33569 149.84 125.438 0.019211 0.00666764 2024-08-29T15:40:11.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:11.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:11.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:12.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:12.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:12.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:12.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:12.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:12.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:12.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:12.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:12.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:12.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:12.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:12.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:12.443 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:12.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:12.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:12 smithi171 ceph-mon[90031]: pgmap v1504: 99 pgs: 99 active+clean; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-08-29T15:40:12.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:12.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:12 smithi027 ceph-mon[98970]: pgmap v1504: 99 pgs: 99 active+clean; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-08-29T15:40:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:12 smithi027 ceph-mon[93921]: pgmap v1504: 99 pgs: 99 active+clean; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2024-08-29T15:40:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:12.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:12.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:12.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:12.720 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:12.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:12.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:12.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:12.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:12.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:12.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 15 16 35585 35569 148.183 125 0.00514353 0.00674265 2024-08-29T15:40:12.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:13.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:13.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:13.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:13.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:13.244 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:13.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:13.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:13.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:13.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:13.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:13.520 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:13.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:13.655 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:13.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.687 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:13.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:13.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:13.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:13.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 16 16 37656 37640 147.011 129.438 0.00379361 0.00679606 2024-08-29T15:40:13.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:13.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:13.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:14.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:14.103 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:14.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:14.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:14.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:14.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:14.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:14 smithi027 ceph-mon[98970]: pgmap v1505: 99 pgs: 99 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-29T15:40:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:14.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:14 smithi027 ceph-mon[93921]: pgmap v1505: 99 pgs: 99 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-29T15:40:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:14.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:14.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:14.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:14.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:14.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:14.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:14.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:14.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:14.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:14 smithi171 ceph-mon[90031]: pgmap v1505: 99 pgs: 99 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2024-08-29T15:40:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:14.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:14.654 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:14.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:14.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:14.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:14.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:14.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:14.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 17 16 39643 39627 145.667 124.188 0.0040894 0.00685999 2024-08-29T15:40:14.897 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:14.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:14.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:14.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:15.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:15.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:15.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:15.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:15.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:15.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:15.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:15.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:15.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:15.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:15.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:15.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:15.483 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:15.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:15.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:15.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:15.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:15.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:15.660 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:40:15.660 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:40:15.660 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:40:15.660 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:40:15.660 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:40:15.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:15.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:15.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:15.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:15.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 18 16 41572 41556 144.272 120.562 0.00819069 0.00692611 2024-08-29T15:40:15.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:15.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:15.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:15.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:16.037 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:16.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:16.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:16.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:16.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:16.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:16.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:16.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:16.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:16 smithi027 ceph-mon[98970]: pgmap v1506: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-08-29T15:40:16.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:16 smithi027 ceph-mon[93921]: pgmap v1506: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-08-29T15:40:16.416 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:16.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:16.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:16 smithi171 ceph-mon[90031]: pgmap v1506: 99 pgs: 99 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-08-29T15:40:16.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:16.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:16.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:16.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:16.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:16.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:16.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:16.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:40:16.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:16.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 19 16 43443 43427 142.833 116.938 0.00528198 0.00699444 2024-08-29T15:40:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:16.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:16.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:16.997 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:17.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:17.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:17.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:17.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:17.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:17.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:17.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:17.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:17.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:17.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:17.474 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-29T15:40:17.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:40:17.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:17.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:17.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:17.621 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:17.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:17.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:17.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:17.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:17.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:17.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:17.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:40:17.880250+0000 min lat: 0.00131181 max lat: 0.356404 avg lat: 0.00707844 2024-08-29T15:40:17.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:40:17.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 20 16 45194 45178 141.162 109.438 0.00447326 0.00707844 2024-08-29T15:40:17.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:17.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:17.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:18.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:18.032 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:18.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:18.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:18.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:18.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:18.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:18.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:18.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:18.386 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:18 smithi027 ceph-mon[93921]: pgmap v1507: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-29T15:40:18.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:18 smithi027 ceph-mon[98970]: pgmap v1507: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-29T15:40:18.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:18.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:18.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:18.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:18 smithi171 ceph-mon[90031]: pgmap v1507: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-29T15:40:18.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:18.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:18.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:18.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:18.725 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:18.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:18.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:18.852 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:40:18.852 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:40:18.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 21 16 46741 46725 139.044 96.6875 0.0143284 0.00718307 2024-08-29T15:40:18.885 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:18.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:18.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:18.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:40:18.909 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:40:18.947 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:18.958 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:40:18.959 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:40:18.998 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:40:18.998 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:40:19.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:19.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:40:19.039 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:40:19.046 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:19.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:19.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:19.077 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:40:19.077 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:40:19.108 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:40:19.108 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:40:19.114 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:40:19.143 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:40:19.144 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:40:19.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:19.168 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3094455921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:19.169 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2003636422' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:40:19.169 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:40:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:40:19] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-08-29T15:40:19.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3094455921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:19.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2003636422' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:40:19.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:19.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:19.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:19.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:19.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:19.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:19.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:19.477 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:19.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3094455921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2003636422' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:40:19.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:19.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:19.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:19.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:19.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:19.757 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:19.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:19.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:19.881 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 22 16 48445 48429 137.564 106.5 0.011554 0.00726452 2024-08-29T15:40:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:19.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:19.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:19.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:20.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:20.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:20.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:20.162 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:20.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:20.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:20.206 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[98970]: pgmap v1508: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-08-29T15:40:20.207 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:40:20.207 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2003636422' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:40:20.207 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[98970]: osdmap e431: 8 total, 8 up, 8 in 2024-08-29T15:40:20.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[93921]: pgmap v1508: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-08-29T15:40:20.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:40:20.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2003636422' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:40:20.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:20 smithi027 ceph-mon[93921]: osdmap e431: 8 total, 8 up, 8 in 2024-08-29T15:40:20.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:20.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:20.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:20.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:20.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:20.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:20.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:20.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:20.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:20 smithi171 ceph-mon[90031]: pgmap v1508: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-08-29T15:40:20.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:20 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:40:20.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:20 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2003636422' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:40:20.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:20 smithi171 ceph-mon[90031]: osdmap e431: 8 total, 8 up, 8 in 2024-08-29T15:40:20.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:20.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:20.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:20.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:20.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:20.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:20.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:20.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:20.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:20.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:20.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 23 16 50100 50084 136.08 103.438 0.0063166 0.00734337 2024-08-29T15:40:20.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:21.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:21.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:21.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:21.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:21.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:21.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:21.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:21.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.447 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:21.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:21.471 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:40:21.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:40:21.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:40:21.587 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:21.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:21.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:21.726 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:21.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:21.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:21.880 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 24 16 51688 51672 134.545 99.25 0.00670326 0.00740981 2024-08-29T15:40:21.952 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:21.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:40:21.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:21.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:22.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:22.128 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:22.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:22.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:22.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:22.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:22.269 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:22.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:22.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:22.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:22.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:22.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:22.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:22 smithi171 ceph-mon[90031]: pgmap v1510: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-08-29T15:40:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:22 smithi171 ceph-mon[90031]: 1.8 scrub starts 2024-08-29T15:40:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:22 smithi171 ceph-mon[90031]: 1.8 scrub ok 2024-08-29T15:40:22.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:22.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:22 smithi027 ceph-mon[98970]: pgmap v1510: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-08-29T15:40:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:22 smithi027 ceph-mon[98970]: 1.8 scrub starts 2024-08-29T15:40:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:22 smithi027 ceph-mon[98970]: 1.8 scrub ok 2024-08-29T15:40:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:22 smithi027 ceph-mon[93921]: pgmap v1510: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-08-29T15:40:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:22 smithi027 ceph-mon[93921]: 1.8 scrub starts 2024-08-29T15:40:22.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:22 smithi027 ceph-mon[93921]: 1.8 scrub ok 2024-08-29T15:40:22.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:22.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:22.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:22.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:22.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:22.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:22.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:22.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:22.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 25 16 53282 53266 133.143 99.625 0.00462102 0.00750532 2024-08-29T15:40:22.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:22.977 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:22.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:23.079 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:23.115 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:23.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:23.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:23.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:23.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:23.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:23.388 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:23.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:23.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:23.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:23.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:23.514 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:23.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:23.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:23.649 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:23.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:23.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:23.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:23.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:23.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:23.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 26 16 54675 54659 131.37 87.0625 0.00611387 0.00760643 2024-08-29T15:40:23.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:23.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:23.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:23.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:24.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:24.045 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:24.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:24.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:24.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:24.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:24.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:24.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:24 smithi027 ceph-mon[93921]: pgmap v1511: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:40:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:24 smithi027 ceph-mon[98970]: pgmap v1511: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:40:24.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:24.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:24.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:24.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:24.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:24.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:24 smithi171 ceph-mon[90031]: pgmap v1511: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:40:24.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:24.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:24.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:24.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:24.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:24.742 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:24.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:24.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:40:24.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:24.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 27 16 56004 55988 129.581 83.0625 0.00702378 0.00771146 2024-08-29T15:40:24.889 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:24.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:24.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:24.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:25.017 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:25.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:25.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:25.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.140 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:25.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:25.241 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:25.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:25.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:25.388 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:25.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:25.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:25.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:25.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:25.703 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:40:25.703 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:40:25.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:25.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:25.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:25.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:25.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 28 16 57265 57249 127.767 78.8125 0.10843 0.00781984 2024-08-29T15:40:25.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:25.971 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:25.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:26.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:26.109 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:26.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:26.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:26.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:26.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:26.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:26.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:26.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:26.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:26.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:26.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:26.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:26.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:26 smithi171 ceph-mon[90031]: pgmap v1512: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-29T15:40:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:26 smithi171 ceph-mon[90031]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:40:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:26 smithi027 ceph-mon[98970]: pgmap v1512: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-29T15:40:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:26 smithi027 ceph-mon[98970]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:40:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:26 smithi027 ceph-mon[93921]: pgmap v1512: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-29T15:40:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:26 smithi027 ceph-mon[93921]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:40:26.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:26.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:26.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:26.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:26.779 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:26.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:26.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:26.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 29 16 58575 58559 126.185 81.875 0.00201751 0.00787805 2024-08-29T15:40:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:26.908 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:26.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:27.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:27.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:27.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:27.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:27.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:27.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:27.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:27.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:27.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:27.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:27.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:27.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:27.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:27.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:27.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:27.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:27.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:27.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:27.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:27.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:27.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:27.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:27.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:27.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:27.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:27.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 30 16 60016 60000 124.98 90.0625 0.00651379 0.00799043 2024-08-29T15:40:27.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:28.005 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:28.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:28.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:28.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:28.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:28.282 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:28.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:28.415 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:28.517 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:28 smithi171 ceph-mon[90031]: pgmap v1513: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:40:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:28.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:28.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:28.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:28 smithi027 ceph-mon[98970]: pgmap v1513: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:40:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:28.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:28 smithi027 ceph-mon[93921]: pgmap v1513: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:40:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:28.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:28.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:28.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:28.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:28.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:28.833 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:28.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:28.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 31 16 61557 61541 124.055 96.3125 0.00826343 0.00805561 2024-08-29T15:40:28.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:28.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:28.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:29.072 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:29.102 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:29.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:29.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:29.204 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:29.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:29.244 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:29.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:29.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:29.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:29.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:40:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:40:29] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-08-29T15:40:29.345 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:29.375 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:29.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:29.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:29.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:29.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:29.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:29.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:29.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:29.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:29.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:29.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:29.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:29.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 32 16 63064 63048 123.122 94.1875 0.00920575 0.0081173 2024-08-29T15:40:29.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:29.923 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:29.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:29.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:30.025 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:30.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:30.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:30.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:30.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:30.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:30.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:30.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:30.297 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:30.330 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:30.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:30.382 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:40:30.384 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:30 smithi027 ceph-mon[93921]: pgmap v1514: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-29T15:40:30.385 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:30 smithi027 ceph-mon[98970]: pgmap v1514: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-29T15:40:30.387 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:40:30.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:30.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:30.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:30.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:30 smithi171 ceph-mon[90031]: pgmap v1514: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-08-29T15:40:30.576 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:30.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:30.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:30.712 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:30.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:30.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:30.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:30.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 33 16 64604 64588 122.307 96.25 0.0204174 0.00817089 2024-08-29T15:40:30.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:30.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:30.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:31.018 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:31.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:31.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:31.160 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:31.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:31.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:40:31.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:31.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:31.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:31.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:31.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:31.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:31.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:31.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:31.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:31.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:31.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:31.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:31.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:31.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:31.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 34 16 66025 66009 121.322 88.8125 0.00290997 0.00823572 2024-08-29T15:40:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:31.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:31.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:31.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:32.031 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:32.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:32.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:32.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:32.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:32.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:32.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:32.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:32.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:32.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:32.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:32.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:32 smithi171 ceph-mon[90031]: pgmap v1515: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-29T15:40:32.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:32.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:32 smithi027 ceph-mon[93921]: pgmap v1515: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-29T15:40:32.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:32 smithi027 ceph-mon[98970]: pgmap v1515: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-29T15:40:32.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:32.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:32.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:32.724 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:32.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:32.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:32.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:32.858 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:32.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:32.882 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 35 16 67516 67500 120.518 93.1875 0.00908401 0.00829255 2024-08-29T15:40:32.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:33.001 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:33.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:33.103 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:33.133 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:33.235 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:33.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.266 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:33.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:33.367 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:33.405 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:33.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:33.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:33.541 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:33.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:33.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:33.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:33.779 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:33.809 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:33.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:33.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 36 16 69022 69006 119.785 94.125 0.00820579 0.00834349 2024-08-29T15:40:33.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:33.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:33.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:34.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:34.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:34.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:34.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:34.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:34.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:34.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:34.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:34.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:34.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:34.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:34 smithi027 ceph-mon[98970]: pgmap v1516: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-08-29T15:40:34.403 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:34 smithi027 ceph-mon[93921]: pgmap v1516: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-08-29T15:40:34.443 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1512,"stamp":"2024-08-29T15:40:24.955005+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3402569701,"num_objects":51874,"num_object_clones":0,"num_object_copies":155622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51874,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":52754,"num_write_kb":3330769,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":180,"num_bytes_recovered":9051723,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54518,"ondisk_log_size":54518,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14804088,"kb_used_data":9969844,"kb_used_omap":43,"kb_used_meta":4834132,"kb_avail":735157128,"statfs":{"total":767960285184,"available":752800899072,"internally_reserved":0,"allocated":10209120256,"data_stored":10187794238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":4950151660},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,0,3,7],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1175924858,"num_objects":17941,"num_object_clones":0,"num_object_copies":53823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17941,"num_whiteouts":0,"num_read":-34,"num_read_kb":-75,"num_write":17921,"num_write_kb":1147941,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003858"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":767,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.130795+0000","last_change":"2024-08-29T15:39:01.754656+0000","last_active":"2024-08-29T15:40:19.130795+0000","last_peered":"2024-08-29T15:40:19.130795+0000","last_clean":"2024-08-29T15:40:19.130795+0000","last_became_active":"2024-08-29T15:39:01.754213+0000","last_became_peered":"2024-08-29T15:39:01.754213+0000","last_unstale":"2024-08-29T15:40:19.130795+0000","last_undegraded":"2024-08-29T15:40:19.130795+0000","last_fullsized":"2024-08-29T15:40:19.130795+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:34.210741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":222,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039258+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:40:08.039258+0000","last_peered":"2024-08-29T15:40:08.039258+0000","last_clean":"2024-08-29T15:40:08.039258+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:40:08.039258+0000","last_undegraded":"2024-08-29T15:40:08.039258+0000","last_fullsized":"2024-08-29T15:40:08.039258+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.029087+0000","last_change":"2024-08-29T15:39:53.258205+0000","last_active":"2024-08-29T15:40:08.029087+0000","last_peered":"2024-08-29T15:40:08.029087+0000","last_clean":"2024-08-29T15:40:08.029087+0000","last_became_active":"2024-08-29T15:39:53.257784+0000","last_became_peered":"2024-08-29T15:39:53.257784+0000","last_unstale":"2024-08-29T15:40:08.029087+0000","last_undegraded":"2024-08-29T15:40:08.029087+0000","last_fullsized":"2024-08-29T15:40:08.029087+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:30:42.270982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":733,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114080+0000","last_change":"2024-08-29T15:39:21.000890+0000","last_active":"2024-08-29T15:40:19.114080+0000","last_peered":"2024-08-29T15:40:19.114080+0000","last_clean":"2024-08-29T15:40:19.114080+0000","last_became_active":"2024-08-29T15:39:21.000547+0000","last_became_peered":"2024-08-29T15:39:21.000547+0000","last_unstale":"2024-08-29T15:40:19.114080+0000","last_undegraded":"2024-08-29T15:40:19.114080+0000","last_fullsized":"2024-08-29T15:40:19.114080+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:43:30.808747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":718,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114776+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:40:19.114776+0000","last_peered":"2024-08-29T15:40:19.114776+0000","last_clean":"2024-08-29T15:40:19.114776+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:40:19.114776+0000","last_undegraded":"2024-08-29T15:40:19.114776+0000","last_fullsized":"2024-08-29T15:40:19.114776+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":215,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.038937+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:40:08.038937+0000","last_peered":"2024-08-29T15:40:08.038937+0000","last_clean":"2024-08-29T15:40:08.038937+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:40:08.038937+0000","last_undegraded":"2024-08-29T15:40:08.038937+0000","last_fullsized":"2024-08-29T15:40:08.038937+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":52,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.129600+0000","last_change":"2024-08-29T15:39:21.005920+0000","last_active":"2024-08-29T15:40:19.129600+0000","last_peered":"2024-08-29T15:40:19.129600+0000","last_clean":"2024-08-29T15:40:19.129600+0000","last_became_active":"2024-08-29T15:39:21.005533+0000","last_became_peered":"2024-08-29T15:39:21.005533+0000","last_unstale":"2024-08-29T15:40:19.129600+0000","last_undegraded":"2024-08-29T15:40:19.129600+0000","last_fullsized":"2024-08-29T15:40:19.129600+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:32:12.424182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":153,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.115922+0000","last_change":"2024-08-29T15:39:53.266248+0000","last_active":"2024-08-29T15:40:19.115922+0000","last_peered":"2024-08-29T15:40:19.115922+0000","last_clean":"2024-08-29T15:40:19.115922+0000","last_became_active":"2024-08-29T15:39:53.258274+0000","last_became_peered":"2024-08-29T15:39:53.258274+0000","last_unstale":"2024-08-29T15:40:19.115922+0000","last_undegraded":"2024-08-29T15:40:19.115922+0000","last_fullsized":"2024-08-29T15:40:19.115922+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:35:50.632965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.5","version":"431'3457","reported_seq":3490,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:23.704443+0000","last_change":"2024-08-29T15:39:37.259396+0000","last_active":"2024-08-29T15:40:23.704443+0000","last_peered":"2024-08-29T15:40:23.704443+0000","last_clean":"2024-08-29T15:40:23.704443+0000","last_became_active":"2024-08-29T15:39:37.259121+0000","last_became_peered":"2024-08-29T15:39:37.259121+0000","last_unstale":"2024-08-29T15:40:23.704443+0000","last_undegraded":"2024-08-29T15:40:23.704443+0000","last_fullsized":"2024-08-29T15:40:23.704443+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3457,"log_dups_size":0,"ondisk_log_size":3457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:09:08.900552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226557952,"num_objects":3457,"num_object_clones":0,"num_object_copies":10371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3457,"num_write_kb":221248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":246,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:09.661197+0000","last_change":"2024-08-29T15:37:26.322453+0000","last_active":"2024-08-29T15:40:09.661197+0000","last_peered":"2024-08-29T15:40:09.661197+0000","last_clean":"2024-08-29T15:40:09.661197+0000","last_became_active":"2024-08-29T15:37:26.308813+0000","last_became_peered":"2024-08-29T15:37:26.308813+0000","last_unstale":"2024-08-29T15:40:09.661197+0000","last_undegraded":"2024-08-29T15:40:09.661197+0000","last_fullsized":"2024-08-29T15:40:09.661197+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:05:55.778279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.027743+0000","last_change":"2024-08-29T15:39:53.258473+0000","last_active":"2024-08-29T15:40:08.027743+0000","last_peered":"2024-08-29T15:40:08.027743+0000","last_clean":"2024-08-29T15:40:08.027743+0000","last_became_active":"2024-08-29T15:39:53.258055+0000","last_became_peered":"2024-08-29T15:39:53.258055+0000","last_unstale":"2024-08-29T15:40:08.027743+0000","last_undegraded":"2024-08-29T15:40:08.027743+0000","last_fullsized":"2024-08-29T15:40:08.027743+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:38:54.441868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"431'3131","reported_seq":3163,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:20.340809+0000","last_change":"2024-08-29T15:39:37.186986+0000","last_active":"2024-08-29T15:40:20.340809+0000","last_peered":"2024-08-29T15:40:20.340809+0000","last_clean":"2024-08-29T15:40:20.340809+0000","last_became_active":"2024-08-29T15:39:37.186669+0000","last_became_peered":"2024-08-29T15:39:37.186669+0000","last_unstale":"2024-08-29T15:40:20.340809+0000","last_undegraded":"2024-08-29T15:40:20.340809+0000","last_fullsized":"2024-08-29T15:40:20.340809+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3131,"log_dups_size":0,"ondisk_log_size":3131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:33:01.221948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205193216,"num_objects":3131,"num_object_clones":0,"num_object_copies":9393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3131,"num_write_kb":200384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":220,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039496+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:40:08.039496+0000","last_peered":"2024-08-29T15:40:08.039496+0000","last_clean":"2024-08-29T15:40:08.039496+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:40:08.039496+0000","last_undegraded":"2024-08-29T15:40:08.039496+0000","last_fullsized":"2024-08-29T15:40:08.039496+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":243,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.038814+0000","last_change":"2024-08-29T15:39:32.183389+0000","last_active":"2024-08-29T15:40:08.038814+0000","last_peered":"2024-08-29T15:40:08.038814+0000","last_clean":"2024-08-29T15:40:08.038814+0000","last_became_active":"2024-08-29T15:39:32.183069+0000","last_became_peered":"2024-08-29T15:39:32.183069+0000","last_unstale":"2024-08-29T15:40:08.038814+0000","last_undegraded":"2024-08-29T15:40:08.038814+0000","last_fullsized":"2024-08-29T15:40:08.038814+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:13:42.679657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"431'3199","reported_seq":3219,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.935071+0000","last_change":"2024-08-29T15:39:37.258750+0000","last_active":"2024-08-29T15:40:21.935071+0000","last_peered":"2024-08-29T15:40:21.935071+0000","last_clean":"2024-08-29T15:40:21.935071+0000","last_became_active":"2024-08-29T15:39:37.258439+0000","last_became_peered":"2024-08-29T15:39:37.258439+0000","last_unstale":"2024-08-29T15:40:21.935071+0000","last_undegraded":"2024-08-29T15:40:21.935071+0000","last_fullsized":"2024-08-29T15:40:21.935071+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3199,"log_dups_size":0,"ondisk_log_size":3199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:43:55.933600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209649664,"num_objects":3199,"num_object_clones":0,"num_object_copies":9597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3199,"num_write_kb":204736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":21,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.028928+0000","last_change":"2024-08-29T15:39:53.262167+0000","last_active":"2024-08-29T15:40:08.028928+0000","last_peered":"2024-08-29T15:40:08.028928+0000","last_clean":"2024-08-29T15:40:08.028928+0000","last_became_active":"2024-08-29T15:39:53.258406+0000","last_became_peered":"2024-08-29T15:39:53.258406+0000","last_unstale":"2024-08-29T15:40:08.028928+0000","last_undegraded":"2024-08-29T15:40:08.028928+0000","last_fullsized":"2024-08-29T15:40:08.028928+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:05:19.731559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":354,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.122611+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:40:19.122611+0000","last_peered":"2024-08-29T15:40:19.122611+0000","last_clean":"2024-08-29T15:40:19.122611+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:40:19.122611+0000","last_undegraded":"2024-08-29T15:40:19.122611+0000","last_fullsized":"2024-08-29T15:40:19.122611+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"431'3199","reported_seq":3219,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.942599+0000","last_change":"2024-08-29T15:39:37.376667+0000","last_active":"2024-08-29T15:40:21.942599+0000","last_peered":"2024-08-29T15:40:21.942599+0000","last_clean":"2024-08-29T15:40:21.942599+0000","last_became_active":"2024-08-29T15:39:37.376372+0000","last_became_peered":"2024-08-29T15:39:37.376372+0000","last_unstale":"2024-08-29T15:40:21.942599+0000","last_undegraded":"2024-08-29T15:40:21.942599+0000","last_fullsized":"2024-08-29T15:40:21.942599+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3199,"log_dups_size":0,"ondisk_log_size":3199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:52:02.092878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209649664,"num_objects":3199,"num_object_clones":0,"num_object_copies":9597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3199,"num_write_kb":204736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":324,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.029290+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:40:08.029290+0000","last_peered":"2024-08-29T15:40:08.029290+0000","last_clean":"2024-08-29T15:40:08.029290+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:40:08.029290+0000","last_undegraded":"2024-08-29T15:40:08.029290+0000","last_fullsized":"2024-08-29T15:40:08.029290+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":733,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.115113+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:40:19.115113+0000","last_peered":"2024-08-29T15:40:19.115113+0000","last_clean":"2024-08-29T15:40:19.115113+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:40:19.115113+0000","last_undegraded":"2024-08-29T15:40:19.115113+0000","last_fullsized":"2024-08-29T15:40:19.115113+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"431'3192","reported_seq":3225,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.494325+0000","last_change":"2024-08-29T15:39:37.311470+0000","last_active":"2024-08-29T15:40:21.494325+0000","last_peered":"2024-08-29T15:40:21.494325+0000","last_clean":"2024-08-29T15:40:21.494325+0000","last_became_active":"2024-08-29T15:39:37.310913+0000","last_became_peered":"2024-08-29T15:39:37.310913+0000","last_unstale":"2024-08-29T15:40:21.494325+0000","last_undegraded":"2024-08-29T15:40:21.494325+0000","last_fullsized":"2024-08-29T15:40:21.494325+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3192,"log_dups_size":0,"ondisk_log_size":3192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:39.550827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209190912,"num_objects":3192,"num_object_clones":0,"num_object_copies":9576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3192,"num_write_kb":204288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":856,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.134660+0000","last_change":"2024-08-29T15:37:18.101543+0000","last_active":"2024-08-29T15:40:19.134660+0000","last_peered":"2024-08-29T15:40:19.134660+0000","last_clean":"2024-08-29T15:40:19.134660+0000","last_became_active":"2024-08-29T15:37:01.308140+0000","last_became_peered":"2024-08-29T15:37:01.308140+0000","last_unstale":"2024-08-29T15:40:19.134660+0000","last_undegraded":"2024-08-29T15:40:19.134660+0000","last_fullsized":"2024-08-29T15:40:19.134660+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:18.101472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:18.101472+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:06:29.187841+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":750,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.130561+0000","last_change":"2024-08-29T15:37:26.340871+0000","last_active":"2024-08-29T15:40:19.130561+0000","last_peered":"2024-08-29T15:40:19.130561+0000","last_clean":"2024-08-29T15:40:19.130561+0000","last_became_active":"2024-08-29T15:37:26.328280+0000","last_became_peered":"2024-08-29T15:37:26.328280+0000","last_unstale":"2024-08-29T15:40:19.130561+0000","last_undegraded":"2024-08-29T15:40:19.130561+0000","last_fullsized":"2024-08-29T15:40:19.130561+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:47:01.239361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":216,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039151+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:40:08.039151+0000","last_peered":"2024-08-29T15:40:08.039151+0000","last_clean":"2024-08-29T15:40:08.039151+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:40:08.039151+0000","last_undegraded":"2024-08-29T15:40:08.039151+0000","last_fullsized":"2024-08-29T15:40:08.039151+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"431'2977","reported_seq":3009,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:20.349939+0000","last_change":"2024-08-29T15:39:37.376626+0000","last_active":"2024-08-29T15:40:20.349939+0000","last_peered":"2024-08-29T15:40:20.349939+0000","last_clean":"2024-08-29T15:40:20.349939+0000","last_became_active":"2024-08-29T15:39:37.376318+0000","last_became_peered":"2024-08-29T15:39:37.376318+0000","last_unstale":"2024-08-29T15:40:20.349939+0000","last_undegraded":"2024-08-29T15:40:20.349939+0000","last_fullsized":"2024-08-29T15:40:20.349939+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":2977,"log_dups_size":0,"ondisk_log_size":2977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:45.198689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195100672,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2977,"num_write_kb":190528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":725,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.404407+0000","last_change":"2024-08-29T15:40:21.405507+0000","last_active":"2024-08-29T15:40:21.404407+0000","last_peered":"2024-08-29T15:40:21.404407+0000","last_clean":"2024-08-29T15:40:21.404407+0000","last_became_active":"2024-08-29T15:39:56.271696+0000","last_became_peered":"2024-08-29T15:39:56.271696+0000","last_unstale":"2024-08-29T15:40:21.404407+0000","last_undegraded":"2024-08-29T15:40:21.404407+0000","last_fullsized":"2024-08-29T15:40:21.404407+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:40:21.404385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:40:21.404385+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:35.409438+0000","scrub_duration":0.04293052,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":202,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.038962+0000","last_change":"2024-08-29T15:39:01.753999+0000","last_active":"2024-08-29T15:40:08.038962+0000","last_peered":"2024-08-29T15:40:08.038962+0000","last_clean":"2024-08-29T15:40:08.038962+0000","last_became_active":"2024-08-29T15:39:01.753603+0000","last_became_peered":"2024-08-29T15:39:01.753603+0000","last_unstale":"2024-08-29T15:40:08.038962+0000","last_undegraded":"2024-08-29T15:40:08.038962+0000","last_fullsized":"2024-08-29T15:40:08.038962+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:23:55.741058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":216,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039431+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:40:08.039431+0000","last_peered":"2024-08-29T15:40:08.039431+0000","last_clean":"2024-08-29T15:40:08.039431+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:40:08.039431+0000","last_undegraded":"2024-08-29T15:40:08.039431+0000","last_fullsized":"2024-08-29T15:40:08.039431+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"431'3197","reported_seq":3230,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.494054+0000","last_change":"2024-08-29T15:39:37.258693+0000","last_active":"2024-08-29T15:40:21.494054+0000","last_peered":"2024-08-29T15:40:21.494054+0000","last_clean":"2024-08-29T15:40:21.494054+0000","last_became_active":"2024-08-29T15:39:37.258303+0000","last_became_peered":"2024-08-29T15:39:37.258303+0000","last_unstale":"2024-08-29T15:40:21.494054+0000","last_undegraded":"2024-08-29T15:40:21.494054+0000","last_fullsized":"2024-08-29T15:40:21.494054+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3197,"log_dups_size":0,"ondisk_log_size":3197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:53:38.911068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518592,"num_objects":3197,"num_object_clones":0,"num_object_copies":9591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3197,"num_write_kb":204608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":216,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.038855+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:40:08.038855+0000","last_peered":"2024-08-29T15:40:08.038855+0000","last_clean":"2024-08-29T15:40:08.038855+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:40:08.038855+0000","last_undegraded":"2024-08-29T15:40:08.038855+0000","last_fullsized":"2024-08-29T15:40:08.038855+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":439,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039505+0000","last_change":"2024-08-29T15:39:01.753897+0000","last_active":"2024-08-29T15:40:08.039505+0000","last_peered":"2024-08-29T15:40:08.039505+0000","last_clean":"2024-08-29T15:40:08.039505+0000","last_became_active":"2024-08-29T15:39:01.753534+0000","last_became_peered":"2024-08-29T15:39:01.753534+0000","last_unstale":"2024-08-29T15:40:08.039505+0000","last_undegraded":"2024-08-29T15:40:08.039505+0000","last_fullsized":"2024-08-29T15:40:08.039505+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:11:01.816741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"431'3223","reported_seq":3256,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.505578+0000","last_change":"2024-08-29T15:39:37.376745+0000","last_active":"2024-08-29T15:40:21.505578+0000","last_peered":"2024-08-29T15:40:21.505578+0000","last_clean":"2024-08-29T15:40:21.505578+0000","last_became_active":"2024-08-29T15:39:37.376397+0000","last_became_peered":"2024-08-29T15:39:37.376397+0000","last_unstale":"2024-08-29T15:40:21.505578+0000","last_undegraded":"2024-08-29T15:40:21.505578+0000","last_fullsized":"2024-08-29T15:40:21.505578+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3223,"log_dups_size":0,"ondisk_log_size":3223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:32:13.118834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211222528,"num_objects":3223,"num_object_clones":0,"num_object_copies":9669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3223,"num_write_kb":206272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":785,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.029057+0000","last_change":"2024-08-29T15:37:24.681545+0000","last_active":"2024-08-29T15:40:08.029057+0000","last_peered":"2024-08-29T15:40:08.029057+0000","last_clean":"2024-08-29T15:40:08.029057+0000","last_became_active":"2024-08-29T15:37:01.312787+0000","last_became_peered":"2024-08-29T15:37:01.312787+0000","last_unstale":"2024-08-29T15:40:08.029057+0000","last_undegraded":"2024-08-29T15:40:08.029057+0000","last_fullsized":"2024-08-29T15:40:08.029057+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:24.681392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:24.681392+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:01:45.435539+0000","scrub_duration":0.0064019280000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":52,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.130079+0000","last_change":"2024-08-29T15:39:21.002903+0000","last_active":"2024-08-29T15:40:19.130079+0000","last_peered":"2024-08-29T15:40:19.130079+0000","last_clean":"2024-08-29T15:40:19.130079+0000","last_became_active":"2024-08-29T15:39:21.002543+0000","last_became_peered":"2024-08-29T15:39:21.002543+0000","last_unstale":"2024-08-29T15:40:19.130079+0000","last_undegraded":"2024-08-29T15:40:19.130079+0000","last_fullsized":"2024-08-29T15:40:19.130079+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:57:08.725163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":723,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.118761+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:40:19.118761+0000","last_peered":"2024-08-29T15:40:19.118761+0000","last_clean":"2024-08-29T15:40:19.118761+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:40:19.118761+0000","last_undegraded":"2024-08-29T15:40:19.118761+0000","last_fullsized":"2024-08-29T15:40:19.118761+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"431'3210","reported_seq":3248,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.502287+0000","last_change":"2024-08-29T15:39:53.258499+0000","last_active":"2024-08-29T15:40:21.502287+0000","last_peered":"2024-08-29T15:40:21.502287+0000","last_clean":"2024-08-29T15:40:21.502287+0000","last_became_active":"2024-08-29T15:39:53.257102+0000","last_became_peered":"2024-08-29T15:39:53.257102+0000","last_unstale":"2024-08-29T15:40:21.502287+0000","last_undegraded":"2024-08-29T15:40:21.502287+0000","last_fullsized":"2024-08-29T15:40:21.502287+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3210,"log_dups_size":0,"ondisk_log_size":3210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:19:54.489027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210370560,"num_objects":3210,"num_object_clones":0,"num_object_copies":9630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3210,"num_write_kb":205440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":855,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.115654+0000","last_change":"2024-08-29T15:37:19.170088+0000","last_active":"2024-08-29T15:40:19.115654+0000","last_peered":"2024-08-29T15:40:19.115654+0000","last_clean":"2024-08-29T15:40:19.115654+0000","last_became_active":"2024-08-29T15:36:54.189617+0000","last_became_peered":"2024-08-29T15:36:54.189617+0000","last_unstale":"2024-08-29T15:40:19.115654+0000","last_undegraded":"2024-08-29T15:40:19.115654+0000","last_fullsized":"2024-08-29T15:40:19.115654+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.170014+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.170014+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:55:52.939920+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":908,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.028833+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:40:08.028833+0000","last_peered":"2024-08-29T15:40:08.028833+0000","last_clean":"2024-08-29T15:40:08.028833+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:40:08.028833+0000","last_undegraded":"2024-08-29T15:40:08.028833+0000","last_fullsized":"2024-08-29T15:40:08.028833+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":137,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.129079+0000","last_change":"2024-08-29T15:37:26.313495+0000","last_active":"2024-08-29T15:40:19.129079+0000","last_peered":"2024-08-29T15:40:19.129079+0000","last_clean":"2024-08-29T15:40:19.129079+0000","last_became_active":"2024-08-29T15:37:26.313188+0000","last_became_peered":"2024-08-29T15:37:26.313188+0000","last_unstale":"2024-08-29T15:40:19.129079+0000","last_undegraded":"2024-08-29T15:40:19.129079+0000","last_fullsized":"2024-08-29T15:40:19.129079+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:39.580490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":182,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039260+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:40:08.039260+0000","last_peered":"2024-08-29T15:40:08.039260+0000","last_clean":"2024-08-29T15:40:08.039260+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:40:08.039260+0000","last_undegraded":"2024-08-29T15:40:08.039260+0000","last_fullsized":"2024-08-29T15:40:08.039260+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"431'3402","reported_seq":3435,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:23.679561+0000","last_change":"2024-08-29T15:39:37.311691+0000","last_active":"2024-08-29T15:40:23.679561+0000","last_peered":"2024-08-29T15:40:23.679561+0000","last_clean":"2024-08-29T15:40:23.679561+0000","last_became_active":"2024-08-29T15:39:37.311136+0000","last_became_peered":"2024-08-29T15:39:37.311136+0000","last_unstale":"2024-08-29T15:40:23.679561+0000","last_undegraded":"2024-08-29T15:40:23.679561+0000","last_fullsized":"2024-08-29T15:40:23.679561+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3402,"log_dups_size":0,"ondisk_log_size":3402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:40:42.703845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3402,"num_write_kb":217728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":855,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.117931+0000","last_change":"2024-08-29T15:37:19.656177+0000","last_active":"2024-08-29T15:40:19.117931+0000","last_peered":"2024-08-29T15:40:19.117931+0000","last_clean":"2024-08-29T15:40:19.117931+0000","last_became_active":"2024-08-29T15:37:01.322473+0000","last_became_peered":"2024-08-29T15:37:01.322473+0000","last_unstale":"2024-08-29T15:40:19.117931+0000","last_undegraded":"2024-08-29T15:40:19.117931+0000","last_fullsized":"2024-08-29T15:40:19.117931+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.654978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.654978+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:21:16.623191+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1197,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.118101+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:40:19.118101+0000","last_peered":"2024-08-29T15:40:19.118101+0000","last_clean":"2024-08-29T15:40:19.118101+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:40:19.118101+0000","last_undegraded":"2024-08-29T15:40:19.118101+0000","last_fullsized":"2024-08-29T15:40:19.118101+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":748,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.113317+0000","last_change":"2024-08-29T15:39:21.003942+0000","last_active":"2024-08-29T15:40:19.113317+0000","last_peered":"2024-08-29T15:40:19.113317+0000","last_clean":"2024-08-29T15:40:19.113317+0000","last_became_active":"2024-08-29T15:39:21.003471+0000","last_became_peered":"2024-08-29T15:39:21.003471+0000","last_unstale":"2024-08-29T15:40:19.113317+0000","last_undegraded":"2024-08-29T15:40:19.113317+0000","last_fullsized":"2024-08-29T15:40:19.113317+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:34:44.102930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":93,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114415+0000","last_change":"2024-08-29T15:38:27.108492+0000","last_active":"2024-08-29T15:40:19.114415+0000","last_peered":"2024-08-29T15:40:19.114415+0000","last_clean":"2024-08-29T15:40:19.114415+0000","last_became_active":"2024-08-29T15:38:27.108046+0000","last_became_peered":"2024-08-29T15:38:27.108046+0000","last_unstale":"2024-08-29T15:40:19.114415+0000","last_undegraded":"2024-08-29T15:40:19.114415+0000","last_fullsized":"2024-08-29T15:40:19.114415+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:22.265093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"431'3286","reported_seq":3302,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.961665+0000","last_change":"2024-08-29T15:39:53.258131+0000","last_active":"2024-08-29T15:40:21.961665+0000","last_peered":"2024-08-29T15:40:21.961665+0000","last_clean":"2024-08-29T15:40:21.961665+0000","last_became_active":"2024-08-29T15:39:53.257746+0000","last_became_peered":"2024-08-29T15:39:53.257746+0000","last_unstale":"2024-08-29T15:40:21.961665+0000","last_undegraded":"2024-08-29T15:40:21.961665+0000","last_fullsized":"2024-08-29T15:40:21.961665+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3286,"log_dups_size":0,"ondisk_log_size":3286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:48.746732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215351296,"num_objects":3286,"num_object_clones":0,"num_object_copies":9858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3286,"num_write_kb":210304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":863,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.115400+0000","last_change":"2024-08-29T15:37:18.155224+0000","last_active":"2024-08-29T15:40:19.115400+0000","last_peered":"2024-08-29T15:40:19.115400+0000","last_clean":"2024-08-29T15:40:19.115400+0000","last_became_active":"2024-08-29T15:37:01.308472+0000","last_became_peered":"2024-08-29T15:37:01.308472+0000","last_unstale":"2024-08-29T15:40:19.115400+0000","last_undegraded":"2024-08-29T15:40:19.115400+0000","last_fullsized":"2024-08-29T15:40:19.115400+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:18.154828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:18.154828+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:55.271955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":765,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.134922+0000","last_change":"2024-08-29T15:37:29.345555+0000","last_active":"2024-08-29T15:40:19.134922+0000","last_peered":"2024-08-29T15:40:19.134922+0000","last_clean":"2024-08-29T15:40:19.134922+0000","last_became_active":"2024-08-29T15:37:29.344738+0000","last_became_peered":"2024-08-29T15:37:29.344738+0000","last_unstale":"2024-08-29T15:40:19.134922+0000","last_undegraded":"2024-08-29T15:40:19.134922+0000","last_fullsized":"2024-08-29T15:40:19.134922+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:14.103195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.032785+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:40:08.032785+0000","last_peered":"2024-08-29T15:40:08.032785+0000","last_clean":"2024-08-29T15:40:08.032785+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:40:08.032785+0000","last_undegraded":"2024-08-29T15:40:08.032785+0000","last_fullsized":"2024-08-29T15:40:08.032785+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.118504+0000","last_change":"2024-08-29T15:39:01.753014+0000","last_active":"2024-08-29T15:40:19.118504+0000","last_peered":"2024-08-29T15:40:19.118504+0000","last_clean":"2024-08-29T15:40:19.118504+0000","last_became_active":"2024-08-29T15:39:01.752682+0000","last_became_peered":"2024-08-29T15:39:01.752682+0000","last_unstale":"2024-08-29T15:40:19.118504+0000","last_undegraded":"2024-08-29T15:40:19.118504+0000","last_fullsized":"2024-08-29T15:40:19.118504+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:00:06.838149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"431'3369","reported_seq":3402,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:23.679134+0000","last_change":"2024-08-29T15:39:37.258623+0000","last_active":"2024-08-29T15:40:23.679134+0000","last_peered":"2024-08-29T15:40:23.679134+0000","last_clean":"2024-08-29T15:40:23.679134+0000","last_became_active":"2024-08-29T15:39:37.258321+0000","last_became_peered":"2024-08-29T15:39:37.258321+0000","last_unstale":"2024-08-29T15:40:23.679134+0000","last_undegraded":"2024-08-29T15:40:23.679134+0000","last_fullsized":"2024-08-29T15:40:23.679134+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3369,"log_dups_size":0,"ondisk_log_size":3369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:09:19.337857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220790784,"num_objects":3369,"num_object_clones":0,"num_object_copies":10107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3369,"num_write_kb":215616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":784,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.028855+0000","last_change":"2024-08-29T15:37:23.716571+0000","last_active":"2024-08-29T15:40:08.028855+0000","last_peered":"2024-08-29T15:40:08.028855+0000","last_clean":"2024-08-29T15:40:08.028855+0000","last_became_active":"2024-08-29T15:37:01.308514+0000","last_became_peered":"2024-08-29T15:37:01.308514+0000","last_unstale":"2024-08-29T15:40:08.028855+0000","last_undegraded":"2024-08-29T15:40:08.028855+0000","last_fullsized":"2024-08-29T15:40:08.028855+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:23.716435+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:23.716435+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:18:15.213865+0000","scrub_duration":0.022267417000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":376,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.029091+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:40:08.029091+0000","last_peered":"2024-08-29T15:40:08.029091+0000","last_clean":"2024-08-29T15:40:08.029091+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:40:08.029091+0000","last_undegraded":"2024-08-29T15:40:08.029091+0000","last_fullsized":"2024-08-29T15:40:08.029091+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":216,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039223+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:40:08.039223+0000","last_peered":"2024-08-29T15:40:08.039223+0000","last_clean":"2024-08-29T15:40:08.039223+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:40:08.039223+0000","last_undegraded":"2024-08-29T15:40:08.039223+0000","last_fullsized":"2024-08-29T15:40:08.039223+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":283,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.028961+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:40:08.028961+0000","last_peered":"2024-08-29T15:40:08.028961+0000","last_clean":"2024-08-29T15:40:08.028961+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:40:08.028961+0000","last_undegraded":"2024-08-29T15:40:08.028961+0000","last_fullsized":"2024-08-29T15:40:08.028961+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"431'3264","reported_seq":3284,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.939521+0000","last_change":"2024-08-29T15:39:37.185853+0000","last_active":"2024-08-29T15:40:21.939521+0000","last_peered":"2024-08-29T15:40:21.939521+0000","last_clean":"2024-08-29T15:40:21.939521+0000","last_became_active":"2024-08-29T15:39:37.185573+0000","last_became_peered":"2024-08-29T15:39:37.185573+0000","last_unstale":"2024-08-29T15:40:21.939521+0000","last_undegraded":"2024-08-29T15:40:21.939521+0000","last_fullsized":"2024-08-29T15:40:21.939521+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3264,"log_dups_size":0,"ondisk_log_size":3264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:12.924373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213909504,"num_objects":3264,"num_object_clones":0,"num_object_copies":9792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3264,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":815,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.129157+0000","last_change":"2024-08-29T15:39:23.121856+0000","last_active":"2024-08-29T15:40:19.129157+0000","last_peered":"2024-08-29T15:40:19.129157+0000","last_clean":"2024-08-29T15:40:19.129157+0000","last_became_active":"2024-08-29T15:39:23.121484+0000","last_became_peered":"2024-08-29T15:39:23.121484+0000","last_unstale":"2024-08-29T15:40:19.129157+0000","last_undegraded":"2024-08-29T15:40:19.129157+0000","last_fullsized":"2024-08-29T15:40:19.129157+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:16.688893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:16.688893+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:39:33.556710+0000","scrub_duration":0.016432537000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":308,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.029012+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:40:08.029012+0000","last_peered":"2024-08-29T15:40:08.029012+0000","last_clean":"2024-08-29T15:40:08.029012+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:40:08.029012+0000","last_undegraded":"2024-08-29T15:40:08.029012+0000","last_fullsized":"2024-08-29T15:40:08.029012+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1161,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.118609+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:40:19.118609+0000","last_peered":"2024-08-29T15:40:19.118609+0000","last_clean":"2024-08-29T15:40:19.118609+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:40:19.118609+0000","last_undegraded":"2024-08-29T15:40:19.118609+0000","last_fullsized":"2024-08-29T15:40:19.118609+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.122523+0000","last_change":"2024-08-29T15:39:53.328424+0000","last_active":"2024-08-29T15:40:19.122523+0000","last_peered":"2024-08-29T15:40:19.122523+0000","last_clean":"2024-08-29T15:40:19.122523+0000","last_became_active":"2024-08-29T15:39:53.328189+0000","last_became_peered":"2024-08-29T15:39:53.328189+0000","last_unstale":"2024-08-29T15:40:19.122523+0000","last_undegraded":"2024-08-29T15:40:19.122523+0000","last_fullsized":"2024-08-29T15:40:19.122523+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:51:04.926614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"431'3408","reported_seq":3427,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:24.655971+0000","last_change":"2024-08-29T15:39:37.258863+0000","last_active":"2024-08-29T15:40:24.655971+0000","last_peered":"2024-08-29T15:40:24.655971+0000","last_clean":"2024-08-29T15:40:24.655971+0000","last_became_active":"2024-08-29T15:39:37.258296+0000","last_became_peered":"2024-08-29T15:39:37.258296+0000","last_unstale":"2024-08-29T15:40:24.655971+0000","last_undegraded":"2024-08-29T15:40:24.655971+0000","last_fullsized":"2024-08-29T15:40:24.655971+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3408,"log_dups_size":0,"ondisk_log_size":3408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:32:01.990004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223346688,"num_objects":3408,"num_object_clones":0,"num_object_copies":10224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3408,"num_write_kb":218112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":867,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114562+0000","last_change":"2024-08-29T15:37:40.133670+0000","last_active":"2024-08-29T15:40:19.114562+0000","last_peered":"2024-08-29T15:40:19.114562+0000","last_clean":"2024-08-29T15:40:19.114562+0000","last_became_active":"2024-08-29T15:37:40.132923+0000","last_became_peered":"2024-08-29T15:37:40.132923+0000","last_unstale":"2024-08-29T15:40:19.114562+0000","last_undegraded":"2024-08-29T15:40:19.114562+0000","last_fullsized":"2024-08-29T15:40:19.114562+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:15.111260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:15.111260+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:27.886225+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":743,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.118520+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:40:19.118520+0000","last_peered":"2024-08-29T15:40:19.118520+0000","last_clean":"2024-08-29T15:40:19.118520+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:40:19.118520+0000","last_undegraded":"2024-08-29T15:40:19.118520+0000","last_fullsized":"2024-08-29T15:40:19.118520+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":889,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.119864+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:40:19.119864+0000","last_peered":"2024-08-29T15:40:19.119864+0000","last_clean":"2024-08-29T15:40:19.119864+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:40:19.119864+0000","last_undegraded":"2024-08-29T15:40:19.119864+0000","last_fullsized":"2024-08-29T15:40:19.119864+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":734,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114916+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:40:19.114916+0000","last_peered":"2024-08-29T15:40:19.114916+0000","last_clean":"2024-08-29T15:40:19.114916+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:40:19.114916+0000","last_undegraded":"2024-08-29T15:40:19.114916+0000","last_fullsized":"2024-08-29T15:40:19.114916+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"431'3097","reported_seq":3129,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:20.338308+0000","last_change":"2024-08-29T15:39:37.259218+0000","last_active":"2024-08-29T15:40:20.338308+0000","last_peered":"2024-08-29T15:40:20.338308+0000","last_clean":"2024-08-29T15:40:20.338308+0000","last_became_active":"2024-08-29T15:39:37.258907+0000","last_became_peered":"2024-08-29T15:39:37.258907+0000","last_unstale":"2024-08-29T15:40:20.338308+0000","last_undegraded":"2024-08-29T15:40:20.338308+0000","last_fullsized":"2024-08-29T15:40:20.338308+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3097,"log_dups_size":0,"ondisk_log_size":3097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:20:07.328162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202964992,"num_objects":3097,"num_object_clones":0,"num_object_copies":9291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3097,"num_write_kb":198208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":801,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039337+0000","last_change":"2024-08-29T15:39:14.809022+0000","last_active":"2024-08-29T15:40:08.039337+0000","last_peered":"2024-08-29T15:40:08.039337+0000","last_clean":"2024-08-29T15:40:08.039337+0000","last_became_active":"2024-08-29T15:39:14.808645+0000","last_became_peered":"2024-08-29T15:39:14.808645+0000","last_unstale":"2024-08-29T15:40:08.039337+0000","last_undegraded":"2024-08-29T15:40:08.039337+0000","last_fullsized":"2024-08-29T15:40:08.039337+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.644669+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.644669+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:51.600506+0000","scrub_duration":0.0030306299999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":728,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.129803+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:40:19.129803+0000","last_peered":"2024-08-29T15:40:19.129803+0000","last_clean":"2024-08-29T15:40:19.129803+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:40:19.129803+0000","last_undegraded":"2024-08-29T15:40:19.129803+0000","last_fullsized":"2024-08-29T15:40:19.129803+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1380,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.119271+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:40:19.119271+0000","last_peered":"2024-08-29T15:40:19.119271+0000","last_clean":"2024-08-29T15:40:19.119271+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:40:19.119271+0000","last_undegraded":"2024-08-29T15:40:19.119271+0000","last_fullsized":"2024-08-29T15:40:19.119271+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":52,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.129761+0000","last_change":"2024-08-29T15:39:21.005897+0000","last_active":"2024-08-29T15:40:19.129761+0000","last_peered":"2024-08-29T15:40:19.129761+0000","last_clean":"2024-08-29T15:40:19.129761+0000","last_became_active":"2024-08-29T15:39:21.005125+0000","last_became_peered":"2024-08-29T15:39:21.005125+0000","last_unstale":"2024-08-29T15:40:19.129761+0000","last_undegraded":"2024-08-29T15:40:19.129761+0000","last_fullsized":"2024-08-29T15:40:19.129761+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:44:20.271835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"431'3235","reported_seq":3268,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.496906+0000","last_change":"2024-08-29T15:39:37.259666+0000","last_active":"2024-08-29T15:40:21.496906+0000","last_peered":"2024-08-29T15:40:21.496906+0000","last_clean":"2024-08-29T15:40:21.496906+0000","last_became_active":"2024-08-29T15:39:37.259380+0000","last_became_peered":"2024-08-29T15:39:37.259380+0000","last_unstale":"2024-08-29T15:40:21.496906+0000","last_undegraded":"2024-08-29T15:40:21.496906+0000","last_fullsized":"2024-08-29T15:40:21.496906+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":3235,"log_dups_size":0,"ondisk_log_size":3235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:02:18.132955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212008960,"num_objects":3235,"num_object_clones":0,"num_object_copies":9705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3235,"num_write_kb":207040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":887,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.129736+0000","last_change":"2024-08-29T15:39:56.274024+0000","last_active":"2024-08-29T15:40:19.129736+0000","last_peered":"2024-08-29T15:40:19.129736+0000","last_clean":"2024-08-29T15:40:19.129736+0000","last_became_active":"2024-08-29T15:39:56.273578+0000","last_became_peered":"2024-08-29T15:39:56.273578+0000","last_unstale":"2024-08-29T15:40:19.129736+0000","last_undegraded":"2024-08-29T15:40:19.129736+0000","last_fullsized":"2024-08-29T15:40:19.129736+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:20.077258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:20.077258+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:20.595923+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":140,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114673+0000","last_change":"2024-08-29T15:37:26.328428+0000","last_active":"2024-08-29T15:40:19.114673+0000","last_peered":"2024-08-29T15:40:19.114673+0000","last_clean":"2024-08-29T15:40:19.114673+0000","last_became_active":"2024-08-29T15:37:26.313757+0000","last_became_peered":"2024-08-29T15:37:26.313757+0000","last_unstale":"2024-08-29T15:40:19.114673+0000","last_undegraded":"2024-08-29T15:40:19.114673+0000","last_fullsized":"2024-08-29T15:40:19.114673+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:29:31.366546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":289,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.035931+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:40:08.035931+0000","last_peered":"2024-08-29T15:40:08.035931+0000","last_clean":"2024-08-29T15:40:08.035931+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:40:08.035931+0000","last_undegraded":"2024-08-29T15:40:08.035931+0000","last_fullsized":"2024-08-29T15:40:08.035931+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":216,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.038877+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:40:08.038877+0000","last_peered":"2024-08-29T15:40:08.038877+0000","last_clean":"2024-08-29T15:40:08.038877+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:40:08.038877+0000","last_undegraded":"2024-08-29T15:40:08.038877+0000","last_fullsized":"2024-08-29T15:40:08.038877+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":215,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039303+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:40:08.039303+0000","last_peered":"2024-08-29T15:40:08.039303+0000","last_clean":"2024-08-29T15:40:08.039303+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:40:08.039303+0000","last_undegraded":"2024-08-29T15:40:08.039303+0000","last_fullsized":"2024-08-29T15:40:08.039303+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":225,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039265+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:40:08.039265+0000","last_peered":"2024-08-29T15:40:08.039265+0000","last_clean":"2024-08-29T15:40:08.039265+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:40:08.039265+0000","last_undegraded":"2024-08-29T15:40:08.039265+0000","last_fullsized":"2024-08-29T15:40:08.039265+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.122130+0000","last_change":"2024-08-29T15:39:01.752879+0000","last_active":"2024-08-29T15:40:19.122130+0000","last_peered":"2024-08-29T15:40:19.122130+0000","last_clean":"2024-08-29T15:40:19.122130+0000","last_became_active":"2024-08-29T15:39:01.752227+0000","last_became_peered":"2024-08-29T15:39:01.752227+0000","last_unstale":"2024-08-29T15:40:19.122130+0000","last_undegraded":"2024-08-29T15:40:19.122130+0000","last_fullsized":"2024-08-29T15:40:19.122130+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:25:05.582312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":491,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.128897+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:40:19.128897+0000","last_peered":"2024-08-29T15:40:19.128897+0000","last_clean":"2024-08-29T15:40:19.128897+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:40:19.128897+0000","last_undegraded":"2024-08-29T15:40:19.128897+0000","last_fullsized":"2024-08-29T15:40:19.128897+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":305,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.029003+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:40:08.029003+0000","last_peered":"2024-08-29T15:40:08.029003+0000","last_clean":"2024-08-29T15:40:08.029003+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:40:08.029003+0000","last_undegraded":"2024-08-29T15:40:08.029003+0000","last_fullsized":"2024-08-29T15:40:08.029003+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":691,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.134763+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:40:19.134763+0000","last_peered":"2024-08-29T15:40:19.134763+0000","last_clean":"2024-08-29T15:40:19.134763+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:40:19.134763+0000","last_undegraded":"2024-08-29T15:40:19.134763+0000","last_fullsized":"2024-08-29T15:40:19.134763+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":744,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.115161+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:40:19.115161+0000","last_peered":"2024-08-29T15:40:19.115161+0000","last_clean":"2024-08-29T15:40:19.115161+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:40:19.115161+0000","last_undegraded":"2024-08-29T15:40:19.115161+0000","last_fullsized":"2024-08-29T15:40:19.115161+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":743,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.130212+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:40:19.130212+0000","last_peered":"2024-08-29T15:40:19.130212+0000","last_clean":"2024-08-29T15:40:19.130212+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:40:19.130212+0000","last_undegraded":"2024-08-29T15:40:19.130212+0000","last_fullsized":"2024-08-29T15:40:19.130212+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":286,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.029213+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:40:08.029213+0000","last_peered":"2024-08-29T15:40:08.029213+0000","last_clean":"2024-08-29T15:40:08.029213+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:40:08.029213+0000","last_undegraded":"2024-08-29T15:40:08.029213+0000","last_fullsized":"2024-08-29T15:40:08.029213+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":250,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.115561+0000","last_change":"2024-08-29T15:39:01.755284+0000","last_active":"2024-08-29T15:40:19.115561+0000","last_peered":"2024-08-29T15:40:19.115561+0000","last_clean":"2024-08-29T15:40:19.115561+0000","last_became_active":"2024-08-29T15:39:01.754638+0000","last_became_peered":"2024-08-29T15:39:01.754638+0000","last_unstale":"2024-08-29T15:40:19.115561+0000","last_undegraded":"2024-08-29T15:40:19.115561+0000","last_fullsized":"2024-08-29T15:40:19.115561+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:49:04.459879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":742,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114880+0000","last_change":"2024-08-29T15:39:21.004535+0000","last_active":"2024-08-29T15:40:19.114880+0000","last_peered":"2024-08-29T15:40:19.114880+0000","last_clean":"2024-08-29T15:40:19.114880+0000","last_became_active":"2024-08-29T15:39:21.001651+0000","last_became_peered":"2024-08-29T15:39:21.001651+0000","last_unstale":"2024-08-29T15:40:19.114880+0000","last_undegraded":"2024-08-29T15:40:19.114880+0000","last_fullsized":"2024-08-29T15:40:19.114880+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:20:18.607584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":723,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.117952+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:40:19.117952+0000","last_peered":"2024-08-29T15:40:19.117952+0000","last_clean":"2024-08-29T15:40:19.117952+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:40:19.117952+0000","last_undegraded":"2024-08-29T15:40:19.117952+0000","last_fullsized":"2024-08-29T15:40:19.117952+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":727,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.128689+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:40:19.128689+0000","last_peered":"2024-08-29T15:40:19.128689+0000","last_clean":"2024-08-29T15:40:19.128689+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:40:19.128689+0000","last_undegraded":"2024-08-29T15:40:19.128689+0000","last_fullsized":"2024-08-29T15:40:19.128689+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":215,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039039+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:40:08.039039+0000","last_peered":"2024-08-29T15:40:08.039039+0000","last_clean":"2024-08-29T15:40:08.039039+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:40:08.039039+0000","last_undegraded":"2024-08-29T15:40:08.039039+0000","last_fullsized":"2024-08-29T15:40:08.039039+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":768,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.118696+0000","last_change":"2024-08-29T15:39:11.832387+0000","last_active":"2024-08-29T15:40:19.118696+0000","last_peered":"2024-08-29T15:40:19.118696+0000","last_clean":"2024-08-29T15:40:19.118696+0000","last_became_active":"2024-08-29T15:39:11.832056+0000","last_became_peered":"2024-08-29T15:39:11.832056+0000","last_unstale":"2024-08-29T15:40:19.118696+0000","last_undegraded":"2024-08-29T15:40:19.118696+0000","last_fullsized":"2024-08-29T15:40:19.118696+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:47:42.049015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":93,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114603+0000","last_change":"2024-08-29T15:38:27.106157+0000","last_active":"2024-08-29T15:40:19.114603+0000","last_peered":"2024-08-29T15:40:19.114603+0000","last_clean":"2024-08-29T15:40:19.114603+0000","last_became_active":"2024-08-29T15:38:27.105635+0000","last_became_peered":"2024-08-29T15:38:27.105635+0000","last_unstale":"2024-08-29T15:40:19.114603+0000","last_undegraded":"2024-08-29T15:40:19.114603+0000","last_fullsized":"2024-08-29T15:40:19.114603+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:27.182762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":233,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039112+0000","last_change":"2024-08-29T15:39:01.753766+0000","last_active":"2024-08-29T15:40:08.039112+0000","last_peered":"2024-08-29T15:40:08.039112+0000","last_clean":"2024-08-29T15:40:08.039112+0000","last_became_active":"2024-08-29T15:39:01.753447+0000","last_became_peered":"2024-08-29T15:39:01.753447+0000","last_unstale":"2024-08-29T15:40:08.039112+0000","last_undegraded":"2024-08-29T15:40:08.039112+0000","last_fullsized":"2024-08-29T15:40:08.039112+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:03:45.741225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":348,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:11.952390+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:40:11.952390+0000","last_peered":"2024-08-29T15:40:11.952390+0000","last_clean":"2024-08-29T15:40:11.952390+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:40:11.952390+0000","last_undegraded":"2024-08-29T15:40:11.952390+0000","last_fullsized":"2024-08-29T15:40:11.952390+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":215,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.039109+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:40:08.039109+0000","last_peered":"2024-08-29T15:40:08.039109+0000","last_clean":"2024-08-29T15:40:08.039109+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:40:08.039109+0000","last_undegraded":"2024-08-29T15:40:08.039109+0000","last_fullsized":"2024-08-29T15:40:08.039109+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":348,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:11.952452+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:40:11.952452+0000","last_peered":"2024-08-29T15:40:11.952452+0000","last_clean":"2024-08-29T15:40:11.952452+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:40:11.952452+0000","last_undegraded":"2024-08-29T15:40:11.952452+0000","last_fullsized":"2024-08-29T15:40:11.952452+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":758,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.134971+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:40:19.134971+0000","last_peered":"2024-08-29T15:40:19.134971+0000","last_clean":"2024-08-29T15:40:19.134971+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:40:19.134971+0000","last_undegraded":"2024-08-29T15:40:19.134971+0000","last_fullsized":"2024-08-29T15:40:19.134971+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":761,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.132204+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:40:19.132204+0000","last_peered":"2024-08-29T15:40:19.132204+0000","last_clean":"2024-08-29T15:40:19.132204+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:40:19.132204+0000","last_undegraded":"2024-08-29T15:40:19.132204+0000","last_fullsized":"2024-08-29T15:40:19.132204+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":215,"reported_epoch":430,"state":"active+clean","last_fresh":"2024-08-29T15:40:08.038787+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:40:08.038787+0000","last_peered":"2024-08-29T15:40:08.038787+0000","last_clean":"2024-08-29T15:40:08.038787+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:40:08.038787+0000","last_undegraded":"2024-08-29T15:40:08.038787+0000","last_fullsized":"2024-08-29T15:40:08.038787+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":112,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.114989+0000","last_change":"2024-08-29T15:39:11.853131+0000","last_active":"2024-08-29T15:40:19.114989+0000","last_peered":"2024-08-29T15:40:19.114989+0000","last_clean":"2024-08-29T15:40:19.114989+0000","last_became_active":"2024-08-29T15:39:11.836033+0000","last_became_peered":"2024-08-29T15:39:11.836033+0000","last_unstale":"2024-08-29T15:40:19.114989+0000","last_undegraded":"2024-08-29T15:40:19.114989+0000","last_fullsized":"2024-08-29T15:40:19.114989+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:51.036468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3397779456,"num_objects":51846,"num_object_clones":0,"num_object_copies":155538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":51846,"num_write_kb":3318144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10301603840,"data_stored":10301603840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":51846,"ondisk_log_size":51846,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":7736,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234549,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2638204,"kb_used_data":1993460,"kb_used_omap":1,"kb_used_meta":644734,"kb_avail":91106948,"statfs":{"total":95995035648,"available":93293514752,"internally_reserved":0,"allocated":2041303040,"data_stored":2038626655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":660208075},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496179,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1055800,"kb_used_data":573484,"kb_used_omap":1,"kb_used_meta":482302,"kb_avail":92689352,"statfs":{"total":95995035648,"available":94913896448,"internally_reserved":0,"allocated":587247616,"data_stored":584579093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":493877703},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426766,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2270064,"kb_used_data":1841704,"kb_used_omap":8,"kb_used_meta":428343,"kb_avail":91475088,"statfs":{"total":95995035648,"available":93670490112,"internally_reserved":0,"allocated":1885904896,"data_stored":1883236781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":438624138},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019459,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1319032,"kb_used_data":674288,"kb_used_omap":1,"kb_used_meta":644734,"kb_avail":92426120,"statfs":{"total":95995035648,"available":94644346880,"internally_reserved":0,"allocated":690470912,"data_stored":687798350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":660208070},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435643,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2394348,"kb_used_data":1608868,"kb_used_omap":4,"kb_used_meta":785467,"kb_avail":91350804,"statfs":{"total":95995035648,"available":93543223296,"internally_reserved":0,"allocated":1647480832,"data_stored":1644816575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":804318473},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903495,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1158632,"kb_used_data":635360,"kb_used_omap":10,"kb_used_meta":523253,"kb_avail":92586520,"statfs":{"total":95995035648,"available":94808596480,"internally_reserved":0,"allocated":650608640,"data_stored":647947825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":535811600},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762696,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2447524,"kb_used_data":1775836,"kb_used_omap":7,"kb_used_meta":671672,"kb_avail":91297628,"statfs":{"total":95995035648,"available":93488771072,"internally_reserved":0,"allocated":1818456064,"data_stored":1815798239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":687792482},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,0,3,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926223,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1520484,"kb_used_data":866844,"kb_used_omap":7,"kb_used_meta":653624,"kb_avail":92224668,"statfs":{"total":95995035648,"available":94438060032,"internally_reserved":0,"allocated":887648256,"data_stored":884990720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":669311119},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":893583360,"data_stored":893583360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1815937024,"data_stored":1815937024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":661127168,"data_stored":661127168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1683554304,"data_stored":1683554304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":685834240,"data_stored":685834240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1911160832,"data_stored":1911160832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":596770816,"data_stored":596770816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2053636096,"data_stored":2053636096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:40:34.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:40:34.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:34.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:34.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:34.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:34 smithi171 ceph-mon[90031]: pgmap v1516: 99 pgs: 99 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-08-29T15:40:34.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:34.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:34.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:34.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:34.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:34.775 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:34.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:34.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:34.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 37 16 70446 70430 118.953 89 0.00692749 0.00840111 2024-08-29T15:40:34.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:34.909 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:34.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:34.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:34.981 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:35.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:35.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:35.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:35.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:35.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:35.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:35.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:35.412 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:35.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:35.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:35.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:35.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:35.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:35.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:35.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:35.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:35.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:35.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:35.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:35.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:35.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 38 16 71650 71634 117.802 75.25 0.018485 0.00848309 2024-08-29T15:40:35.953 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:35.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:35.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:36.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:36.132 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:36.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:36.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:36.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:36.276 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:36.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:36.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:36.378 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:36.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:36.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:36.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:36.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:36 smithi171 ceph-mon[90031]: pgmap v1517: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-29T15:40:36.552 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:36.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:36.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:36 smithi027 ceph-mon[93921]: pgmap v1517: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-29T15:40:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:36 smithi027 ceph-mon[98970]: pgmap v1517: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-08-29T15:40:36.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:36.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:36.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:36.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:36.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:36.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:36.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:36.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 39 16 72830 72814 116.672 73.75 0.00844178 0.0085653 2024-08-29T15:40:36.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:36.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:36.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:36.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:37.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:37.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:37.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:37.238 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:37.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:37.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:37.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:37.411 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:37.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:37.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:37.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:37.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:37.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:37.660 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:40:37.660 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:40:37.660 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:40:37.660 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:40:37.660 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:40:37.660 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:40:37.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:37.670 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:40:37.671 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:40:37.699 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:37.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:37.705 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:40:37.705 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:40:37.745 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:40:37.746 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:40:37.766 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:40:37.766 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:40:37.783 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:40:37.783 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:40:37.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:37.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:37.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:37.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:40:37.883364+0000 min lat: 0.00131181 max lat: 0.356404 avg lat: 0.00865806 2024-08-29T15:40:37.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:40:37.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 40 16 73903 73887 115.432 67.0625 0.0109167 0.00865806 2024-08-29T15:40:37.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:37.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:37.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:38.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:38.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:38.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:38.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:38.185 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:40:38.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:38.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:38.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:38.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:38.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:38.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:38.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:38.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:38 smithi171 ceph-mon[90031]: pgmap v1518: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-29T15:40:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3268466635' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:40:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:38 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:40:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:38 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3015333135' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:38.522 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:38.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:38.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:38.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[93921]: pgmap v1518: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-29T15:40:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3268466635' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:40:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:40:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3015333135' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[98970]: pgmap v1518: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-29T15:40:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3268466635' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:40:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:40:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:38 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3015333135' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:38.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:38.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:38.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:38.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:38.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:38.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:38.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 41 16 75247 75231 114.665 84 0.00976693 0.00871576 2024-08-29T15:40:38.906 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:38.944 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:38.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:38.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:39.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:39.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:39.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:39.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:39.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:39.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:39.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:40:39] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-08-29T15:40:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:40:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:40:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:40:39.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[98970]: osdmap e432: 8 total, 8 up, 8 in 2024-08-29T15:40:39.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:40:39.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:40:39.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:40:39.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:39 smithi027 ceph-mon[93921]: osdmap e432: 8 total, 8 up, 8 in 2024-08-29T15:40:39.329 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:39.362 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:39.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:39.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:39.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:39.501 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:39.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:39.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:39 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:40:39.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:39 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:40:39.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:39 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:40:39.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:39 smithi171 ceph-mon[90031]: osdmap e432: 8 total, 8 up, 8 in 2024-08-29T15:40:39.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:39.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:39.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:39.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:39.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:39.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:39.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:39.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:39.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 42 16 76536 76520 113.853 80.5625 0.0240954 0.00877819 2024-08-29T15:40:39.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:39.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:39.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:40.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:40.068 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:40.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:40.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:40.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:40.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:40.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:40.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:40.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:40.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:40.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:40.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:40.447 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:40.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:40.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:40 smithi171 ceph-mon[90031]: pgmap v1520: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-29T15:40:40.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:40 smithi027 ceph-mon[98970]: pgmap v1520: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-29T15:40:40.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:40 smithi027 ceph-mon[93921]: pgmap v1520: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-29T15:40:40.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:40.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:40.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:40.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:40.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:40.769 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:40.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:40.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:40.772 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:40.871 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:40.883 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 43 16 77880 77864 113.158 84 0.00680843 0.00883243 2024-08-29T15:40:40.907 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:40.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:41.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:41.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:41.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:41.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:41.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:41.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:41.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:41.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:41.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:41.326 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:41.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:41.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:41.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:41.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:41.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:41.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:41.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:40:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:41.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:41.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:41.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:41.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:41.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:41.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:41.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:41.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 44 16 79166 79150 112.413 80.375 0.0185013 0.0088904 2024-08-29T15:40:41.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:41.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:42.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:42.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:42.028 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:42.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:42.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:42.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:42.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:42.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:42.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:42.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:42.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:42 smithi027 ceph-mon[93921]: pgmap v1521: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-08-29T15:40:42.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:42 smithi027 ceph-mon[98970]: pgmap v1521: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-08-29T15:40:42.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:42.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:42.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:42.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:42 smithi171 ceph-mon[90031]: pgmap v1521: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-08-29T15:40:42.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:40:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=cleanup t=2024-08-29T15:40:42.200204665Z level=info msg="Completed cleanup jobs" duration=3.269359ms 2024-08-29T15:40:42.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:40:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugins.update.checker t=2024-08-29T15:40:42.35178051Z level=info msg="Update check succeeded" duration=62.446245ms 2024-08-29T15:40:42.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:40:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana.update.checker t=2024-08-29T15:40:42.366408399Z level=info msg="Update check succeeded" duration=82.370508ms 2024-08-29T15:40:42.537 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:42.572 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:42.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:42.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:42.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:42.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:42.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:42.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:40:42.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:42.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 45 16 80452 80436 111.701 80.375 0.0120617 0.00894769 2024-08-29T15:40:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:42.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:42.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:42.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:43.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:43.037 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:43.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:43.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:43.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:43.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:43.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:43.272 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:43.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:43.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:43.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:43.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:43.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:43.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:43.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:43.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:43.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:43.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:43.727 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:43.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:43.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:43.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:43.863 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:43.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:43.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 46 16 81697 81681 110.964 77.8125 0.00582156 0.00900673 2024-08-29T15:40:43.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:44.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:44.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:44.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.143 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:44.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:81185 9.d 9:b58cf4ac:::benchmark_data_smithi027_341149_object81184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e432)", 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:43.486433+0000", 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0081780819999999997, 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082102749999999995, 2024-08-29T15:40:44.178 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 81185 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486433+0000", 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:40:44.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486433+0000", 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486435+0000", 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7829999999999999e-06 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486458+0000", 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3431e-05 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486474+0000", 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5483000000000001e-05 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486481+0000", 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6149999999999999e-06 2024-08-29T15:40:44.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486540+0000", 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8634999999999997e-05 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486684+0000", 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000143537 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:40:44.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.486837+0000", 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015295400000000001 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492895+0000", 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060585769999999999 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492943+0000", 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7617000000000002e-05 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:81192 9.0 9:04e2103e:::benchmark_data_smithi027_341149_object81191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e432)", 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:43.492032+0000", 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0025791239999999999, 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026899530000000001, 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:40:44.184 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 81192 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492032+0000", 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492032+0000", 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492033+0000", 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.265e-06 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492056+0000", 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3139999999999999e-05 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492066+0000", 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.414e-06 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492089+0000", 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2807999999999999e-05 2024-08-29T15:40:44.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492150+0000", 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1017999999999998e-05 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492298+0000", 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000148023 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:40:44.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.492448+0000", 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014976 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:81194 9.f e432/420 9:f8e4e4d2:::benchmark_data_smithi027_341149_object81193:head v 432'5074, mlcod=432'5071)", 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:43.494008+0000", 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00060309199999999995, 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077393600000000002, 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:40:44.189 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494008+0000", 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494008+0000", 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:40:44.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494009+0000", 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5899999999999995e-07 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494118+0000", 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108628 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494133+0000", 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5469e-05 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494140+0000", 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7159999999999997e-06 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494380+0000", 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000240406 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:43.494398+0000", 2024-08-29T15:40:44.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8142999999999999e-05 2024-08-29T15:40:44.193 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:40:44.193 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:40:44.193 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:40:44.193 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:40:44.193 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:40:44.193 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 3 2024-08-29T15:40:44.193 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:40:44.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:44.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:44.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:44.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:44 smithi171 ceph-mon[90031]: pgmap v1522: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:40:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:44.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:44.529 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:44.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:44.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:44 smithi027 ceph-mon[98970]: pgmap v1522: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:40:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:44 smithi027 ceph-mon[93921]: pgmap v1522: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:40:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:40:44.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:44.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:44.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:44.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:44.848 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:44.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:44.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 47 16 82950 82934 110.269 78.3125 0.00907067 0.00906427 2024-08-29T15:40:44.950 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:44.983 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:44.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:44.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:45.085 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:45.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:45.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:45.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:45.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:45.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:45.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:45.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:45.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:45.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:45.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:45.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:45.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:45.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:45.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:45.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:45.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:45.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:45.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:45.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:45.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 48 16 84184 84168 109.579 77.125 0.00532595 0.00912075 2024-08-29T15:40:45.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:45.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:45.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:45.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:46.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:46.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:46.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:46.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:46.223 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:46.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:46.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:46.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:46.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:46.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:46.471 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:46.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:46.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:46.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:46 smithi171 ceph-mon[90031]: pgmap v1523: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:40:46.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:46 smithi027 ceph-mon[93921]: pgmap v1523: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:40:46.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:46 smithi027 ceph-mon[98970]: pgmap v1523: 99 pgs: 99 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:40:46.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:46.649 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:46.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:46.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:46.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:46.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:46.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:46.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 49 16 85362 85346 108.845 73.625 0.0104917 0.00918209 2024-08-29T15:40:46.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:46.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:46.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:46.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:47.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:47.067 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:47.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:47.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:47.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:47.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:47.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:47.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:47.337 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:47.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:47.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:47.439 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:47.460 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:47.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:47.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:47.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:47.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:47.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:47.699 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:47.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:47.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:47.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:47.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:47.872 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:47.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:47.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:47.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 50 16 86350 86334 107.903 61.75 0.008677 0.00926297 2024-08-29T15:40:47.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:48.008 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:48.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:48.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:48.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:48.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:48.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:48.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:48.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:48.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:48.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:48.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:48.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:48.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:48 smithi171 ceph-mon[90031]: pgmap v1524: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:40:48.526 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:48 smithi027 ceph-mon[93921]: pgmap v1524: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:40:48.527 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:48 smithi027 ceph-mon[98970]: pgmap v1524: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-29T15:40:48.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:48.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:48.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:48.692 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:48.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:48.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:48.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:48.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:48.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:48.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 51 16 87599 87583 107.318 78.0625 0.0096015 0.00931263 2024-08-29T15:40:48.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:49.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:40:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:49.020 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:49.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:49.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:49.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:49.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:49.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:49.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:49.280 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:40:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:40:49] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-08-29T15:40:49.285 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:49.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:49.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:49.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:49.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:49.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:49.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:49.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:49.582 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:49.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:49.646 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:49.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:49.717 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:40:49.718 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:40:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:49.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:49.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:49.822 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:49.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:49.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:49.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 52 16 88679 88663 106.552 67.5 0.0106974 0.00937976 2024-08-29T15:40:49.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:50.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:50.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:50.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:50.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:50.253 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:50.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:50.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:50.312 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:50 smithi027 ceph-mon[98970]: pgmap v1525: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:40:50.313 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:50 smithi027 ceph-mon[93921]: pgmap v1525: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:40:50.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:50.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:50.509 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:50 smithi171 ceph-mon[90031]: pgmap v1525: 99 pgs: 99 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:40:50.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:50.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:50.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:50.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:50.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:50.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:50.804 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:50.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:50.884 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 53 16 89868 89852 105.943 74.3125 0.0117546 0.00943362 2024-08-29T15:40:50.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:50.983 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:50.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:51.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:51.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:51.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:51.235 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:51.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:51.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:51.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:51.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:51.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:51.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:51.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:51.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:51 smithi027 ceph-mon[98970]: from='client.56494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:40:51.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:51 smithi027 ceph-mon[93921]: from='client.56494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:40:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:51 smithi171 ceph-mon[90031]: from='client.56494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:40:51.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:51.544 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1525,"stamp":"2024-08-29T15:40:48.965667+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5582821349,"num_objects":85142,"num_object_clones":0,"num_object_copies":255426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85142,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":86022,"num_write_kb":5459921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":180,"num_bytes_recovered":9051723,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87786,"ondisk_log_size":87786,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21132948,"kb_used_data":16158676,"kb_used_omap":43,"kb_used_meta":4974164,"kb_avail":728828268,"statfs":{"total":767960285184,"available":746320146432,"internally_reserved":0,"allocated":16546484224,"data_stored":16525127374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":5093544428},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,4,5,11,10,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":897777664,"num_objects":13699,"num_object_clones":0,"num_object_copies":41097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13699,"num_write_kb":876736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.786179"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":769,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.181135+0000","last_change":"2024-08-29T15:39:01.754656+0000","last_active":"2024-08-29T15:40:38.181135+0000","last_peered":"2024-08-29T15:40:38.181135+0000","last_clean":"2024-08-29T15:40:38.181135+0000","last_became_active":"2024-08-29T15:39:01.754213+0000","last_became_peered":"2024-08-29T15:39:01.754213+0000","last_unstale":"2024-08-29T15:40:38.181135+0000","last_undegraded":"2024-08-29T15:40:38.181135+0000","last_fullsized":"2024-08-29T15:40:38.181135+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:34.210741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":223,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123393+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:40:19.123393+0000","last_peered":"2024-08-29T15:40:19.123393+0000","last_clean":"2024-08-29T15:40:19.123393+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:40:19.123393+0000","last_undegraded":"2024-08-29T15:40:19.123393+0000","last_fullsized":"2024-08-29T15:40:19.123393+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":18,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.112557+0000","last_change":"2024-08-29T15:39:53.258205+0000","last_active":"2024-08-29T15:40:19.112557+0000","last_peered":"2024-08-29T15:40:19.112557+0000","last_clean":"2024-08-29T15:40:19.112557+0000","last_became_active":"2024-08-29T15:39:53.257784+0000","last_became_peered":"2024-08-29T15:39:53.257784+0000","last_unstale":"2024-08-29T15:40:19.112557+0000","last_undegraded":"2024-08-29T15:40:19.112557+0000","last_fullsized":"2024-08-29T15:40:19.112557+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:30:42.270982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":735,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.184614+0000","last_change":"2024-08-29T15:39:21.000890+0000","last_active":"2024-08-29T15:40:38.184614+0000","last_peered":"2024-08-29T15:40:38.184614+0000","last_clean":"2024-08-29T15:40:38.184614+0000","last_became_active":"2024-08-29T15:39:21.000547+0000","last_became_peered":"2024-08-29T15:39:21.000547+0000","last_unstale":"2024-08-29T15:40:38.184614+0000","last_undegraded":"2024-08-29T15:40:38.184614+0000","last_fullsized":"2024-08-29T15:40:38.184614+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:43:30.808747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":720,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.186060+0000","last_change":"2024-08-29T15:08:58.888472+0000","last_active":"2024-08-29T15:40:38.186060+0000","last_peered":"2024-08-29T15:40:38.186060+0000","last_clean":"2024-08-29T15:40:38.186060+0000","last_became_active":"2024-08-29T15:08:58.888188+0000","last_became_peered":"2024-08-29T15:08:58.888188+0000","last_unstale":"2024-08-29T15:40:38.186060+0000","last_undegraded":"2024-08-29T15:40:38.186060+0000","last_fullsized":"2024-08-29T15:40:38.186060+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:29.634207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":216,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123267+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:40:19.123267+0000","last_peered":"2024-08-29T15:40:19.123267+0000","last_clean":"2024-08-29T15:40:19.123267+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:40:19.123267+0000","last_undegraded":"2024-08-29T15:40:19.123267+0000","last_fullsized":"2024-08-29T15:40:19.123267+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.181778+0000","last_change":"2024-08-29T15:39:21.005920+0000","last_active":"2024-08-29T15:40:38.181778+0000","last_peered":"2024-08-29T15:40:38.181778+0000","last_clean":"2024-08-29T15:40:38.181778+0000","last_became_active":"2024-08-29T15:39:21.005533+0000","last_became_peered":"2024-08-29T15:39:21.005533+0000","last_unstale":"2024-08-29T15:40:38.181778+0000","last_undegraded":"2024-08-29T15:40:38.181778+0000","last_fullsized":"2024-08-29T15:40:38.181778+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:32:12.424182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":155,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.183547+0000","last_change":"2024-08-29T15:39:53.266248+0000","last_active":"2024-08-29T15:40:38.183547+0000","last_peered":"2024-08-29T15:40:38.183547+0000","last_clean":"2024-08-29T15:40:38.183547+0000","last_became_active":"2024-08-29T15:39:53.258274+0000","last_became_peered":"2024-08-29T15:39:53.258274+0000","last_unstale":"2024-08-29T15:40:38.183547+0000","last_undegraded":"2024-08-29T15:40:38.183547+0000","last_fullsized":"2024-08-29T15:40:38.183547+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:35:50.632965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.5","version":"432'5566","reported_seq":5601,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:48.701114+0000","last_change":"2024-08-29T15:39:37.259396+0000","last_active":"2024-08-29T15:40:48.701114+0000","last_peered":"2024-08-29T15:40:48.701114+0000","last_clean":"2024-08-29T15:40:48.701114+0000","last_became_active":"2024-08-29T15:39:37.259121+0000","last_became_peered":"2024-08-29T15:39:37.259121+0000","last_unstale":"2024-08-29T15:40:48.701114+0000","last_undegraded":"2024-08-29T15:40:48.701114+0000","last_fullsized":"2024-08-29T15:40:48.701114+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5566,"log_dups_size":0,"ondisk_log_size":5566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:09:08.900552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364773376,"num_objects":5566,"num_object_clones":0,"num_object_copies":16698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5566,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":247,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123581+0000","last_change":"2024-08-29T15:37:26.322453+0000","last_active":"2024-08-29T15:40:19.123581+0000","last_peered":"2024-08-29T15:40:19.123581+0000","last_clean":"2024-08-29T15:40:19.123581+0000","last_became_active":"2024-08-29T15:37:26.308813+0000","last_became_peered":"2024-08-29T15:37:26.308813+0000","last_unstale":"2024-08-29T15:40:19.123581+0000","last_undegraded":"2024-08-29T15:40:19.123581+0000","last_fullsized":"2024-08-29T15:40:19.123581+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:05:55.778279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":18,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.111558+0000","last_change":"2024-08-29T15:39:53.258473+0000","last_active":"2024-08-29T15:40:19.111558+0000","last_peered":"2024-08-29T15:40:19.111558+0000","last_clean":"2024-08-29T15:40:19.111558+0000","last_became_active":"2024-08-29T15:39:53.258055+0000","last_became_peered":"2024-08-29T15:39:53.258055+0000","last_unstale":"2024-08-29T15:40:19.111558+0000","last_undegraded":"2024-08-29T15:40:19.111558+0000","last_fullsized":"2024-08-29T15:40:19.111558+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:38:54.441868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"432'5172","reported_seq":5206,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:45.340518+0000","last_change":"2024-08-29T15:39:37.186986+0000","last_active":"2024-08-29T15:40:45.340518+0000","last_peered":"2024-08-29T15:40:45.340518+0000","last_clean":"2024-08-29T15:40:45.340518+0000","last_became_active":"2024-08-29T15:39:37.186669+0000","last_became_peered":"2024-08-29T15:39:37.186669+0000","last_unstale":"2024-08-29T15:40:45.340518+0000","last_undegraded":"2024-08-29T15:40:45.340518+0000","last_fullsized":"2024-08-29T15:40:45.340518+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5172,"log_dups_size":0,"ondisk_log_size":5172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:33:01.221948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338952192,"num_objects":5172,"num_object_clones":0,"num_object_copies":15516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":331008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":221,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123720+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:40:19.123720+0000","last_peered":"2024-08-29T15:40:19.123720+0000","last_clean":"2024-08-29T15:40:19.123720+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:40:19.123720+0000","last_undegraded":"2024-08-29T15:40:19.123720+0000","last_fullsized":"2024-08-29T15:40:19.123720+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":244,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123160+0000","last_change":"2024-08-29T15:39:32.183389+0000","last_active":"2024-08-29T15:40:19.123160+0000","last_peered":"2024-08-29T15:40:19.123160+0000","last_clean":"2024-08-29T15:40:19.123160+0000","last_became_active":"2024-08-29T15:39:32.183069+0000","last_became_peered":"2024-08-29T15:39:32.183069+0000","last_unstale":"2024-08-29T15:40:19.123160+0000","last_undegraded":"2024-08-29T15:40:19.123160+0000","last_fullsized":"2024-08-29T15:40:19.123160+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:13:42.679657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"432'5268","reported_seq":5288,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.982830+0000","last_change":"2024-08-29T15:39:37.258750+0000","last_active":"2024-08-29T15:40:46.982830+0000","last_peered":"2024-08-29T15:40:46.982830+0000","last_clean":"2024-08-29T15:40:46.982830+0000","last_became_active":"2024-08-29T15:39:37.258439+0000","last_became_peered":"2024-08-29T15:39:37.258439+0000","last_unstale":"2024-08-29T15:40:46.982830+0000","last_undegraded":"2024-08-29T15:40:46.982830+0000","last_fullsized":"2024-08-29T15:40:46.982830+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:43:55.933600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5268,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":22,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.111476+0000","last_change":"2024-08-29T15:39:53.262167+0000","last_active":"2024-08-29T15:40:19.111476+0000","last_peered":"2024-08-29T15:40:19.111476+0000","last_clean":"2024-08-29T15:40:19.111476+0000","last_became_active":"2024-08-29T15:39:53.258406+0000","last_became_peered":"2024-08-29T15:39:53.258406+0000","last_unstale":"2024-08-29T15:40:19.111476+0000","last_undegraded":"2024-08-29T15:40:19.111476+0000","last_fullsized":"2024-08-29T15:40:19.111476+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:05:19.731559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":356,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.189894+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:40:38.189894+0000","last_peered":"2024-08-29T15:40:38.189894+0000","last_clean":"2024-08-29T15:40:38.189894+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:40:38.189894+0000","last_undegraded":"2024-08-29T15:40:38.189894+0000","last_fullsized":"2024-08-29T15:40:38.189894+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"432'5324","reported_seq":5345,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.957885+0000","last_change":"2024-08-29T15:39:37.376667+0000","last_active":"2024-08-29T15:40:46.957885+0000","last_peered":"2024-08-29T15:40:46.957885+0000","last_clean":"2024-08-29T15:40:46.957885+0000","last_became_active":"2024-08-29T15:39:37.376372+0000","last_became_peered":"2024-08-29T15:39:37.376372+0000","last_unstale":"2024-08-29T15:40:46.957885+0000","last_undegraded":"2024-08-29T15:40:46.957885+0000","last_fullsized":"2024-08-29T15:40:46.957885+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:52:02.092878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348913664,"num_objects":5324,"num_object_clones":0,"num_object_copies":15972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5324,"num_write_kb":340736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":325,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.111642+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:40:19.111642+0000","last_peered":"2024-08-29T15:40:19.111642+0000","last_clean":"2024-08-29T15:40:19.111642+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:40:19.111642+0000","last_undegraded":"2024-08-29T15:40:19.111642+0000","last_fullsized":"2024-08-29T15:40:19.111642+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":735,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.184180+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:40:38.184180+0000","last_peered":"2024-08-29T15:40:38.184180+0000","last_clean":"2024-08-29T15:40:38.184180+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:40:38.184180+0000","last_undegraded":"2024-08-29T15:40:38.184180+0000","last_fullsized":"2024-08-29T15:40:38.184180+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"432'5291","reported_seq":5326,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.549356+0000","last_change":"2024-08-29T15:39:37.311470+0000","last_active":"2024-08-29T15:40:46.549356+0000","last_peered":"2024-08-29T15:40:46.549356+0000","last_clean":"2024-08-29T15:40:46.549356+0000","last_became_active":"2024-08-29T15:39:37.310913+0000","last_became_peered":"2024-08-29T15:39:37.310913+0000","last_unstale":"2024-08-29T15:40:46.549356+0000","last_undegraded":"2024-08-29T15:40:46.549356+0000","last_fullsized":"2024-08-29T15:40:46.549356+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5291,"log_dups_size":0,"ondisk_log_size":5291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:39.550827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346750976,"num_objects":5291,"num_object_clones":0,"num_object_copies":15873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5291,"num_write_kb":338624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":858,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.179553+0000","last_change":"2024-08-29T15:37:18.101543+0000","last_active":"2024-08-29T15:40:38.179553+0000","last_peered":"2024-08-29T15:40:38.179553+0000","last_clean":"2024-08-29T15:40:38.179553+0000","last_became_active":"2024-08-29T15:37:01.308140+0000","last_became_peered":"2024-08-29T15:37:01.308140+0000","last_unstale":"2024-08-29T15:40:38.179553+0000","last_undegraded":"2024-08-29T15:40:38.179553+0000","last_fullsized":"2024-08-29T15:40:38.179553+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:18.101472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:18.101472+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:06:29.187841+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":752,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.188011+0000","last_change":"2024-08-29T15:37:26.340871+0000","last_active":"2024-08-29T15:40:38.188011+0000","last_peered":"2024-08-29T15:40:38.188011+0000","last_clean":"2024-08-29T15:40:38.188011+0000","last_became_active":"2024-08-29T15:37:26.328280+0000","last_became_peered":"2024-08-29T15:37:26.328280+0000","last_unstale":"2024-08-29T15:40:38.188011+0000","last_undegraded":"2024-08-29T15:40:38.188011+0000","last_fullsized":"2024-08-29T15:40:38.188011+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:47:01.239361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":217,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123896+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:40:19.123896+0000","last_peered":"2024-08-29T15:40:19.123896+0000","last_clean":"2024-08-29T15:40:19.123896+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:40:19.123896+0000","last_undegraded":"2024-08-29T15:40:19.123896+0000","last_fullsized":"2024-08-29T15:40:19.123896+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"432'5081","reported_seq":5115,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:45.394040+0000","last_change":"2024-08-29T15:39:37.376626+0000","last_active":"2024-08-29T15:40:45.394040+0000","last_peered":"2024-08-29T15:40:45.394040+0000","last_clean":"2024-08-29T15:40:45.394040+0000","last_became_active":"2024-08-29T15:39:37.376318+0000","last_became_peered":"2024-08-29T15:39:37.376318+0000","last_unstale":"2024-08-29T15:40:45.394040+0000","last_undegraded":"2024-08-29T15:40:45.394040+0000","last_fullsized":"2024-08-29T15:40:45.394040+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5081,"log_dups_size":0,"ondisk_log_size":5081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:45.198689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332988416,"num_objects":5081,"num_object_clones":0,"num_object_copies":15243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5081,"num_write_kb":325184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":726,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:21.405507+0000","last_change":"2024-08-29T15:40:21.405507+0000","last_active":"2024-08-29T15:40:21.405507+0000","last_peered":"2024-08-29T15:40:21.405507+0000","last_clean":"2024-08-29T15:40:21.405507+0000","last_became_active":"2024-08-29T15:39:56.271696+0000","last_became_peered":"2024-08-29T15:39:56.271696+0000","last_unstale":"2024-08-29T15:40:21.405507+0000","last_undegraded":"2024-08-29T15:40:21.405507+0000","last_fullsized":"2024-08-29T15:40:21.405507+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:40:21.404385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:40:21.404385+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:35.409438+0000","scrub_duration":0.04293052,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":203,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123283+0000","last_change":"2024-08-29T15:39:01.753999+0000","last_active":"2024-08-29T15:40:19.123283+0000","last_peered":"2024-08-29T15:40:19.123283+0000","last_clean":"2024-08-29T15:40:19.123283+0000","last_became_active":"2024-08-29T15:39:01.753603+0000","last_became_peered":"2024-08-29T15:39:01.753603+0000","last_unstale":"2024-08-29T15:40:19.123283+0000","last_undegraded":"2024-08-29T15:40:19.123283+0000","last_fullsized":"2024-08-29T15:40:19.123283+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:23:55.741058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":217,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123541+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:40:19.123541+0000","last_peered":"2024-08-29T15:40:19.123541+0000","last_clean":"2024-08-29T15:40:19.123541+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:40:19.123541+0000","last_undegraded":"2024-08-29T15:40:19.123541+0000","last_fullsized":"2024-08-29T15:40:19.123541+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"432'5333","reported_seq":5368,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.555080+0000","last_change":"2024-08-29T15:39:37.258693+0000","last_active":"2024-08-29T15:40:46.555080+0000","last_peered":"2024-08-29T15:40:46.555080+0000","last_clean":"2024-08-29T15:40:46.555080+0000","last_became_active":"2024-08-29T15:39:37.258303+0000","last_became_peered":"2024-08-29T15:39:37.258303+0000","last_unstale":"2024-08-29T15:40:46.555080+0000","last_undegraded":"2024-08-29T15:40:46.555080+0000","last_fullsized":"2024-08-29T15:40:46.555080+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5333,"log_dups_size":0,"ondisk_log_size":5333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:53:38.911068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349503488,"num_objects":5333,"num_object_clones":0,"num_object_copies":15999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5333,"num_write_kb":341312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123171+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:40:19.123171+0000","last_peered":"2024-08-29T15:40:19.123171+0000","last_clean":"2024-08-29T15:40:19.123171+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:40:19.123171+0000","last_undegraded":"2024-08-29T15:40:19.123171+0000","last_fullsized":"2024-08-29T15:40:19.123171+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":441,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.188344+0000","last_change":"2024-08-29T15:39:01.753897+0000","last_active":"2024-08-29T15:40:38.188344+0000","last_peered":"2024-08-29T15:40:38.188344+0000","last_clean":"2024-08-29T15:40:38.188344+0000","last_became_active":"2024-08-29T15:39:01.753534+0000","last_became_peered":"2024-08-29T15:39:01.753534+0000","last_unstale":"2024-08-29T15:40:38.188344+0000","last_undegraded":"2024-08-29T15:40:38.188344+0000","last_fullsized":"2024-08-29T15:40:38.188344+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:11:01.816741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"432'5299","reported_seq":5332,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.540895+0000","last_change":"2024-08-29T15:39:37.376745+0000","last_active":"2024-08-29T15:40:46.540895+0000","last_peered":"2024-08-29T15:40:46.540895+0000","last_clean":"2024-08-29T15:40:46.540895+0000","last_became_active":"2024-08-29T15:39:37.376397+0000","last_became_peered":"2024-08-29T15:39:37.376397+0000","last_unstale":"2024-08-29T15:40:46.540895+0000","last_undegraded":"2024-08-29T15:40:46.540895+0000","last_fullsized":"2024-08-29T15:40:46.540895+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5299,"log_dups_size":0,"ondisk_log_size":5299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:32:13.118834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347275264,"num_objects":5299,"num_object_clones":0,"num_object_copies":15897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5299,"num_write_kb":339136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":786,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.111305+0000","last_change":"2024-08-29T15:37:24.681545+0000","last_active":"2024-08-29T15:40:19.111305+0000","last_peered":"2024-08-29T15:40:19.111305+0000","last_clean":"2024-08-29T15:40:19.111305+0000","last_became_active":"2024-08-29T15:37:01.312787+0000","last_became_peered":"2024-08-29T15:37:01.312787+0000","last_unstale":"2024-08-29T15:40:19.111305+0000","last_undegraded":"2024-08-29T15:40:19.111305+0000","last_fullsized":"2024-08-29T15:40:19.111305+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:24.681392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:24.681392+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:01:45.435539+0000","scrub_duration":0.0064019280000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.180285+0000","last_change":"2024-08-29T15:39:21.002903+0000","last_active":"2024-08-29T15:40:38.180285+0000","last_peered":"2024-08-29T15:40:38.180285+0000","last_clean":"2024-08-29T15:40:38.180285+0000","last_became_active":"2024-08-29T15:39:21.002543+0000","last_became_peered":"2024-08-29T15:39:21.002543+0000","last_unstale":"2024-08-29T15:40:38.180285+0000","last_undegraded":"2024-08-29T15:40:38.180285+0000","last_fullsized":"2024-08-29T15:40:38.180285+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:57:08.725163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":725,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.190193+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:40:38.190193+0000","last_peered":"2024-08-29T15:40:38.190193+0000","last_clean":"2024-08-29T15:40:38.190193+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:40:38.190193+0000","last_undegraded":"2024-08-29T15:40:38.190193+0000","last_fullsized":"2024-08-29T15:40:38.190193+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"432'5316","reported_seq":5357,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.527456+0000","last_change":"2024-08-29T15:39:53.258499+0000","last_active":"2024-08-29T15:40:46.527456+0000","last_peered":"2024-08-29T15:40:46.527456+0000","last_clean":"2024-08-29T15:40:46.527456+0000","last_became_active":"2024-08-29T15:39:53.257102+0000","last_became_peered":"2024-08-29T15:39:53.257102+0000","last_unstale":"2024-08-29T15:40:46.527456+0000","last_undegraded":"2024-08-29T15:40:46.527456+0000","last_fullsized":"2024-08-29T15:40:46.527456+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5316,"log_dups_size":0,"ondisk_log_size":5316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:19:54.489027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348389376,"num_objects":5316,"num_object_clones":0,"num_object_copies":15948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5316,"num_write_kb":340224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":857,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.183982+0000","last_change":"2024-08-29T15:37:19.170088+0000","last_active":"2024-08-29T15:40:38.183982+0000","last_peered":"2024-08-29T15:40:38.183982+0000","last_clean":"2024-08-29T15:40:38.183982+0000","last_became_active":"2024-08-29T15:36:54.189617+0000","last_became_peered":"2024-08-29T15:36:54.189617+0000","last_unstale":"2024-08-29T15:40:38.183982+0000","last_undegraded":"2024-08-29T15:40:38.183982+0000","last_fullsized":"2024-08-29T15:40:38.183982+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.170014+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.170014+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:55:52.939920+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":910,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:36.975826+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:40:36.975826+0000","last_peered":"2024-08-29T15:40:36.975826+0000","last_clean":"2024-08-29T15:40:36.975826+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:40:36.975826+0000","last_undegraded":"2024-08-29T15:40:36.975826+0000","last_fullsized":"2024-08-29T15:40:36.975826+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":139,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.179309+0000","last_change":"2024-08-29T15:37:26.313495+0000","last_active":"2024-08-29T15:40:38.179309+0000","last_peered":"2024-08-29T15:40:38.179309+0000","last_clean":"2024-08-29T15:40:38.179309+0000","last_became_active":"2024-08-29T15:37:26.313188+0000","last_became_peered":"2024-08-29T15:37:26.313188+0000","last_unstale":"2024-08-29T15:40:38.179309+0000","last_undegraded":"2024-08-29T15:40:38.179309+0000","last_fullsized":"2024-08-29T15:40:38.179309+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:39.580490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":183,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123473+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:40:19.123473+0000","last_peered":"2024-08-29T15:40:19.123473+0000","last_clean":"2024-08-29T15:40:19.123473+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:40:19.123473+0000","last_undegraded":"2024-08-29T15:40:19.123473+0000","last_fullsized":"2024-08-29T15:40:19.123473+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"432'5499","reported_seq":5534,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:48.677172+0000","last_change":"2024-08-29T15:39:37.311691+0000","last_active":"2024-08-29T15:40:48.677172+0000","last_peered":"2024-08-29T15:40:48.677172+0000","last_clean":"2024-08-29T15:40:48.677172+0000","last_became_active":"2024-08-29T15:39:37.311136+0000","last_became_peered":"2024-08-29T15:39:37.311136+0000","last_unstale":"2024-08-29T15:40:48.677172+0000","last_undegraded":"2024-08-29T15:40:48.677172+0000","last_fullsized":"2024-08-29T15:40:48.677172+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5499,"log_dups_size":0,"ondisk_log_size":5499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:40:42.703845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360382464,"num_objects":5499,"num_object_clones":0,"num_object_copies":16497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5499,"num_write_kb":351936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":857,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.187356+0000","last_change":"2024-08-29T15:37:19.656177+0000","last_active":"2024-08-29T15:40:38.187356+0000","last_peered":"2024-08-29T15:40:38.187356+0000","last_clean":"2024-08-29T15:40:38.187356+0000","last_became_active":"2024-08-29T15:37:01.322473+0000","last_became_peered":"2024-08-29T15:37:01.322473+0000","last_unstale":"2024-08-29T15:40:38.187356+0000","last_undegraded":"2024-08-29T15:40:38.187356+0000","last_fullsized":"2024-08-29T15:40:38.187356+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.654978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.654978+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:21:16.623191+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1199,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.188476+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:40:38.188476+0000","last_peered":"2024-08-29T15:40:38.188476+0000","last_clean":"2024-08-29T15:40:38.188476+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:40:38.188476+0000","last_undegraded":"2024-08-29T15:40:38.188476+0000","last_fullsized":"2024-08-29T15:40:38.188476+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":751,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.184100+0000","last_change":"2024-08-29T15:39:21.003942+0000","last_active":"2024-08-29T15:40:38.184100+0000","last_peered":"2024-08-29T15:40:38.184100+0000","last_clean":"2024-08-29T15:40:38.184100+0000","last_became_active":"2024-08-29T15:39:21.003471+0000","last_became_peered":"2024-08-29T15:39:21.003471+0000","last_unstale":"2024-08-29T15:40:38.184100+0000","last_undegraded":"2024-08-29T15:40:38.184100+0000","last_fullsized":"2024-08-29T15:40:38.184100+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:34:44.102930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":96,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.185787+0000","last_change":"2024-08-29T15:38:27.108492+0000","last_active":"2024-08-29T15:40:38.185787+0000","last_peered":"2024-08-29T15:40:38.185787+0000","last_clean":"2024-08-29T15:40:38.185787+0000","last_became_active":"2024-08-29T15:38:27.108046+0000","last_became_peered":"2024-08-29T15:38:27.108046+0000","last_unstale":"2024-08-29T15:40:38.185787+0000","last_undegraded":"2024-08-29T15:40:38.185787+0000","last_fullsized":"2024-08-29T15:40:38.185787+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:22.265093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"432'5377","reported_seq":5395,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.964951+0000","last_change":"2024-08-29T15:39:53.258131+0000","last_active":"2024-08-29T15:40:46.964951+0000","last_peered":"2024-08-29T15:40:46.964951+0000","last_clean":"2024-08-29T15:40:46.964951+0000","last_became_active":"2024-08-29T15:39:53.257746+0000","last_became_peered":"2024-08-29T15:39:53.257746+0000","last_unstale":"2024-08-29T15:40:46.964951+0000","last_undegraded":"2024-08-29T15:40:46.964951+0000","last_fullsized":"2024-08-29T15:40:46.964951+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5377,"log_dups_size":0,"ondisk_log_size":5377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:48.746732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352387072,"num_objects":5377,"num_object_clones":0,"num_object_copies":16131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5377,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":865,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.184318+0000","last_change":"2024-08-29T15:37:18.155224+0000","last_active":"2024-08-29T15:40:38.184318+0000","last_peered":"2024-08-29T15:40:38.184318+0000","last_clean":"2024-08-29T15:40:38.184318+0000","last_became_active":"2024-08-29T15:37:01.308472+0000","last_became_peered":"2024-08-29T15:37:01.308472+0000","last_unstale":"2024-08-29T15:40:38.184318+0000","last_undegraded":"2024-08-29T15:40:38.184318+0000","last_fullsized":"2024-08-29T15:40:38.184318+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:18.154828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:18.154828+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:55.271955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":767,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.179886+0000","last_change":"2024-08-29T15:37:29.345555+0000","last_active":"2024-08-29T15:40:38.179886+0000","last_peered":"2024-08-29T15:40:38.179886+0000","last_clean":"2024-08-29T15:40:38.179886+0000","last_became_active":"2024-08-29T15:37:29.344738+0000","last_became_peered":"2024-08-29T15:37:29.344738+0000","last_unstale":"2024-08-29T15:40:38.179886+0000","last_undegraded":"2024-08-29T15:40:38.179886+0000","last_fullsized":"2024-08-29T15:40:38.179886+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:14.103195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":170,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:29.531972+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:40:29.531972+0000","last_peered":"2024-08-29T15:40:29.531972+0000","last_clean":"2024-08-29T15:40:29.531972+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:40:29.531972+0000","last_undegraded":"2024-08-29T15:40:29.531972+0000","last_fullsized":"2024-08-29T15:40:29.531972+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.189363+0000","last_change":"2024-08-29T15:39:01.753014+0000","last_active":"2024-08-29T15:40:38.189363+0000","last_peered":"2024-08-29T15:40:38.189363+0000","last_clean":"2024-08-29T15:40:38.189363+0000","last_became_active":"2024-08-29T15:39:01.752682+0000","last_became_peered":"2024-08-29T15:39:01.752682+0000","last_unstale":"2024-08-29T15:40:38.189363+0000","last_undegraded":"2024-08-29T15:40:38.189363+0000","last_fullsized":"2024-08-29T15:40:38.189363+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:00:06.838149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"432'5453","reported_seq":5488,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:48.657653+0000","last_change":"2024-08-29T15:39:37.258623+0000","last_active":"2024-08-29T15:40:48.657653+0000","last_peered":"2024-08-29T15:40:48.657653+0000","last_clean":"2024-08-29T15:40:48.657653+0000","last_became_active":"2024-08-29T15:39:37.258321+0000","last_became_peered":"2024-08-29T15:39:37.258321+0000","last_unstale":"2024-08-29T15:40:48.657653+0000","last_undegraded":"2024-08-29T15:40:48.657653+0000","last_fullsized":"2024-08-29T15:40:48.657653+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5453,"log_dups_size":0,"ondisk_log_size":5453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:09:19.337857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357367808,"num_objects":5453,"num_object_clones":0,"num_object_copies":16359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5453,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":785,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.111099+0000","last_change":"2024-08-29T15:37:23.716571+0000","last_active":"2024-08-29T15:40:19.111099+0000","last_peered":"2024-08-29T15:40:19.111099+0000","last_clean":"2024-08-29T15:40:19.111099+0000","last_became_active":"2024-08-29T15:37:01.308514+0000","last_became_peered":"2024-08-29T15:37:01.308514+0000","last_unstale":"2024-08-29T15:40:19.111099+0000","last_undegraded":"2024-08-29T15:40:19.111099+0000","last_fullsized":"2024-08-29T15:40:19.111099+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:23.716435+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:23.716435+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:18:15.213865+0000","scrub_duration":0.022267417000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":378,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:31.975259+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:40:31.975259+0000","last_peered":"2024-08-29T15:40:31.975259+0000","last_clean":"2024-08-29T15:40:31.975259+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:40:31.975259+0000","last_undegraded":"2024-08-29T15:40:31.975259+0000","last_fullsized":"2024-08-29T15:40:31.975259+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":218,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.188036+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:40:38.188036+0000","last_peered":"2024-08-29T15:40:38.188036+0000","last_clean":"2024-08-29T15:40:38.188036+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:40:38.188036+0000","last_undegraded":"2024-08-29T15:40:38.188036+0000","last_fullsized":"2024-08-29T15:40:38.188036+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":285,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:36.975845+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:40:36.975845+0000","last_peered":"2024-08-29T15:40:36.975845+0000","last_clean":"2024-08-29T15:40:36.975845+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:40:36.975845+0000","last_undegraded":"2024-08-29T15:40:36.975845+0000","last_fullsized":"2024-08-29T15:40:36.975845+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"432'5339","reported_seq":5360,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.918441+0000","last_change":"2024-08-29T15:39:37.185853+0000","last_active":"2024-08-29T15:40:46.918441+0000","last_peered":"2024-08-29T15:40:46.918441+0000","last_clean":"2024-08-29T15:40:46.918441+0000","last_became_active":"2024-08-29T15:39:37.185573+0000","last_became_peered":"2024-08-29T15:39:37.185573+0000","last_unstale":"2024-08-29T15:40:46.918441+0000","last_undegraded":"2024-08-29T15:40:46.918441+0000","last_fullsized":"2024-08-29T15:40:46.918441+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5339,"log_dups_size":0,"ondisk_log_size":5339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:12.924373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349896704,"num_objects":5339,"num_object_clones":0,"num_object_copies":16017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5339,"num_write_kb":341696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":817,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.181858+0000","last_change":"2024-08-29T15:39:23.121856+0000","last_active":"2024-08-29T15:40:38.181858+0000","last_peered":"2024-08-29T15:40:38.181858+0000","last_clean":"2024-08-29T15:40:38.181858+0000","last_became_active":"2024-08-29T15:39:23.121484+0000","last_became_peered":"2024-08-29T15:39:23.121484+0000","last_unstale":"2024-08-29T15:40:38.181858+0000","last_undegraded":"2024-08-29T15:40:38.181858+0000","last_fullsized":"2024-08-29T15:40:38.181858+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:16.688893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:16.688893+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:39:33.556710+0000","scrub_duration":0.016432537000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":310,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:36.975892+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:40:36.975892+0000","last_peered":"2024-08-29T15:40:36.975892+0000","last_clean":"2024-08-29T15:40:36.975892+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:40:36.975892+0000","last_undegraded":"2024-08-29T15:40:36.975892+0000","last_fullsized":"2024-08-29T15:40:36.975892+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1163,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.185117+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:40:38.185117+0000","last_peered":"2024-08-29T15:40:38.185117+0000","last_clean":"2024-08-29T15:40:38.185117+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:40:38.185117+0000","last_undegraded":"2024-08-29T15:40:38.185117+0000","last_fullsized":"2024-08-29T15:40:38.185117+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":82,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.189772+0000","last_change":"2024-08-29T15:39:53.328424+0000","last_active":"2024-08-29T15:40:38.189772+0000","last_peered":"2024-08-29T15:40:38.189772+0000","last_clean":"2024-08-29T15:40:38.189772+0000","last_became_active":"2024-08-29T15:39:53.328189+0000","last_became_peered":"2024-08-29T15:39:53.328189+0000","last_unstale":"2024-08-29T15:40:38.189772+0000","last_undegraded":"2024-08-29T15:40:38.189772+0000","last_fullsized":"2024-08-29T15:40:38.189772+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:51:04.926614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"432'5141","reported_seq":5161,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:44.656341+0000","last_change":"2024-08-29T15:39:37.258863+0000","last_active":"2024-08-29T15:40:44.656341+0000","last_peered":"2024-08-29T15:40:44.656341+0000","last_clean":"2024-08-29T15:40:44.656341+0000","last_became_active":"2024-08-29T15:39:37.258296+0000","last_became_peered":"2024-08-29T15:39:37.258296+0000","last_unstale":"2024-08-29T15:40:44.656341+0000","last_undegraded":"2024-08-29T15:40:44.656341+0000","last_fullsized":"2024-08-29T15:40:44.656341+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5141,"log_dups_size":0,"ondisk_log_size":5141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:32:01.990004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336920576,"num_objects":5141,"num_object_clones":0,"num_object_copies":15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5141,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":869,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.185849+0000","last_change":"2024-08-29T15:37:40.133670+0000","last_active":"2024-08-29T15:40:38.185849+0000","last_peered":"2024-08-29T15:40:38.185849+0000","last_clean":"2024-08-29T15:40:38.185849+0000","last_became_active":"2024-08-29T15:37:40.132923+0000","last_became_peered":"2024-08-29T15:37:40.132923+0000","last_unstale":"2024-08-29T15:40:38.185849+0000","last_undegraded":"2024-08-29T15:40:38.185849+0000","last_fullsized":"2024-08-29T15:40:38.185849+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:15.111260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:15.111260+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:27.886225+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":745,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.185017+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:40:38.185017+0000","last_peered":"2024-08-29T15:40:38.185017+0000","last_clean":"2024-08-29T15:40:38.185017+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:40:38.185017+0000","last_undegraded":"2024-08-29T15:40:38.185017+0000","last_fullsized":"2024-08-29T15:40:38.185017+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":890,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:24.529671+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:40:24.529671+0000","last_peered":"2024-08-29T15:40:24.529671+0000","last_clean":"2024-08-29T15:40:24.529671+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:40:24.529671+0000","last_undegraded":"2024-08-29T15:40:24.529671+0000","last_fullsized":"2024-08-29T15:40:24.529671+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":737,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.184026+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:40:38.184026+0000","last_peered":"2024-08-29T15:40:38.184026+0000","last_clean":"2024-08-29T15:40:38.184026+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:40:38.184026+0000","last_undegraded":"2024-08-29T15:40:38.184026+0000","last_fullsized":"2024-08-29T15:40:38.184026+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"432'5242","reported_seq":5276,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:45.362060+0000","last_change":"2024-08-29T15:39:37.259218+0000","last_active":"2024-08-29T15:40:45.362060+0000","last_peered":"2024-08-29T15:40:45.362060+0000","last_clean":"2024-08-29T15:40:45.362060+0000","last_became_active":"2024-08-29T15:39:37.258907+0000","last_became_peered":"2024-08-29T15:39:37.258907+0000","last_unstale":"2024-08-29T15:40:45.362060+0000","last_undegraded":"2024-08-29T15:40:45.362060+0000","last_fullsized":"2024-08-29T15:40:45.362060+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5242,"log_dups_size":0,"ondisk_log_size":5242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:20:07.328162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343539712,"num_objects":5242,"num_object_clones":0,"num_object_copies":15726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5242,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":802,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123281+0000","last_change":"2024-08-29T15:39:14.809022+0000","last_active":"2024-08-29T15:40:19.123281+0000","last_peered":"2024-08-29T15:40:19.123281+0000","last_clean":"2024-08-29T15:40:19.123281+0000","last_became_active":"2024-08-29T15:39:14.808645+0000","last_became_peered":"2024-08-29T15:39:14.808645+0000","last_unstale":"2024-08-29T15:40:19.123281+0000","last_undegraded":"2024-08-29T15:40:19.123281+0000","last_fullsized":"2024-08-29T15:40:19.123281+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.644669+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.644669+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:51.600506+0000","scrub_duration":0.0030306299999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":730,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.182444+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:40:38.182444+0000","last_peered":"2024-08-29T15:40:38.182444+0000","last_clean":"2024-08-29T15:40:38.182444+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:40:38.182444+0000","last_undegraded":"2024-08-29T15:40:38.182444+0000","last_fullsized":"2024-08-29T15:40:38.182444+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1382,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.190448+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:40:38.190448+0000","last_peered":"2024-08-29T15:40:38.190448+0000","last_clean":"2024-08-29T15:40:38.190448+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:40:38.190448+0000","last_undegraded":"2024-08-29T15:40:38.190448+0000","last_fullsized":"2024-08-29T15:40:38.190448+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":54,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.180211+0000","last_change":"2024-08-29T15:39:21.005897+0000","last_active":"2024-08-29T15:40:38.180211+0000","last_peered":"2024-08-29T15:40:38.180211+0000","last_clean":"2024-08-29T15:40:38.180211+0000","last_became_active":"2024-08-29T15:39:21.005125+0000","last_became_peered":"2024-08-29T15:39:21.005125+0000","last_unstale":"2024-08-29T15:40:38.180211+0000","last_undegraded":"2024-08-29T15:40:38.180211+0000","last_fullsized":"2024-08-29T15:40:38.180211+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:44:20.271835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"432'5413","reported_seq":5448,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:46.551461+0000","last_change":"2024-08-29T15:39:37.259666+0000","last_active":"2024-08-29T15:40:46.551461+0000","last_peered":"2024-08-29T15:40:46.551461+0000","last_clean":"2024-08-29T15:40:46.551461+0000","last_became_active":"2024-08-29T15:39:37.259380+0000","last_became_peered":"2024-08-29T15:39:37.259380+0000","last_unstale":"2024-08-29T15:40:46.551461+0000","last_undegraded":"2024-08-29T15:40:46.551461+0000","last_fullsized":"2024-08-29T15:40:46.551461+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":5413,"log_dups_size":0,"ondisk_log_size":5413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:02:18.132955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354746368,"num_objects":5413,"num_object_clones":0,"num_object_copies":16239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5413,"num_write_kb":346432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":889,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.180671+0000","last_change":"2024-08-29T15:39:56.274024+0000","last_active":"2024-08-29T15:40:38.180671+0000","last_peered":"2024-08-29T15:40:38.180671+0000","last_clean":"2024-08-29T15:40:38.180671+0000","last_became_active":"2024-08-29T15:39:56.273578+0000","last_became_peered":"2024-08-29T15:39:56.273578+0000","last_unstale":"2024-08-29T15:40:38.180671+0000","last_undegraded":"2024-08-29T15:40:38.180671+0000","last_fullsized":"2024-08-29T15:40:38.180671+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:20.077258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:20.077258+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:20.595923+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":142,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.185963+0000","last_change":"2024-08-29T15:37:26.328428+0000","last_active":"2024-08-29T15:40:38.185963+0000","last_peered":"2024-08-29T15:40:38.185963+0000","last_clean":"2024-08-29T15:40:38.185963+0000","last_became_active":"2024-08-29T15:37:26.313757+0000","last_became_peered":"2024-08-29T15:37:26.313757+0000","last_unstale":"2024-08-29T15:40:38.185963+0000","last_undegraded":"2024-08-29T15:40:38.185963+0000","last_fullsized":"2024-08-29T15:40:38.185963+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:29:31.366546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":290,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.120686+0000","last_change":"2024-08-29T15:18:34.989422+0000","last_active":"2024-08-29T15:40:19.120686+0000","last_peered":"2024-08-29T15:40:19.120686+0000","last_clean":"2024-08-29T15:40:19.120686+0000","last_became_active":"2024-08-29T15:18:34.988731+0000","last_became_peered":"2024-08-29T15:18:34.988731+0000","last_unstale":"2024-08-29T15:40:19.120686+0000","last_undegraded":"2024-08-29T15:40:19.120686+0000","last_fullsized":"2024-08-29T15:40:19.120686+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:36:19.542087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":218,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.187863+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:40:38.187863+0000","last_peered":"2024-08-29T15:40:38.187863+0000","last_clean":"2024-08-29T15:40:38.187863+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:40:38.187863+0000","last_undegraded":"2024-08-29T15:40:38.187863+0000","last_fullsized":"2024-08-29T15:40:38.187863+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":216,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123386+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:40:19.123386+0000","last_peered":"2024-08-29T15:40:19.123386+0000","last_clean":"2024-08-29T15:40:19.123386+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:40:19.123386+0000","last_undegraded":"2024-08-29T15:40:19.123386+0000","last_fullsized":"2024-08-29T15:40:19.123386+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":226,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123918+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:40:19.123918+0000","last_peered":"2024-08-29T15:40:19.123918+0000","last_clean":"2024-08-29T15:40:19.123918+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:40:19.123918+0000","last_undegraded":"2024-08-29T15:40:19.123918+0000","last_fullsized":"2024-08-29T15:40:19.123918+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.189683+0000","last_change":"2024-08-29T15:39:01.752879+0000","last_active":"2024-08-29T15:40:38.189683+0000","last_peered":"2024-08-29T15:40:38.189683+0000","last_clean":"2024-08-29T15:40:38.189683+0000","last_became_active":"2024-08-29T15:39:01.752227+0000","last_became_peered":"2024-08-29T15:39:01.752227+0000","last_unstale":"2024-08-29T15:40:38.189683+0000","last_undegraded":"2024-08-29T15:40:38.189683+0000","last_fullsized":"2024-08-29T15:40:38.189683+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:25:05.582312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":493,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.180698+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:40:38.180698+0000","last_peered":"2024-08-29T15:40:38.180698+0000","last_clean":"2024-08-29T15:40:38.180698+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:40:38.180698+0000","last_undegraded":"2024-08-29T15:40:38.180698+0000","last_fullsized":"2024-08-29T15:40:38.180698+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.111794+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:40:19.111794+0000","last_peered":"2024-08-29T15:40:19.111794+0000","last_clean":"2024-08-29T15:40:19.111794+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:40:19.111794+0000","last_undegraded":"2024-08-29T15:40:19.111794+0000","last_fullsized":"2024-08-29T15:40:19.111794+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":693,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.179738+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:40:38.179738+0000","last_peered":"2024-08-29T15:40:38.179738+0000","last_clean":"2024-08-29T15:40:38.179738+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:40:38.179738+0000","last_undegraded":"2024-08-29T15:40:38.179738+0000","last_fullsized":"2024-08-29T15:40:38.179738+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":746,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.183954+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:40:38.183954+0000","last_peered":"2024-08-29T15:40:38.183954+0000","last_clean":"2024-08-29T15:40:38.183954+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:40:38.183954+0000","last_undegraded":"2024-08-29T15:40:38.183954+0000","last_fullsized":"2024-08-29T15:40:38.183954+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":745,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.180373+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:40:38.180373+0000","last_peered":"2024-08-29T15:40:38.180373+0000","last_clean":"2024-08-29T15:40:38.180373+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:40:38.180373+0000","last_undegraded":"2024-08-29T15:40:38.180373+0000","last_fullsized":"2024-08-29T15:40:38.180373+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":288,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.172338+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:40:38.172338+0000","last_peered":"2024-08-29T15:40:38.172338+0000","last_clean":"2024-08-29T15:40:38.172338+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:40:38.172338+0000","last_undegraded":"2024-08-29T15:40:38.172338+0000","last_fullsized":"2024-08-29T15:40:38.172338+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":252,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.183868+0000","last_change":"2024-08-29T15:39:01.755284+0000","last_active":"2024-08-29T15:40:38.183868+0000","last_peered":"2024-08-29T15:40:38.183868+0000","last_clean":"2024-08-29T15:40:38.183868+0000","last_became_active":"2024-08-29T15:39:01.754638+0000","last_became_peered":"2024-08-29T15:39:01.754638+0000","last_unstale":"2024-08-29T15:40:38.183868+0000","last_undegraded":"2024-08-29T15:40:38.183868+0000","last_fullsized":"2024-08-29T15:40:38.183868+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:49:04.459879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":744,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.186136+0000","last_change":"2024-08-29T15:39:21.004535+0000","last_active":"2024-08-29T15:40:38.186136+0000","last_peered":"2024-08-29T15:40:38.186136+0000","last_clean":"2024-08-29T15:40:38.186136+0000","last_became_active":"2024-08-29T15:39:21.001651+0000","last_became_peered":"2024-08-29T15:39:21.001651+0000","last_unstale":"2024-08-29T15:40:38.186136+0000","last_undegraded":"2024-08-29T15:40:38.186136+0000","last_fullsized":"2024-08-29T15:40:38.186136+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:20:18.607584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":725,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.184674+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:40:38.184674+0000","last_peered":"2024-08-29T15:40:38.184674+0000","last_clean":"2024-08-29T15:40:38.184674+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:40:38.184674+0000","last_undegraded":"2024-08-29T15:40:38.184674+0000","last_fullsized":"2024-08-29T15:40:38.184674+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":729,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.180565+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:40:38.180565+0000","last_peered":"2024-08-29T15:40:38.180565+0000","last_clean":"2024-08-29T15:40:38.180565+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:40:38.180565+0000","last_undegraded":"2024-08-29T15:40:38.180565+0000","last_fullsized":"2024-08-29T15:40:38.180565+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":216,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123259+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:40:19.123259+0000","last_peered":"2024-08-29T15:40:19.123259+0000","last_clean":"2024-08-29T15:40:19.123259+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:40:19.123259+0000","last_undegraded":"2024-08-29T15:40:19.123259+0000","last_fullsized":"2024-08-29T15:40:19.123259+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":770,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.185218+0000","last_change":"2024-08-29T15:39:11.832387+0000","last_active":"2024-08-29T15:40:38.185218+0000","last_peered":"2024-08-29T15:40:38.185218+0000","last_clean":"2024-08-29T15:40:38.185218+0000","last_became_active":"2024-08-29T15:39:11.832056+0000","last_became_peered":"2024-08-29T15:39:11.832056+0000","last_unstale":"2024-08-29T15:40:38.185218+0000","last_undegraded":"2024-08-29T15:40:38.185218+0000","last_fullsized":"2024-08-29T15:40:38.185218+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:47:42.049015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":95,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.183718+0000","last_change":"2024-08-29T15:38:27.106157+0000","last_active":"2024-08-29T15:40:38.183718+0000","last_peered":"2024-08-29T15:40:38.183718+0000","last_clean":"2024-08-29T15:40:38.183718+0000","last_became_active":"2024-08-29T15:38:27.105635+0000","last_became_peered":"2024-08-29T15:38:27.105635+0000","last_unstale":"2024-08-29T15:40:38.183718+0000","last_undegraded":"2024-08-29T15:40:38.183718+0000","last_fullsized":"2024-08-29T15:40:38.183718+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:27.182762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":234,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123266+0000","last_change":"2024-08-29T15:39:01.753766+0000","last_active":"2024-08-29T15:40:19.123266+0000","last_peered":"2024-08-29T15:40:19.123266+0000","last_clean":"2024-08-29T15:40:19.123266+0000","last_became_active":"2024-08-29T15:39:01.753447+0000","last_became_peered":"2024-08-29T15:39:01.753447+0000","last_unstale":"2024-08-29T15:40:19.123266+0000","last_undegraded":"2024-08-29T15:40:19.123266+0000","last_fullsized":"2024-08-29T15:40:19.123266+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:03:45.741225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":349,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.111571+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:40:19.111571+0000","last_peered":"2024-08-29T15:40:19.111571+0000","last_clean":"2024-08-29T15:40:19.111571+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:40:19.111571+0000","last_undegraded":"2024-08-29T15:40:19.111571+0000","last_fullsized":"2024-08-29T15:40:19.111571+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":216,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123354+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:40:19.123354+0000","last_peered":"2024-08-29T15:40:19.123354+0000","last_clean":"2024-08-29T15:40:19.123354+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:40:19.123354+0000","last_undegraded":"2024-08-29T15:40:19.123354+0000","last_fullsized":"2024-08-29T15:40:19.123354+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":349,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.112647+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:40:19.112647+0000","last_peered":"2024-08-29T15:40:19.112647+0000","last_clean":"2024-08-29T15:40:19.112647+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:40:19.112647+0000","last_undegraded":"2024-08-29T15:40:19.112647+0000","last_fullsized":"2024-08-29T15:40:19.112647+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":761,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.179969+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:40:38.179969+0000","last_peered":"2024-08-29T15:40:38.179969+0000","last_clean":"2024-08-29T15:40:38.179969+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:40:38.179969+0000","last_undegraded":"2024-08-29T15:40:38.179969+0000","last_fullsized":"2024-08-29T15:40:38.179969+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":763,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.181353+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:40:38.181353+0000","last_peered":"2024-08-29T15:40:38.181353+0000","last_clean":"2024-08-29T15:40:38.181353+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:40:38.181353+0000","last_undegraded":"2024-08-29T15:40:38.181353+0000","last_fullsized":"2024-08-29T15:40:38.181353+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":431,"state":"active+clean","last_fresh":"2024-08-29T15:40:19.123084+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:40:19.123084+0000","last_peered":"2024-08-29T15:40:19.123084+0000","last_clean":"2024-08-29T15:40:19.123084+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:40:19.123084+0000","last_undegraded":"2024-08-29T15:40:19.123084+0000","last_fullsized":"2024-08-29T15:40:19.123084+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":115,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-08-29T15:40:38.183717+0000","last_change":"2024-08-29T15:39:11.853131+0000","last_active":"2024-08-29T15:40:38.183717+0000","last_peered":"2024-08-29T15:40:38.183717+0000","last_clean":"2024-08-29T15:40:38.183717+0000","last_became_active":"2024-08-29T15:39:11.836033+0000","last_became_peered":"2024-08-29T15:39:11.836033+0000","last_unstale":"2024-08-29T15:40:38.183717+0000","last_undegraded":"2024-08-29T15:40:38.183717+0000","last_fullsized":"2024-08-29T15:40:38.183717+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:51.036468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5578031104,"num_objects":85114,"num_object_clones":0,"num_object_copies":255342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85114,"num_write_kb":5447296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16604135424,"data_stored":16604135424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85114,"ondisk_log_size":85114,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":7736,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234553,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3653384,"kb_used_data":2982400,"kb_used_omap":1,"kb_used_meta":670974,"kb_avail":90091768,"statfs":{"total":95995035648,"available":92253970432,"internally_reserved":0,"allocated":3053977600,"data_stored":3051297361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":687077835},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496184,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1500928,"kb_used_data":988280,"kb_used_omap":1,"kb_used_meta":512638,"kb_avail":92244224,"statfs":{"total":95995035648,"available":94458085376,"internally_reserved":0,"allocated":1011998720,"data_stored":1009326343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":524941767},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426771,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3492540,"kb_used_data":3062516,"kb_used_omap":8,"kb_used_meta":430007,"kb_avail":90252612,"statfs":{"total":95995035648,"available":92418674688,"internally_reserved":0,"allocated":3136016384,"data_stored":3133344415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":440328074},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019463,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1645316,"kb_used_data":996476,"kb_used_omap":1,"kb_used_meta":648830,"kb_avail":92099836,"statfs":{"total":95995035648,"available":94310232064,"internally_reserved":0,"allocated":1020391424,"data_stored":1017715008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":664402374},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435648,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3528952,"kb_used_data":2717104,"kb_used_omap":4,"kb_used_meta":811835,"kb_avail":90216200,"statfs":{"total":95995035648,"available":92381388800,"internally_reserved":0,"allocated":2782314496,"data_stored":2779646385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":831319305},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,3,0,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903500,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1590580,"kb_used_data":1043756,"kb_used_omap":10,"kb_used_meta":546805,"kb_avail":92154572,"statfs":{"total":95995035648,"available":94366281728,"internally_reserved":0,"allocated":1068806144,"data_stored":1066141475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":559928848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762701,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3666736,"kb_used_data":2963112,"kb_used_omap":7,"kb_used_meta":703608,"kb_avail":90078416,"statfs":{"total":95995035648,"available":92240297984,"internally_reserved":0,"allocated":3034226688,"data_stored":3031565009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":720494946},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,7,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926228,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2054512,"kb_used_data":1405032,"kb_used_omap":7,"kb_used_meta":649464,"kb_avail":91690640,"statfs":{"total":95995035648,"available":93891215360,"internally_reserved":0,"allocated":1438752768,"data_stored":1436091378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":665051279},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1441792000,"data_stored":1441792000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3072720896,"data_stored":3072720896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1062797312,"data_stored":1062797312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2782330880,"data_stored":2782330880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1017053184,"data_stored":1017053184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3158376448,"data_stored":3158376448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1018888192,"data_stored":1018888192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3050176512,"data_stored":3050176512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:40:51.549 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-29T15:40:51.550 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:40:51.550 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-29T15:40:51.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-29T15:40:51.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:51.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:51.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:51.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:51.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:51.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:51.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:51.836 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:51.885 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 54 16 91038 91022 105.334 73.125 0.0192489 0.00948866 2024-08-29T15:40:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:51.890 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:51.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:51.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:52.028 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:52.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:52.130 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:52.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:52.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:52.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:52.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:52.296 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:52.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:52.397 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:52.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:52.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:52 smithi171 ceph-mon[90031]: pgmap v1526: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-29T15:40:52.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:52.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:52 smithi027 ceph-mon[98970]: pgmap v1526: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-29T15:40:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:52 smithi027 ceph-mon[93921]: pgmap v1526: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-29T15:40:52.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:52.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:52.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:52.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:52.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:52.730 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:52.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:52.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:52.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:52.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:52.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:52.885 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 55 16 92224 92208 104.767 74.125 0.00738486 0.00954001 2024-08-29T15:40:52.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:53.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:53.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:53.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:53.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:53.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:53.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:53.263 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:53.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:53.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:53.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:53.426 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:53.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:53.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:53.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:53.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:53.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:53.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:53.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:53.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:53.858 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:53.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:53.885 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 56 16 93320 93304 104.119 68.5 0.0137886 0.00959921 2024-08-29T15:40:53.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:54.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:54.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:54.094 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:54.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:54.178 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:40:54.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:54.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:54.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:54.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:54.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:54.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:54.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[98970]: pgmap v1527: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:40:54.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[93921]: pgmap v1527: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:40:54.402 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:54.476 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:54.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:54.519 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:54 smithi171 ceph-mon[90031]: pgmap v1527: 99 pgs: 99 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:40:54.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:40:54.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:40:54.520 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:40:54.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:54.617 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:54.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:54.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:54.752 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:54.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:54.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:54.885 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:54.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:54.886 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 57 16 94464 94448 103.546 71.5 0.00749145 0.00965183 2024-08-29T15:40:54.986 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:55.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:55.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:55.126 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:55.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:55.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:55.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:55.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:55.339 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:40:55.339 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:40:55.384 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:40:55.385 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:40:55.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:55.420 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:40:55.420 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:40:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:55.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:55.457 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:40:55.457 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:40:55.500 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:40:55.500 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:40:55.533 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:40:55.533 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:40:55.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:55.571 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:40:55.571 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:40:55.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:55.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:55.607 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:40:55.607 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:40:55.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:55.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:55.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:55.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:55.886 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 58 16 95568 95552 102.951 69 0.00897286 0.00970818 2024-08-29T15:40:55.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:55.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:55.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:56.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:56.139 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:56.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:56.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:56.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:56.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:56.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:56.395 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:56.423 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:56.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:56.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:56 smithi171 ceph-mon[90031]: pgmap v1528: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:40:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2678630808' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:56.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:56.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:56.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:56 smithi027 ceph-mon[98970]: pgmap v1528: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:40:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2678630808' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:56.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:56 smithi027 ceph-mon[93921]: pgmap v1528: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:40:56.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2678630808' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:40:56.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:40:56.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:56.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:56.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:40:56.793 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:56.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:56.826 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:56.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:56.886 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 59 16 96671 96655 102.374 68.9375 0.0165999 0.00976333 2024-08-29T15:40:56.927 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:56.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:56.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:57.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:57.089 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:57.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:57.120 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:40:57.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:57.224 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:57.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:57.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:57.326 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:57.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:57.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:57.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:57.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:57.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3247869934' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:40:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:57 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:40:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3247869934' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:40:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:57 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:40:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3247869934' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:40:57.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:57 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:40:57.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:57.636 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:57.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:57.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:40:57.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:57.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:40:57.873 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:57.886 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:40:57.886113+0000 min lat: 0.00131181 max lat: 0.356404 avg lat: 0.00980858 2024-08-29T15:40:57.886 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:40:57.886 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 60 16 97851 97835 101.897 73.75 0.0210237 0.00980858 2024-08-29T15:40:57.906 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:57.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:57.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:58.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:58.039 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:58.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:58.139 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:58.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:58.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:58.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:58.192 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[93921]: pgmap v1529: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:40:58.192 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:40:58.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:40:58.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[93921]: osdmap e433: 8 total, 8 up, 8 in 2024-08-29T15:40:58.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[98970]: pgmap v1529: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:40:58.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:40:58.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:40:58.193 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:58 smithi027 ceph-mon[98970]: osdmap e433: 8 total, 8 up, 8 in 2024-08-29T15:40:58.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:58.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:58.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:58.432 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:58.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:58.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:58.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:58 smithi171 ceph-mon[90031]: pgmap v1529: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-29T15:40:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:58 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:40:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:58 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:40:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:58 smithi171 ceph-mon[90031]: osdmap e433: 8 total, 8 up, 8 in 2024-08-29T15:40:58.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:58.626 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:58.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:58.728 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:58.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:58.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:58.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:58.888 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 61 16 98964 98948 101.363 69.5625 0.0227251 0.00985986 2024-08-29T15:40:58.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:58.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:58.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:59.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:40:59.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:59.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:59.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:40:59.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:40:59] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.51.0" 2024-08-29T15:40:59.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:40:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:59.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:59.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:40:59.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:59.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:59.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:40:59.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:40:59.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:59.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:59.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:40:59.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:40:59.625 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:40:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:59.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:40:59.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:40:59.741 INFO:teuthology.orchestra.run.smithi027.stdout:moved 11 / 297 (3.7037%) 2024-08-29T15:40:59.741 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:40:59.741 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 4.37143 -> 4.39993 (expected baseline 5.69951) 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.2 with 29 -> 30 pgs (0.781145 -> 0.808081 * mean) 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.7 with 43 -> 42 pgs (1.15825 -> 1.13131 * mean) 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 0.05 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0313 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0344 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 weight 1.0000 -> 0.9500 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 weight 1.0000 -> 0.9500 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 weight 1.0000 -> 0.9500 2024-08-29T15:40:59.742 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 1.0000 -> 0.9500 2024-08-29T15:40:59.775 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:59.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:59.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:40:59.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:40:59.888 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 62 16 99981 99965 100.753 63.5625 0.00403139 0.00990725 2024-08-29T15:40:59.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:40:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:40:59.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:00.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:41:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:59 smithi171 ceph-mon[90031]: pgmap v1531: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:41:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:59 smithi171 ceph-mon[90031]: from='client.56506 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:40:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"} : dispatch 2024-08-29T15:41:00.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:41:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[98970]: pgmap v1531: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:41:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[98970]: from='client.56506 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"} : dispatch 2024-08-29T15:41:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:41:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[93921]: pgmap v1531: 99 pgs: 99 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:41:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[93921]: from='client.56506 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:40:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"} : dispatch 2024-08-29T15:41:00.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:00.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:00.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:00.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:00.272 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:00.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:00.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:41:00.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:00.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:00.530 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:00.543 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-29T15:41:00.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-29T15:41:00.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:00.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:00.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:00.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:00.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:00.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:00.888 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 63 16 99981 99965 99.1541 0 - 0.00990725 2024-08-29T15:41:00.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:00.927 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:00.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:01.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:00 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]': finished 2024-08-29T15:41:01.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:00 smithi027 ceph-mon[98970]: osdmap e434: 8 total, 8 up, 8 in 2024-08-29T15:41:01.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:00 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]': finished 2024-08-29T15:41:01.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:00 smithi027 ceph-mon[93921]: osdmap e434: 8 total, 8 up, 8 in 2024-08-29T15:41:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:00 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]': finished 2024-08-29T15:41:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:00 smithi171 ceph-mon[90031]: osdmap e434: 8 total, 8 up, 8 in 2024-08-29T15:41:01.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:01.051 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:01.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:01.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:01.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:01.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:01.282 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:01.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:01.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:01.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:01.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:01.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:01.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:01.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:01.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:01.684 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:01.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:01.786 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:01.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:41:01.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:01.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 64 16 99981 99965 97.605 0 - 0.00990725 2024-08-29T15:41:01.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:01.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:01.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:01.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:01 smithi171 ceph-mon[90031]: osdmap e435: 8 total, 8 up, 8 in 2024-08-29T15:41:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:01 smithi171 ceph-mon[90031]: pgmap v1534: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:41:02.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:01 smithi027 ceph-mon[93921]: osdmap e435: 8 total, 8 up, 8 in 2024-08-29T15:41:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:01 smithi027 ceph-mon[93921]: pgmap v1534: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:41:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:01 smithi027 ceph-mon[98970]: osdmap e435: 8 total, 8 up, 8 in 2024-08-29T15:41:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:01 smithi027 ceph-mon[98970]: pgmap v1534: 99 pgs: 99 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:41:02.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.077 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:02.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:02.178 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:02.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:02.301 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:02.324 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:02.426 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:02.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:02.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:02.572 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:02.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:02.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:02.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:02.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:02.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:02.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:02.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:02.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 65 16 100099 100083 96.2169 2.45833 0.00416501 0.0100864 2024-08-29T15:41:02.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:02.948 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:02.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:02 smithi171 ceph-mon[90031]: osdmap e436: 8 total, 8 up, 8 in 2024-08-29T15:41:03.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:02 smithi171 ceph-mon[90031]: osdmap e437: 8 total, 8 up, 8 in 2024-08-29T15:41:03.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:03.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:02 smithi027 ceph-mon[98970]: osdmap e436: 8 total, 8 up, 8 in 2024-08-29T15:41:03.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:02 smithi027 ceph-mon[98970]: osdmap e437: 8 total, 8 up, 8 in 2024-08-29T15:41:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:02 smithi027 ceph-mon[93921]: osdmap e436: 8 total, 8 up, 8 in 2024-08-29T15:41:03.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:02 smithi027 ceph-mon[93921]: osdmap e437: 8 total, 8 up, 8 in 2024-08-29T15:41:03.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:03.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:03.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:03.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:03.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:03.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:03.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:03.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:03.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:03.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:03.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:03.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:03.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:03.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:03.824 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:03.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 66 16 100099 100083 94.7592 0 - 0.0100864 2024-08-29T15:41:03.925 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:03.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:03.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:03.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:04.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:04.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:04.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:04.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:04.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:04.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:04.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:04.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:04 smithi027 ceph-mon[98970]: pgmap v1537: 99 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 97 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 6273/298695 objects misplaced (2.100%) 2024-08-29T15:41:04.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:04 smithi027 ceph-mon[98970]: osdmap e438: 8 total, 8 up, 8 in 2024-08-29T15:41:04.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:04 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:41:04.366 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:04 smithi027 ceph-mon[93921]: pgmap v1537: 99 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 97 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 6273/298695 objects misplaced (2.100%) 2024-08-29T15:41:04.366 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:04 smithi027 ceph-mon[93921]: osdmap e438: 8 total, 8 up, 8 in 2024-08-29T15:41:04.366 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:04 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:41:04.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:04.451 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:04 smithi171 ceph-mon[90031]: pgmap v1537: 99 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 97 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 6273/298695 objects misplaced (2.100%) 2024-08-29T15:41:04.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:04 smithi171 ceph-mon[90031]: osdmap e438: 8 total, 8 up, 8 in 2024-08-29T15:41:04.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:04 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:41:04.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:04.579 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:04.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:04.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:04.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:04.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:04.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:04.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 67 16 100099 100083 93.345 0 - 0.0100864 2024-08-29T15:41:04.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:04.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:04.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:04.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:05.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:05.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:05.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:05.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:05.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:05.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:05.335 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:05.436 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:05.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:05.560 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:05.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:05.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:05.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:05.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:05.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:05.833 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:05.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:05.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 68 16 100099 100083 91.9724 0 - 0.0100864 2024-08-29T15:41:05.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:05.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:05.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:06.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:06.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:06.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:06.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:06.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:06.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:06.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:06.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:06.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:06 smithi171 ceph-mon[90031]: pgmap v1539: 99 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 97 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 MiB/s wr, 477 op/s; 6273/298728 objects misplaced (2.100%) 2024-08-29T15:41:06.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:06.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:06 smithi027 ceph-mon[98970]: pgmap v1539: 99 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 97 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 MiB/s wr, 477 op/s; 6273/298728 objects misplaced (2.100%) 2024-08-29T15:41:06.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:06 smithi027 ceph-mon[93921]: pgmap v1539: 99 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 97 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 MiB/s wr, 477 op/s; 6273/298728 objects misplaced (2.100%) 2024-08-29T15:41:06.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:06.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:06.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:06.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:06.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:06.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:06.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:06.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 69 16 100099 100083 90.6396 0 - 0.0100864 2024-08-29T15:41:06.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:06.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:06.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:07.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:07.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:07.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:07.212 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:07.314 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:07.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:07.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:07.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:07.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:07.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:07 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-29T15:41:07.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:07.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:07 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-29T15:41:07.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:07 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-29T15:41:07.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:07.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:07.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:07.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:07.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:07.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:07.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:07.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 70 16 100099 100083 89.3448 0 - 0.0100864 2024-08-29T15:41:07.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:07.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:07.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:08.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:08.086 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:08.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:08.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:08.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:08.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:08.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:08.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:08.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:08.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:08.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:08 smithi171 ceph-mon[90031]: pgmap v1540: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 6273/298851 objects misplaced (2.099%); 43 KiB/s, 0 objects/s recovering 2024-08-29T15:41:08.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:08.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:08 smithi027 ceph-mon[98970]: pgmap v1540: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 6273/298851 objects misplaced (2.099%); 43 KiB/s, 0 objects/s recovering 2024-08-29T15:41:08.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:08 smithi027 ceph-mon[93921]: pgmap v1540: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 6273/298851 objects misplaced (2.099%); 43 KiB/s, 0 objects/s recovering 2024-08-29T15:41:08.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:08.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:08.712 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:08.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:08.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:08.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:08.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:08.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 71 16 100099 100083 88.0866 0 - 0.0100864 2024-08-29T15:41:08.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:08.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:08.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:09.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:09.080 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:41:09.080 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:41:09.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:09.191 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:09.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:09.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:41:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:41:09] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.51.0" 2024-08-29T15:41:09.319 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:09.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:09.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:09.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:09.470 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:09.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:09.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:09.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:09.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:09.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:09.755 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:09.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:09.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:09.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:09.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 72 16 100099 100083 86.8632 0 - 0.0100864 2024-08-29T15:41:09.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:09.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:09.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:10.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:10.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:10.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:10.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:10.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:10 smithi027 ceph-mon[98970]: pgmap v1541: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 161 op/s; 5886/298977 objects misplaced (1.969%); 3.4 MiB/s, 53 objects/s recovering 2024-08-29T15:41:10.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:10 smithi027 ceph-mon[93921]: pgmap v1541: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 161 op/s; 5886/298977 objects misplaced (1.969%); 3.4 MiB/s, 53 objects/s recovering 2024-08-29T15:41:10.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:10.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:10.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:10.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:41:10.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:10.476 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:10 smithi171 ceph-mon[90031]: pgmap v1541: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 10 MiB/s wr, 161 op/s; 5886/298977 objects misplaced (1.969%); 3.4 MiB/s, 53 objects/s recovering 2024-08-29T15:41:10.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:10.603 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:10.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:10.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:10.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:10.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.861 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:10.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:10.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 73 16 100099 100083 85.6734 0 - 0.0100864 2024-08-29T15:41:10.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:10.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:10.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:11.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:11.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:11.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:11.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:11.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:11.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:11.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:11.460 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:11.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:11.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:11.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:11.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:11.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:11.731 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:11.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:11.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:11.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 74 16 100099 100083 84.5157 0 - 0.0100864 2024-08-29T15:41:11.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:11.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:11.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:11.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:12.056 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:12.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:12.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:12.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:12.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:12 smithi027 ceph-mon[98970]: pgmap v1542: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 681 KiB/s wr, 10 op/s; 5886/298977 objects misplaced (1.969%); 2.8 MiB/s, 43 objects/s recovering 2024-08-29T15:41:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1525029688' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:12 smithi027 ceph-mon[93921]: pgmap v1542: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 681 KiB/s wr, 10 op/s; 5886/298977 objects misplaced (1.969%); 2.8 MiB/s, 43 objects/s recovering 2024-08-29T15:41:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1525029688' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:12.336 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:12.357 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:12.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:12.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:12 smithi171 ceph-mon[90031]: pgmap v1542: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 681 KiB/s wr, 10 op/s; 5886/298977 objects misplaced (1.969%); 2.8 MiB/s, 43 objects/s recovering 2024-08-29T15:41:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1525029688' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:12.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:12.606 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:12.610 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:12.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:12.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:12.835 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:12.857 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:12.889 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 75 16 100099 100083 83.3889 0 - 0.0100864 2024-08-29T15:41:12.959 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:12.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:12.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:12.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:13.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:13.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:13.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:13.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:13.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:13.333 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:13.355 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:13.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:13.482 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:13.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:13.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:13.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:13.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:13.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:13.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:13.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:13.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:13.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 76 16 100099 100083 82.2918 0 - 0.0100864 2024-08-29T15:41:13.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:13.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:13.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:13.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:14.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:14.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:14.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:14.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:14.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:14.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:14.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:14.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:14.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:14.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:14 smithi171 ceph-mon[90031]: pgmap v1543: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 601 KiB/s wr, 9 op/s; 5472/298977 objects misplaced (1.830%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:41:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:14 smithi027 ceph-mon[98970]: pgmap v1543: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 601 KiB/s wr, 9 op/s; 5472/298977 objects misplaced (1.830%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:41:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:14.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:14 smithi027 ceph-mon[93921]: pgmap v1543: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 601 KiB/s wr, 9 op/s; 5472/298977 objects misplaced (1.830%); 5.0 MiB/s, 80 objects/s recovering 2024-08-29T15:41:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:14.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:14.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:14.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:14.738 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:14.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:14.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:14.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:14.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 77 16 100099 100083 81.2232 0 - 0.0100864 2024-08-29T15:41:14.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:14.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:14.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:15.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:15.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.115 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:15.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:moved 7 / 297 (2.3569%) 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 4.39993 -> 2.93417 (expected baseline 5.69951) 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.2 with 30 -> 32 pgs (0.808081 -> 0.861953 * mean) 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.3 with 44 -> 41 pgs (1.18519 -> 1.10438 * mean) 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 0.05 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0326 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0358 2024-08-29T15:41:15.207 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 weight 0.9500 -> 0.9000 2024-08-29T15:41:15.208 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 weight 0.9500 -> 0.9000 2024-08-29T15:41:15.208 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 weight 0.9500 -> 0.9000 2024-08-29T15:41:15.208 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.9500 -> 0.9000 2024-08-29T15:41:15.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:15.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:15.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:15.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:15.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:15.493 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:15.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:15 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'58984','7':'58984'}"} : dispatch 2024-08-29T15:41:15.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:15 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'58984','7':'58984'}"} : dispatch 2024-08-29T15:41:15.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:15 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'58984','7':'58984'}"} : dispatch 2024-08-29T15:41:15.595 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:15.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:15.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:15.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:15.740 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:15.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:15.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:15.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 78 16 100099 100083 80.1819 0 - 0.0100864 2024-08-29T15:41:15.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:15.993 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:15.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:15.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:16.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:16.119 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:16.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:16.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:16.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:16.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:16.371 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:16.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:16.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:16.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:16.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:16 smithi171 ceph-mon[90031]: from='client.36951 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:16 smithi171 ceph-mon[90031]: pgmap v1544: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 508 KiB/s wr, 7 op/s; 5472/298977 objects misplaced (1.830%); 4.3 MiB/s, 67 objects/s recovering 2024-08-29T15:41:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:16 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'58984','7':'58984'}"}]': finished 2024-08-29T15:41:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:16 smithi171 ceph-mon[90031]: osdmap e439: 8 total, 8 up, 8 in 2024-08-29T15:41:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[98970]: from='client.36951 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[98970]: pgmap v1544: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 508 KiB/s wr, 7 op/s; 5472/298977 objects misplaced (1.830%); 4.3 MiB/s, 67 objects/s recovering 2024-08-29T15:41:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'58984','7':'58984'}"}]': finished 2024-08-29T15:41:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[98970]: osdmap e439: 8 total, 8 up, 8 in 2024-08-29T15:41:16.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[93921]: from='client.36951 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:16.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[93921]: pgmap v1544: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 508 KiB/s wr, 7 op/s; 5472/298977 objects misplaced (1.830%); 4.3 MiB/s, 67 objects/s recovering 2024-08-29T15:41:16.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'58984','3':'58984','5':'58984','7':'58984'}"}]': finished 2024-08-29T15:41:16.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:16 smithi027 ceph-mon[93921]: osdmap e439: 8 total, 8 up, 8 in 2024-08-29T15:41:16.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:16.620 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:16.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:16.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:16.746 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:16.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:16.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:16.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:16.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 79 16 100099 100083 79.167 0 - 0.0100864 2024-08-29T15:41:16.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:16.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:16.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:17.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:17.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:17.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:17.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:17.248 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:17.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:17.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:17.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:17.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:17.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:17.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:17.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:17.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:17 smithi171 ceph-mon[90031]: osdmap e440: 8 total, 8 up, 8 in 2024-08-29T15:41:17.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:17 smithi171 ceph-mon[90031]: osdmap e441: 8 total, 8 up, 8 in 2024-08-29T15:41:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:17 smithi027 ceph-mon[93921]: osdmap e440: 8 total, 8 up, 8 in 2024-08-29T15:41:17.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:17 smithi027 ceph-mon[93921]: osdmap e441: 8 total, 8 up, 8 in 2024-08-29T15:41:17.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:17 smithi027 ceph-mon[98970]: osdmap e440: 8 total, 8 up, 8 in 2024-08-29T15:41:17.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:17 smithi027 ceph-mon[98970]: osdmap e441: 8 total, 8 up, 8 in 2024-08-29T15:41:17.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:17.627 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:17.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:17.729 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:17.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:17.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:17.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:17.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:17.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:17.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:17.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:41:17.890160+0000 min lat: 0.00131181 max lat: 2.24305 avg lat: 0.0100864 2024-08-29T15:41:17.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:41:17.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 80 16 100099 100083 78.1775 0 - 0.0100864 2024-08-29T15:41:17.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:18.001 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:18.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:18.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:18.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:18.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:18.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:18.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:18.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:18.357 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:18.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:18.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:18.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:18.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:18 smithi171 ceph-mon[90031]: pgmap v1547: 99 pgs: 1 remapped+peering, 1 unknown, 1 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5472/298977 objects misplaced (1.830%); 3.2 MiB/s, 51 objects/s recovering 2024-08-29T15:41:18.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:18 smithi027 ceph-mon[93921]: pgmap v1547: 99 pgs: 1 remapped+peering, 1 unknown, 1 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5472/298977 objects misplaced (1.830%); 3.2 MiB/s, 51 objects/s recovering 2024-08-29T15:41:18.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:18 smithi027 ceph-mon[98970]: pgmap v1547: 99 pgs: 1 remapped+peering, 1 unknown, 1 active+recovering+undersized+remapped, 96 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5472/298977 objects misplaced (1.830%); 3.2 MiB/s, 51 objects/s recovering 2024-08-29T15:41:18.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:18.624 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:18.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:18.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:18.747 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:18.849 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:18.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:18.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 81 16 100099 100083 77.2124 0 - 0.0100864 2024-08-29T15:41:18.973 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:18.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:18.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:18.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:19.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:19.123 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:19.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:19.241 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:41:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:41:19] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-08-29T15:41:19.250 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:19.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:19.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:19.478 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:19.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:19 smithi171 ceph-mon[90031]: osdmap e442: 8 total, 8 up, 8 in 2024-08-29T15:41:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:19 smithi171 ceph-mon[90031]: osdmap e443: 8 total, 8 up, 8 in 2024-08-29T15:41:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:19 smithi027 ceph-mon[93921]: osdmap e442: 8 total, 8 up, 8 in 2024-08-29T15:41:19.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:19 smithi027 ceph-mon[93921]: osdmap e443: 8 total, 8 up, 8 in 2024-08-29T15:41:19.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:19 smithi027 ceph-mon[98970]: osdmap e442: 8 total, 8 up, 8 in 2024-08-29T15:41:19.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:19 smithi027 ceph-mon[98970]: osdmap e443: 8 total, 8 up, 8 in 2024-08-29T15:41:19.602 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.623 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:19.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:19.696 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:41:19.697 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:41:19.718 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:41:19.718 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:41:19.725 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:19.736 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:41:19.736 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:41:19.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:19.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:19.754 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:41:19.755 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:41:19.772 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:41:19.773 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:41:19.791 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:41:19.791 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:41:19.808 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:41:19.808 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:41:19.825 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:41:19.825 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:41:19.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:19.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:19.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:19.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 82 16 100099 100083 76.2708 0 - 0.0100864 2024-08-29T15:41:19.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:19.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:19.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:19.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:20.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:20.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:20.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:20.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:20.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:20.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:20.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:20.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:20.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:20.481 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:20 smithi027 ceph-mon[93921]: pgmap v1550: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 1 unknown, 95 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5038/298977 objects misplaced (1.685%); 6.8 MiB/s, 108 objects/s recovering 2024-08-29T15:41:20.481 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:20 smithi027 ceph-mon[98970]: pgmap v1550: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 1 unknown, 95 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5038/298977 objects misplaced (1.685%); 6.8 MiB/s, 108 objects/s recovering 2024-08-29T15:41:20.496 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:20 smithi171 ceph-mon[90031]: pgmap v1550: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 1 unknown, 95 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5038/298977 objects misplaced (1.685%); 6.8 MiB/s, 108 objects/s recovering 2024-08-29T15:41:20.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:20.625 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:20.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:20.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:20.851 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:20.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:20.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 83 16 100099 100083 75.352 0 - 0.0100864 2024-08-29T15:41:20.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:20.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:20.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:21.055 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-29T15:41:21.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-08-29T15:41:21.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:21.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:21.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:21.245 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:21.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:21.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:21.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:21.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:21.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:21 smithi027 ceph-mon[98970]: Health check failed: Degraded data redundancy: 18749/300336 objects degraded (6.243%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:21.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:21 smithi027 ceph-mon[93921]: Health check failed: Degraded data redundancy: 18749/300336 objects degraded (6.243%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:21.601 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:21.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:21.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:21.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:21.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:21.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:21 smithi171 ceph-mon[90031]: Health check failed: Degraded data redundancy: 18749/300336 objects degraded (6.243%), 2 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:21.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:21.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:21.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:21.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 84 16 100099 100083 74.455 0 - 0.0100864 2024-08-29T15:41:21.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:21.996 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:21.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:22.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:22.125 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:22.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:22.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:22.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:22.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:22.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:22.373 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:22.475 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:22.496 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:22 smithi027 ceph-mon[93921]: pgmap v1552: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 93 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.9 MiB/s wr, 95 op/s; 18749/300336 objects degraded (6.243%); 10319/300336 objects misplaced (3.436%); 19 MiB/s, 312 objects/s recovering 2024-08-29T15:41:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:22 smithi027 ceph-mon[98970]: pgmap v1552: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 93 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.9 MiB/s wr, 95 op/s; 18749/300336 objects degraded (6.243%); 10319/300336 objects misplaced (3.436%); 19 MiB/s, 312 objects/s recovering 2024-08-29T15:41:22.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:22.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:22.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:22.745 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:22 smithi171 ceph-mon[90031]: pgmap v1552: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 remapped+peering, 93 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.9 MiB/s wr, 95 op/s; 18749/300336 objects degraded (6.243%); 10319/300336 objects misplaced (3.436%); 19 MiB/s, 312 objects/s recovering 2024-08-29T15:41:22.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:22.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:22.890 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 85 16 100099 100083 73.5791 0 - 0.0100864 2024-08-29T15:41:22.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:22.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:22.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:23.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:23.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:23.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:23.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:23.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:23.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:23.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:23.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:23.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:23.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:23.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:23.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:23.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:23.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:23.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:23.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:23.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:23.750 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:23.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:23.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:148195 7.5 7.29cfb6a5 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e310)", 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:41:23.760 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 598.56569812600003, 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.001068407, 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148195 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191638+0000", 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191640+0000", 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2989999999999999e-06 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191646+0000", 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8690000000000001e-06 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191664+0000", 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8189000000000002e-05 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.191673+0000", 2024-08-29T15:41:23.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7229999999999993e-06 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.198665+0000", 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069919730000000003 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:25.198675+0000", 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8740000000000007e-06 2024-08-29T15:41:23.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192690+0000", 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99401576199999997 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:31:26.192706+0000", 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5718000000000001e-05 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191681 7.5 7:a9d21a90:::benchmark_data_smithi027_271924_object75340:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.81088825400002, 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.12813253, 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.766 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191681 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946448+0000", 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946449+0000", 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6509999999999999e-06 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946455+0000", 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8769999999999998e-06 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946472+0000", 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7169000000000001e-05 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946481+0000", 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3289999999999994e-06 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946564+0000", 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3615000000000005e-05 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op must wait for map", 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946571+0000", 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0849999999999999e-06 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.952704+0000", 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061325340000000002 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.952713+0000", 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2709999999999998e-06 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042516+0000", 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.089802597 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042641+0000", 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012506600000000001 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074432+0000", 2024-08-29T15:41:23.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.031791257000000003 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074580+0000", 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014807899999999999 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191685 7.5 7:a0ab1ff6:::benchmark_data_smithi027_271924_object75344:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:41:23.772 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.81083377899995, 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1510580690000001, 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191685 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946502+0000", 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946503+0000", 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4199999999999996e-07 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946505+0000", 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2589999999999999e-06 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946512+0000", 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9999999999999999e-06 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946517+0000", 2024-08-29T15:41:23.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.079e-06 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946614+0000", 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.692e-05 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting_for_map not empty", 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.946619+0000", 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1200000000000001e-06 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.953271+0000", 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066521210000000004 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.953278+0000", 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.204e-06 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.042965+0000", 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.089686486 2024-08-29T15:41:23.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043078+0000", 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113163 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097459+0000", 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.054381391000000001 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.097560+0000", 2024-08-29T15:41:23.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100784 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191701 7.5 7:a40d5d11:::benchmark_data_smithi027_271924_object75360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.77579104300003, 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.099438114, 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.780 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191701 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981545+0000", 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981546+0000", 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.291e-06 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981551+0000", 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9779999999999996e-06 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981563+0000", 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1787000000000001e-05 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981570+0000", 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4240000000000004e-06 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981641+0000", 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1018000000000003e-05 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981649+0000", 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9379999999999992e-06 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043134+0000", 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0614847059999999 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043265+0000", 2024-08-29T15:41:23.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013072100000000001 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.043851+0000", 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058628399999999996 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.067960+0000", 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024109189 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.069771+0000", 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018104740000000001 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080779+0000", 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011008132 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080848+0000", 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9249000000000001e-05 2024-08-29T15:41:23.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080983+0000", 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013492300000000001 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191702 7.5 7:a63127fd:::benchmark_data_smithi027_271924_object75361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.775748669, 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.099668844, 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.787 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191702 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981587+0000", 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981588+0000", 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7399999999999998e-07 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981590+0000", 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.266e-06 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981597+0000", 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.579e-06 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981602+0000", 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3329999999999996e-06 2024-08-29T15:41:23.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981671+0000", 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9581999999999999e-05 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.981677+0000", 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9000000000000003e-06 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065439+0000", 2024-08-29T15:41:23.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0837620489999999 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065523+0000", 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3906000000000006e-05 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.065611+0000", 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7347000000000002e-05 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072205+0000", 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065941050000000003 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072219+0000", 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3817e-05 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081102+0000", 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088836209999999995 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081154+0000", 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1907000000000003e-05 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081256+0000", 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101958 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.794 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191704 7.5 7:adaa8151:::benchmark_data_smithi027_271924_object75363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.77038311299998, 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0945452710000001, 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191704 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986953+0000", 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986955+0000", 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.781e-06 2024-08-29T15:41:23.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986961+0000", 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2349999999999996e-06 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986976+0000", 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5676e-05 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.986984+0000", 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8369999999999997e-06 2024-08-29T15:41:23.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.001951+0000", 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014966225 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.001960+0000", 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8379999999999997e-06 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.069808+0000", 2024-08-29T15:41:23.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.067847722 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.070348+0000", 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00053970699999999995 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.070512+0000", 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016436399999999999 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074579+0000", 2024-08-29T15:41:23.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004066315 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080650+0000", 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060713909999999998 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081349+0000", 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00069884000000000005 2024-08-29T15:41:23.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081398+0000", 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9524999999999998e-05 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081498+0000", 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9815000000000006e-05 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.801 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191709 7.5 7:aeaa9bf4:::benchmark_data_smithi027_271924_object75368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.76888018600005, 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0993208800000001, 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.802 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191709 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988456+0000", 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988457+0000", 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4270000000000001e-06 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988462+0000", 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0159999999999999e-06 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988477+0000", 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5116999999999999e-05 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:48.988484+0000", 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1799999999999999e-06 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.003492+0000", 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015007802000000001 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.003518+0000", 2024-08-29T15:41:23.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5970999999999999e-05 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072246+0000", 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.068727446 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072383+0000", 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013752099999999999 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.072536+0000", 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015290200000000001 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077323+0000", 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047870919999999997 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081054+0000", 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037308939999999998 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087498+0000", 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006444015 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087668+0000", 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016991900000000001 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087777+0000", 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108578 2024-08-29T15:41:23.808 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191717 7.5 7:a9f87d2e:::benchmark_data_smithi027_271924_object75376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.75054743400005, 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0898794949999999, 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.809 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191717 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006788+0000", 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006790+0000", 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9970000000000001e-06 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006797+0000", 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4570000000000002e-06 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006830+0000", 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2867999999999998e-05 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006840+0000", 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.01e-05 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006951+0000", 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011073600000000001 2024-08-29T15:41:23.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006963+0000", 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1977999999999999e-05 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074609+0000", 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0676467890000001 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074705+0000", 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5200999999999999e-05 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.074792+0000", 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6955999999999999e-05 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.080664+0000", 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058721310000000001 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081300+0000", 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00063660400000000003 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096470+0000", 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015169472 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096534+0000", 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3911999999999994e-05 2024-08-29T15:41:23.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096668+0000", 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134294 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191718 7.5 7:a50fea8b:::benchmark_data_smithi027_271924_object75377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.75046797899995, 2024-08-29T15:41:23.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.090114451, 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191718 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006868+0000", 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006869+0000", 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2099999999999996e-07 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006872+0000", 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2729999999999998e-06 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.819 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006882+0000", 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8779999999999993e-06 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006887+0000", 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6409999999999999e-06 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006985+0000", 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7449999999999997e-05 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.006994+0000", 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5799999999999998e-06 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077356+0000", 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0703613439999999 2024-08-29T15:41:23.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077468+0000", 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112322 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.077596+0000", 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012779999999999999 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087397+0000", 2024-08-29T15:41:23.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098012389999999998 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087443+0000", 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6204e-05 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096811+0000", 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093673639999999996 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096888+0000", 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7793000000000005e-05 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.096982+0000", 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3942000000000003e-05 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191761 7.5 7:a52c750c:::benchmark_data_smithi027_271924_object75420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.66909792000001, 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0210746239999999, 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191761 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.825 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088238+0000", 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088240+0000", 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6300000000000001e-06 2024-08-29T15:41:23.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088247+0000", 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9990000000000002e-06 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088261+0000", 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4416000000000001e-05 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088269+0000", 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1629999999999996e-06 2024-08-29T15:41:23.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088333+0000", 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.347e-05 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.088342+0000", 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.363e-06 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081533+0000", 2024-08-29T15:41:23.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99319060000000003 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081648+0000", 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115339 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.081776+0000", 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012817300000000001 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087457+0000", 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056813039999999999 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087842+0000", 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038465299999999999 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109101+0000", 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021259324 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109209+0000", 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107187 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109313+0000", 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104003 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.36273.0:191763 7.5 7:a9d168ed:::benchmark_data_smithi027_271924_object75422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 514.665792173, 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.018041102, 2024-08-29T15:41:23.831 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.36273", 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/657524192", 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 191763 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091544+0000", 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091547+0000", 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8590000000000001e-06 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091556+0000", 2024-08-29T15:41:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4599999999999992e-06 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091580+0000", 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4314000000000001e-05 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091592+0000", 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1768e-05 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091686+0000", 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4029000000000004e-05 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:49.091696+0000", 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7410000000000004e-06 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084389+0000", 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99269340699999997 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084526+0000", 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136444 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.084664+0000", 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013856400000000001 2024-08-29T15:41:23.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087792+0000", 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003127286 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.087905+0000", 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113612 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109403+0000", 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021497480999999999 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109491+0000", 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7936000000000006e-05 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:32:50.109585+0000", 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4201000000000002e-05 2024-08-29T15:41:23.838 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185381 8.a 8:593dd5d4:::benchmark_data_smithi027_306999_object58602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 227.379555939, 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8689627519999998, 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.839 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185381 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377781+0000", 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377785+0000", 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5480000000000002e-06 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377794+0000", 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6290000000000005e-06 2024-08-29T15:41:23.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377804+0000", 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3079999999999997e-06 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.387009+0000", 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092053380000000004 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.387019+0000", 2024-08-29T15:41:23.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8940000000000005e-06 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088587+0000", 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70156798499999995 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088600+0000", 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2778e-05 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238539+0000", 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1499392150000001 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238659+0000", 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012044699999999999 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238741+0000", 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1568000000000002e-05 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246057+0000", 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007315963 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246083+0000", 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5687999999999999e-05 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246298+0000", 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021570700000000001 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246644+0000", 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034599799999999998 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246681+0000", 2024-08-29T15:41:23.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6872999999999997e-05 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246743+0000", 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1496999999999998e-05 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185404 8.a 8:510e3083:::benchmark_data_smithi027_306999_object58625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 227.33387422600001, 2024-08-29T15:41:23.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.823589948, 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185404 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423463+0000", 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4500000000000001e-06 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423469+0000", 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1870000000000002e-06 2024-08-29T15:41:23.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423483+0000", 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3193000000000001e-05 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423489+0000", 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0220000000000001e-06 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423549+0000", 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0466e-05 2024-08-29T15:41:23.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423556+0000", 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9029999999999998e-06 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088626+0000", 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.66507021300000002 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088634+0000", 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9379999999999992e-06 2024-08-29T15:41:23.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242305+0000", 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1536706910000001 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242434+0000", 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129484 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242523+0000", 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8343000000000004e-05 2024-08-29T15:41:23.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246560+0000", 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040375419999999999 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246569+0000", 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6570000000000003e-06 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246605+0000", 2024-08-29T15:41:23.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6331000000000002e-05 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246965+0000", 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00036038900000000002 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246996+0000", 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0054000000000001e-05 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.247052+0000", 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6085000000000003e-05 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218179 8.a 8:58241dc9:::benchmark_data_smithi027_306999_object91400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.086769+0000", 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 182.67056641900001, 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0426059379999999, 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218179 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086769+0000", 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086769+0000", 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086772+0000", 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3649999999999998e-06 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086779+0000", 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3629999999999996e-06 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086803+0000", 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3312999999999999e-05 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086812+0000", 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6879999999999994e-06 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.088104+0000", 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012921460000000001 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.088245+0000", 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000140309 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.088389+0000", 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014480600000000001 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.097809+0000", 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094199079999999994 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.097859+0000", 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9901999999999998e-05 2024-08-29T15:41:23.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.108191+0000", 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010332134 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110586+0000", 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002394611 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110597+0000", 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1059e-05 2024-08-29T15:41:23.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129298+0000", 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.018701063 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129375+0000", 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7270999999999999e-05 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.861 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218195 8.a 8:55cc424e:::benchmark_data_smithi027_306999_object91416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.098464+0000", 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 182.658872339, 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0314149020000001, 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218195 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.862 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098464+0000", 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098464+0000", 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098466+0000", 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.951e-06 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098471+0000", 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4029999999999998e-06 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098488+0000", 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6866000000000001e-05 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098496+0000", 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1280000000000008e-06 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098814+0000", 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031784800000000002 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.099180+0000", 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00036671700000000002 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.099316+0000", 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013535700000000001 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110692+0000", 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011376569 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110701+0000", 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1249999999999993e-06 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129806+0000", 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0191050269999999 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129878+0000", 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2910999999999994e-05 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218198 8.a 8:5081af65:::benchmark_data_smithi027_306999_object91419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:41:23.867 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.104328+0000", 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 182.653007492, 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414401179999999, 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218198 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104328+0000", 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104328+0000", 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104330+0000", 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104335+0000", 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8790000000000002e-06 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104347+0000", 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2407e-05 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104354+0000", 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1810000000000004e-06 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110741+0000", 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006386404 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110750+0000", 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6569999999999992e-06 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129925+0000", 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.019174354 2024-08-29T15:41:23.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.130034+0000", 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010951999999999999 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.130160+0000", 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012536900000000001 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137494+0000", 2024-08-29T15:41:23.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073345889999999999 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142034+0000", 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045402749999999999 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145649+0000", 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036144290000000002 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145713+0000", 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4381999999999994e-05 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145769+0000", 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5356000000000003e-05 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218219 8.a 8:5cb7da76:::benchmark_data_smithi027_306999_object91440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:41:23.874 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.131766+0000", 2024-08-29T15:41:23.875 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 182.62557018800001, 2024-08-29T15:41:23.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:23.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0142084140000001, 2024-08-29T15:41:23.876 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.876 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.876 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.876 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218219 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131766+0000", 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131766+0000", 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131768+0000", 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.782e-06 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131773+0000", 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9880000000000001e-06 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131792+0000", 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8474999999999998e-05 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131800+0000", 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3070000000000003e-06 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.132949+0000", 2024-08-29T15:41:23.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001149207 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.132957+0000", 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0069999999999997e-06 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.133800+0000", 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.000842011 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.133898+0000", 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8140000000000006e-05 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.133972+0000", 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4816999999999999e-05 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137603+0000", 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036306039999999999 2024-08-29T15:41:23.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145782+0000", 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0081790950000000008 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145894+0000", 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011136700000000001 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145933+0000", 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9104999999999999e-05 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145974+0000", 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1508999999999998e-05 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218223 8.a 8:5d762aaa:::benchmark_data_smithi027_306999_object91444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:41:23.883 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.134391+0000", 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 182.622944953, 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0117640050000001, 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218223 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134391+0000", 2024-08-29T15:41:23.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134391+0000", 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134393+0000", 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6700000000000001e-06 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134398+0000", 2024-08-29T15:41:23.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9529999999999998e-06 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134410+0000", 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2024e-05 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134417+0000", 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1350000000000003e-06 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134470+0000", 2024-08-29T15:41:23.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3601999999999999e-05 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134478+0000", 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9019999999999999e-06 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137660+0000", 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.003182156 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137751+0000", 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0586000000000003e-05 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137875+0000", 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012370299999999999 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145834+0000", 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079591920000000003 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145984+0000", 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015044099999999999 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146083+0000", 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8190999999999996e-05 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146112+0000", 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9539999999999998e-05 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146155+0000", 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2910000000000001e-05 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218252 8.a 8:50ec3c52:::benchmark_data_smithi027_306999_object91473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.166915+0000", 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 182.590420545, 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99340993799999999, 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.890 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218252 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166915+0000", 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166915+0000", 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166917+0000", 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.469e-06 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166922+0000", 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8760000000000004e-06 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166934+0000", 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1935e-05 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166941+0000", 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7209999999999994e-06 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166998+0000", 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6833999999999998e-05 2024-08-29T15:41:23.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.167006+0000", 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8630000000000008e-06 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142076+0000", 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97506998899999997 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142153+0000", 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7260000000000002e-05 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142233+0000", 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9566000000000004e-05 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145988+0000", 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003755525 2024-08-29T15:41:23.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146029+0000", 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0460000000000002e-05 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160213+0000", 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014184311999999999 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160276+0000", 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2817999999999994e-05 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160325+0000", 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9310000000000001e-05 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218281 8.a 8:52ee9bfa:::benchmark_data_smithi027_306999_object91502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.197901+0000", 2024-08-29T15:41:23.897 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 182.55943477700001, 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96259614299999996, 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218281 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197901+0000", 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197901+0000", 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197902+0000", 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4099999999999997e-07 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:23.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197906+0000", 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1949999999999996e-06 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197916+0000", 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7680000000000003e-06 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197923+0000", 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1330000000000001e-06 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197987+0000", 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4399999999999993e-05 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197995+0000", 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2320000000000004e-06 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146210+0000", 2024-08-29T15:41:23.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94821484599999994 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146314+0000", 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010488999999999999 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146425+0000", 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110705 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160082+0000", 2024-08-29T15:41:23.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013656975 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160099+0000", 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6920999999999999e-05 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160393+0000", 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029413600000000002 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:41:23.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160434+0000", 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1162999999999999e-05 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160497+0000", 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2937999999999999e-05 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:23.904 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:41:23.905 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 86 16 100099 100083 72.7236 0 - 0.0100864 2024-08-29T15:41:23.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:23.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:23.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:24.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:24.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:41:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.054 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:24.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:24.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:24.187 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:24.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:24.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:24.324 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:41:24.339 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:24 smithi027 ceph-mon[93921]: pgmap v1553: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 4.7 MiB/s wr, 75 op/s; 24939/300336 objects degraded (8.304%); 9861/300336 objects misplaced (3.283%); 20 MiB/s, 323 objects/s recovering 2024-08-29T15:41:24.339 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1910050994' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:41:24.339 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:24 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:41:24.339 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:24 smithi027 ceph-mon[98970]: pgmap v1553: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 4.7 MiB/s wr, 75 op/s; 24939/300336 objects degraded (8.304%); 9861/300336 objects misplaced (3.283%); 20 MiB/s, 323 objects/s recovering 2024-08-29T15:41:24.339 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1910050994' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:41:24.339 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:24 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:41:24.413 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:24.435 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:24.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:24.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:24.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:24.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:24.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:24.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:24.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:24 smithi171 ceph-mon[90031]: pgmap v1553: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 4.7 MiB/s wr, 75 op/s; 24939/300336 objects degraded (8.304%); 9861/300336 objects misplaced (3.283%); 20 MiB/s, 323 objects/s recovering 2024-08-29T15:41:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1910050994' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:41:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:24 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:41:24.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:24.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:24.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:24.893 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 87 16 100171 100155 71.9378 0.204545 0.014091 0.013895 2024-08-29T15:41:24.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:24.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:24.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:25.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:25.104 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:25.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:25.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:25.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:25.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:25.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:25.384 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:25.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:25.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:25.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:25.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:25 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:41:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:25 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:41:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:25 smithi027 ceph-mon[98970]: osdmap e444: 8 total, 8 up, 8 in 2024-08-29T15:41:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:25 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:41:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:25 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:41:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:25 smithi027 ceph-mon[93921]: osdmap e444: 8 total, 8 up, 8 in 2024-08-29T15:41:25.629 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:25.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:25.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:25 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:41:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:25 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:41:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:25 smithi171 ceph-mon[90031]: osdmap e444: 8 total, 8 up, 8 in 2024-08-29T15:41:25.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:25.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:25.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:25.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:25.893 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 88 16 100955 100939 71.6771 49 0.0084793 0.0139455 2024-08-29T15:41:25.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:25.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:25.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:26.061 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:26.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:26.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:26.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:26.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:26.231 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:26.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:26.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:26.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:26.367 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:26.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:26.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:26.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:26.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:26.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:26.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:26.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:26.651 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:26.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:26.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:26 smithi171 ceph-mon[90031]: pgmap v1555: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 4.2 MiB/s wr, 67 op/s; 24938/300339 objects degraded (8.303%); 9532/300339 objects misplaced (3.174%); 17 MiB/s, 272 objects/s recovering 2024-08-29T15:41:26.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:26.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:26.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:26.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:26 smithi027 ceph-mon[93921]: pgmap v1555: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 4.2 MiB/s wr, 67 op/s; 24938/300339 objects degraded (8.303%); 9532/300339 objects misplaced (3.174%); 17 MiB/s, 272 objects/s recovering 2024-08-29T15:41:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:26 smithi027 ceph-mon[98970]: pgmap v1555: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 4.2 MiB/s wr, 67 op/s; 24938/300339 objects degraded (8.303%); 9532/300339 objects misplaced (3.174%); 17 MiB/s, 272 objects/s recovering 2024-08-29T15:41:26.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:41:26.893 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 89 16 101751 101735 71.4307 49.75 0.0243583 0.0139935 2024-08-29T15:41:26.906 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:26.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:26.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:26.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:27.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:27.093 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:27.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:27.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:27.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:27.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:27.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:27.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:27.317 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:27.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:27.414 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:27.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:27.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:27.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:27.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:27.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 24938/300339 objects degraded (8.303%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:27.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 24938/300339 objects degraded (8.303%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:27.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:27.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:27.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 24938/300339 objects degraded (8.303%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:27.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:27.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:27.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:27.893 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 90 4 102427 102423 71.1148 43 0.0137928 0.0140558 2024-08-29T15:41:27.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:27.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:27.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:27.985 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:28.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:28.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:28.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:28.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:28.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:28.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:28.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:28.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:28.384 INFO:teuthology.orchestra.run.smithi027.stdout:moved 45 / 297 (15.1515%) 2024-08-29T15:41:28.384 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:41:28.384 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 2.93417 -> 7.30475 (expected baseline 5.69951) 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.2 with 32 -> 39 pgs (0.861953 -> 1.05051 * mean) 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.3 with 41 -> 43 pgs (1.10438 -> 1.15825 * mean) 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 3 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0329 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0362 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 weight 0.9000 -> 0.6210 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 weight 0.9000 -> 0.6406 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.9000 -> 0.6824 2024-08-29T15:41:28.385 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 weight 0.9000 -> 0.6839 2024-08-29T15:41:28.386 INFO:teuthology.orchestra.run.smithi027.stderr:no change 2024-08-29T15:41:28.386 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:28 smithi027 ceph-mon[93921]: pgmap v1556: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 6.4 MiB/s wr, 101 op/s; 24938/301425 objects degraded (8.273%); 9532/301425 objects misplaced (3.162%); 14 MiB/s, 229 objects/s recovering 2024-08-29T15:41:28.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:28 smithi027 ceph-mon[98970]: pgmap v1556: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 6.4 MiB/s wr, 101 op/s; 24938/301425 objects degraded (8.273%); 9532/301425 objects misplaced (3.162%); 14 MiB/s, 229 objects/s recovering 2024-08-29T15:41:28.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:28.458 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:28.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:28.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:28.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:28.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:28.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:28.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:28.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:28 smithi171 ceph-mon[90031]: pgmap v1556: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 6.4 MiB/s wr, 101 op/s; 24938/301425 objects degraded (8.273%); 9532/301425 objects misplaced (3.162%); 14 MiB/s, 229 objects/s recovering 2024-08-29T15:41:28.794 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:41:28 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:41:28.382+0000 7f7dea97a640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-08-29T15:41:28.811 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:28.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:28.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:28.893 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total time run: 90.0278 2024-08-29T15:41:28.893 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total writes made: 102427 2024-08-29T15:41:28.893 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Write size: 65536 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object size: 65536 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Bandwidth (MB/sec): 71.1079 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Bandwidth: 54.3216 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max bandwidth (MB/sec): 223.875 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min bandwidth (MB/sec): 0 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average IOPS: 1137 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev IOPS: 869.154 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max IOPS: 3582 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min IOPS: 0 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average Latency(s): 0.0140565 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Latency(s): 0.298762 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max latency(s): 25.1542 2024-08-29T15:41:28.894 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min latency(s): 0.00131181 2024-08-29T15:41:28.905 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Cleaning up (deleting benchmark objects) 2024-08-29T15:41:28.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:28.957 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:28.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:29.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:29.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:41:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:41:29] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-29T15:41:29.095 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:29.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:29.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:29.197 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:29.225 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:29.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:29.327 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:29.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:41:29.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:29.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:29.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:29.455 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:29.494 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:29.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:29.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:29.643 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:29.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:29.744 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:29 smithi171 ceph-mon[90031]: from='client.36963 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:41:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:29.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:29.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:29 smithi027 ceph-mon[93921]: from='client.36963 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:41:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:29.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:29 smithi027 ceph-mon[98970]: from='client.36963 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:29.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:41:29.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:29.876 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:29.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:29.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:29.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:30.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:30.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:30.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:30.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:30.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:30.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:30.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:30.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:30.413 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:30.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:30.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:30.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:30.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:30.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:30.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:30.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:30 smithi171 ceph-mon[90031]: pgmap v1557: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 8.9 MiB/s wr, 141 op/s; 24938/303123 objects degraded (8.227%); 9180/303123 objects misplaced (3.028%); 14 MiB/s, 224 objects/s recovering 2024-08-29T15:41:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:30 smithi027 ceph-mon[93921]: pgmap v1557: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 8.9 MiB/s wr, 141 op/s; 24938/303123 objects degraded (8.227%); 9180/303123 objects misplaced (3.028%); 14 MiB/s, 224 objects/s recovering 2024-08-29T15:41:30.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:30 smithi027 ceph-mon[98970]: pgmap v1557: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 8.9 MiB/s wr, 141 op/s; 24938/303123 objects degraded (8.227%); 9180/303123 objects misplaced (3.028%); 14 MiB/s, 224 objects/s recovering 2024-08-29T15:41:30.822 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:30.848 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:30.950 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:30.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:30.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:31.075 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:31.154 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:31.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:31.256 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:31.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:31.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:31.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:31.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:31.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:31.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:31.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:31.515 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:31.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:31.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:31.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:31.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:31.671 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:31.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:31.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:31.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:31.810 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:31.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:31.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:31.946 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:31.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:32.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:32.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:32.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:32.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:32.225 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:32.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:32.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:32.330 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-29T15:41:32.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-29T15:41:32.365 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:32.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:32.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:32.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:32.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:32.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:32 smithi027 ceph-mon[93921]: pgmap v1558: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 12 MiB/s wr, 229 op/s; 24868/305334 objects degraded (8.145%); 9015/305334 objects misplaced (2.953%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:41:32.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 24868/305334 objects degraded (8.145%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:32.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:32 smithi027 ceph-mon[98970]: pgmap v1558: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 12 MiB/s wr, 229 op/s; 24868/305334 objects degraded (8.145%); 9015/305334 objects misplaced (2.953%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:41:32.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 24868/305334 objects degraded (8.145%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:32.597 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:32.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:32.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:32.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:32.732 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:32.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:32.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:32 smithi171 ceph-mon[90031]: pgmap v1558: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 12 MiB/s wr, 229 op/s; 24868/305334 objects degraded (8.145%); 9015/305334 objects misplaced (2.953%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:41:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 24868/305334 objects degraded (8.145%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:32.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:32.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:32.908 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:32.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:33.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:33.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:33.145 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:33.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:33.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:33.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:33.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:33.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:33.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:33.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:33.585 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:33.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:33.724 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:33.826 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:33.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:33.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:33.956 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:33.986 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:33.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:34.088 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:34.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:34.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:34.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:34.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:34.266 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:34.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:34.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:34.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:34.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:34.438 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:34.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:34.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:34.565 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:34.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:34.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:34 smithi027 ceph-mon[98970]: pgmap v1559: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 351 op/s; 24868/303843 objects degraded (8.184%); 8690/303843 objects misplaced (2.860%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:41:34.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:34 smithi027 ceph-mon[93921]: pgmap v1559: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 351 op/s; 24868/303843 objects degraded (8.184%); 8690/303843 objects misplaced (2.860%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:41:34.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:34.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:34.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:34.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:34.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:41:34.727 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:41:34.765 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:41:34.765 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:41:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:34 smithi171 ceph-mon[90031]: pgmap v1559: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 351 op/s; 24868/303843 objects degraded (8.184%); 8690/303843 objects misplaced (2.860%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:41:34.799 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:41:34.799 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:41:34.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:34.837 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:105815 9.7 e444/440 9:e8705d26:::benchmark_data_smithi027_341149_object3385:head v 444'6541, mlcod=0'0)", 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:41:34.570703+0000", 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.018137059000000001, 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018164801000000001, 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.570703+0000", 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.570703+0000", 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.570704+0000", 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1400000000000001e-06 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.570712+0000", 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7130000000000005e-06 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.570732+0000", 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0236000000000001e-05 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.570740+0000", 2024-08-29T15:41:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4659999999999999e-06 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.570948+0000", 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020702200000000001 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.571004+0000", 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6063000000000002e-05 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:105819 9.7 e444/440 9:edd12c46:::benchmark_data_smithi027_341149_object3389:head v 444'6542, mlcod=0'0)", 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:41:34.576725+0000", 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012115302999999999, 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012203324, 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.576725+0000", 2024-08-29T15:41:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.576725+0000", 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.576726+0000", 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2200000000000002e-07 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.576732+0000", 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6499999999999999e-06 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.576744+0000", 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1681999999999999e-05 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.576750+0000", 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7969999999999997e-06 2024-08-29T15:41:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577564+0000", 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00081386100000000003 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577589+0000", 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5058999999999998e-05 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:105823 9.7 e444/440)", 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:41:34.577485+0000", 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.01135516, 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011491078, 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577485+0000", 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577485+0000", 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577485+0000", 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9699999999999996e-07 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577491+0000", 2024-08-29T15:41:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0839999999999999e-06 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577503+0000", 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2208999999999999e-05 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.577508+0000", 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7830000000000004e-06 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:41:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:41:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 3 2024-08-29T15:41:34.849 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:41:34.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:41:34.850 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:41:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:34.851 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:34.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:34.876 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:41:34.877 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:41:34.911 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:41:34.912 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:41:34.946 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:41:34.946 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:41:34.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:34.977 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:41:34.977 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:41:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:34.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:34.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:35.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:35.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:35.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:35.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:35.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:35.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:35.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:35.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:35.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:35.468 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1224204650' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:35.468 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1224204650' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:35.508 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:35.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:35.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:35.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:35.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:35.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:35.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:35.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1224204650' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:35.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:35.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:35.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:35.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:35.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:35.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:35.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:35.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:36.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:36.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:36.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:36.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:36.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:36.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:36.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:36.370 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:36.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:36.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:36.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:36.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:36.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:36.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:36.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:36.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:36.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:36.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:36.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:36.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:36 smithi171 ceph-mon[90031]: pgmap v1560: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 96 B/s rd, 14 MiB/s wr, 418 op/s; 24449/301032 objects degraded (8.122%); 8552/301032 objects misplaced (2.841%); 9.1 MiB/s, 145 objects/s recovering 2024-08-29T15:41:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:36 smithi027 ceph-mon[98970]: pgmap v1560: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 96 B/s rd, 14 MiB/s wr, 418 op/s; 24449/301032 objects degraded (8.122%); 8552/301032 objects misplaced (2.841%); 9.1 MiB/s, 145 objects/s recovering 2024-08-29T15:41:36.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:36 smithi027 ceph-mon[93921]: pgmap v1560: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 96 B/s rd, 14 MiB/s wr, 418 op/s; 24449/301032 objects degraded (8.122%); 8552/301032 objects misplaced (2.841%); 9.1 MiB/s, 145 objects/s recovering 2024-08-29T15:41:36.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:36.888 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:36.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:36.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:36.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:37.016 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:37.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:37.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:37.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:37.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:37.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:37.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:37.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:37.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:37.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:37.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:37.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 24449/301032 objects degraded (8.122%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:37.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 24449/301032 objects degraded (8.122%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:37.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:41:37.523 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:37.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:37.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:37.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:37.560 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:37.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:37.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.691 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:37.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 24449/301032 objects degraded (8.122%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:37.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:37.826 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:37.928 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:37.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:37.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:38.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:38.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:38.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:38.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:38.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:38.227 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:38.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:38.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:38.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:38.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:38.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:38.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:38.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:38.503 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:38.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:38.604 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:38.628 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:38.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:38.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:38.668 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:38 smithi171 ceph-mon[90031]: pgmap v1561: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 469 op/s; 24449/297498 objects degraded (8.218%); 8552/297498 objects misplaced (2.875%); 6.4 MiB/s, 101 objects/s recovering 2024-08-29T15:41:38.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:38.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:38.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:38.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:38 smithi027 ceph-mon[98970]: pgmap v1561: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 469 op/s; 24449/297498 objects degraded (8.218%); 8552/297498 objects misplaced (2.875%); 6.4 MiB/s, 101 objects/s recovering 2024-08-29T15:41:38.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:38 smithi027 ceph-mon[93921]: pgmap v1561: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 469 op/s; 24449/297498 objects degraded (8.218%); 8552/297498 objects misplaced (2.875%); 6.4 MiB/s, 101 objects/s recovering 2024-08-29T15:41:38.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:38.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:38.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:38.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:39.004 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:39.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:39.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:41:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:41:39] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-29T15:41:39.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:39.161 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:39.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:39.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:41:39.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:39.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:39.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:39.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:39.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:39.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:39.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:39.711 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:39.716 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:39.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:39.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:39.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:39.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:39.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:40.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:40.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:40.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:40.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:40.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:40.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:40.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:40.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:40.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:40.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:40.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:40.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:40.456 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:40 smithi027 ceph-mon[98970]: pgmap v1562: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 522 op/s; 24449/294513 objects degraded (8.302%); 8258/294513 objects misplaced (2.804%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:41:40.456 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:40 smithi027 ceph-mon[93921]: pgmap v1562: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 522 op/s; 24449/294513 objects degraded (8.302%); 8258/294513 objects misplaced (2.804%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:41:40.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:40.569 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:40.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:40.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:40.702 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:40.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:40.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:40 smithi171 ceph-mon[90031]: pgmap v1562: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 10 MiB/s wr, 522 op/s; 24449/294513 objects degraded (8.302%); 8258/294513 objects misplaced (2.804%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:41:40.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:40.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:40.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:40.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:40.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:40.965 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:40.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:40.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:41.067 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:41.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:41.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:41.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:41.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:41.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:41.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:41.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:41.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:41.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:41.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:41.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:41.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:41.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:41.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:41.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:41.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:41.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:41.734 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:41.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:41.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:41.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:41.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:41.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:41.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:41.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:42.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:42.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:42.168 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:42.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:42.299 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:42.401 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:42.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:42.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:42.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:42.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:42.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:42.695 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:42 smithi171 ceph-mon[90031]: pgmap v1563: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 602 op/s; 24035/289905 objects degraded (8.291%); 8120/289905 objects misplaced (2.801%); 7.9 MiB/s, 125 objects/s recovering 2024-08-29T15:41:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 24035/289905 objects degraded (8.291%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:42.797 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:42 smithi027 ceph-mon[98970]: pgmap v1563: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 602 op/s; 24035/289905 objects degraded (8.291%); 8120/289905 objects misplaced (2.801%); 7.9 MiB/s, 125 objects/s recovering 2024-08-29T15:41:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 24035/289905 objects degraded (8.291%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:42.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:42 smithi027 ceph-mon[93921]: pgmap v1563: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.4 MiB/s wr, 602 op/s; 24035/289905 objects degraded (8.291%); 8120/289905 objects misplaced (2.801%); 7.9 MiB/s, 125 objects/s recovering 2024-08-29T15:41:42.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 24035/289905 objects degraded (8.291%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:42.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:42.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:42.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:42.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:42.959 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:42.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:43.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:43.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:43.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:43.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:43.207 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:43.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:43.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:43.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:43.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:43.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:43.418 INFO:teuthology.orchestra.run.smithi027.stdout:moved 12 / 297 (4.0404%) 2024-08-29T15:41:43.418 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:41:43.418 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 2.93417 -> 1.45237 (expected baseline 5.69951) 2024-08-29T15:41:43.418 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.2 with 32 -> 35 pgs (0.861953 -> 0.942761 * mean) 2024-08-29T15:41:43.418 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.3 with 41 -> 38 pgs (1.10438 -> 1.02357 * mean) 2024-08-29T15:41:43.418 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:41:43.418 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 0.05 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0323 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0355 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 weight 0.9000 -> 0.8500 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 weight 0.9000 -> 0.8500 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 weight 0.9000 -> 0.8500 2024-08-29T15:41:43.419 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.9000 -> 0.8500 2024-08-29T15:41:43.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:43.500 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:43.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:43.501 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'55708','3':'55708','5':'55708','7':'55708'}"} : dispatch 2024-08-29T15:41:43.501 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'55708','3':'55708','5':'55708','7':'55708'}"} : dispatch 2024-08-29T15:41:43.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:43.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:43.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:43.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:43.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:43.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'55708','3':'55708','5':'55708','7':'55708'}"} : dispatch 2024-08-29T15:41:43.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:43.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:43.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:43.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:44.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:44.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:44.102 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:44.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:44.225 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:44.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:44.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:44.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:44.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:44.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:44.422 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:41:44.475 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:44.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:44.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:44.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:44.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:44.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:44.748 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:44 smithi171 ceph-mon[90031]: from='client.56539 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:44 smithi171 ceph-mon[90031]: pgmap v1564: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.9 MiB/s wr, 622 op/s; 24035/285069 objects degraded (8.431%); 7827/285069 objects misplaced (2.746%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:41:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'55708','3':'55708','5':'55708','7':'55708'}"}]': finished 2024-08-29T15:41:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:44 smithi171 ceph-mon[90031]: osdmap e445: 8 total, 8 up, 8 in 2024-08-29T15:41:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/977864322' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:41:44.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[93921]: from='client.56539 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:44.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[93921]: pgmap v1564: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.9 MiB/s wr, 622 op/s; 24035/285069 objects degraded (8.431%); 7827/285069 objects misplaced (2.746%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:41:44.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'55708','3':'55708','5':'55708','7':'55708'}"}]': finished 2024-08-29T15:41:44.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[93921]: osdmap e445: 8 total, 8 up, 8 in 2024-08-29T15:41:44.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:44.799 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/977864322' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:41:44.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[98970]: from='client.56539 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:41:44.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[98970]: pgmap v1564: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 1.9 MiB/s wr, 622 op/s; 24035/285069 objects degraded (8.431%); 7827/285069 objects misplaced (2.746%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:41:44.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'55708','3':'55708','5':'55708','7':'55708'}"}]': finished 2024-08-29T15:41:44.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[98970]: osdmap e445: 8 total, 8 up, 8 in 2024-08-29T15:41:44.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:44.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/977864322' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:41:44.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:44.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:44.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:44.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:44.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:44.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:45.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:45.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:45.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:45.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:45.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:45.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:45.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:45.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:45.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:45.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:45.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:45.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:45.475 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:45.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:45.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:45.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:45.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:45.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:45.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:45.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:45.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:45.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:45 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:41:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/977864322' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:41:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:45 smithi171 ceph-mon[90031]: osdmap e446: 8 total, 8 up, 8 in 2024-08-29T15:41:45.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:45 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:41:45.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/977864322' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:41:45.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:45 smithi027 ceph-mon[98970]: osdmap e446: 8 total, 8 up, 8 in 2024-08-29T15:41:45.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:45 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:41:45.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/977864322' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:41:45.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:45 smithi027 ceph-mon[93921]: osdmap e446: 8 total, 8 up, 8 in 2024-08-29T15:41:45.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:45.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:45.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:45.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:46.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:46.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:46.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:46.178 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:46.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:46.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:46.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:46.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:46.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:46.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:46.519 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-29T15:41:46.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-08-29T15:41:46.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:46.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:46.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:46.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:46.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:46.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:46 smithi027 ceph-mon[98970]: pgmap v1567: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 23726/282810 objects degraded (8.389%); 7632/282810 objects misplaced (2.699%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:41:46.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:46 smithi027 ceph-mon[98970]: osdmap e447: 8 total, 8 up, 8 in 2024-08-29T15:41:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:46 smithi027 ceph-mon[93921]: pgmap v1567: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 23726/282810 objects degraded (8.389%); 7632/282810 objects misplaced (2.699%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:41:46.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:46 smithi027 ceph-mon[93921]: osdmap e447: 8 total, 8 up, 8 in 2024-08-29T15:41:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:46 smithi171 ceph-mon[90031]: pgmap v1567: 99 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 94 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 23726/282810 objects degraded (8.389%); 7632/282810 objects misplaced (2.699%); 10 MiB/s, 160 objects/s recovering 2024-08-29T15:41:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:46 smithi171 ceph-mon[90031]: osdmap e447: 8 total, 8 up, 8 in 2024-08-29T15:41:46.810 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:41:46.810 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:41:46.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:46.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:46.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:46.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:46.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:46.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:47.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:47.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:47.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:47.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:47.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:47.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:47.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:47.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:47.406 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:47.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:47.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:47.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:47.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:47.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 23726/282810 objects degraded (8.389%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:47.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 23726/282810 objects degraded (8.389%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:47.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:47.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:47.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:47.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 23726/282810 objects degraded (8.389%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:47.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:47.816 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:47.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:47.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:47.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:47.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:47.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:48.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:48.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:48.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:48.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:48.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:48.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:48.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:48.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:41:48.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:48.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:48.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:48.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:48.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:48.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:48.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:48 smithi027 ceph-mon[98970]: pgmap v1569: 99 pgs: 1 activating+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 23726/280776 objects degraded (8.450%); 13886/280776 objects misplaced (4.946%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:41:48.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:48 smithi027 ceph-mon[93921]: pgmap v1569: 99 pgs: 1 activating+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 23726/280776 objects degraded (8.450%); 13886/280776 objects misplaced (4.946%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:41:48.584 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:48.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:48.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:48.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:48.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:48.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:48.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:48 smithi171 ceph-mon[90031]: pgmap v1569: 99 pgs: 1 activating+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 23726/280776 objects degraded (8.450%); 13886/280776 objects misplaced (4.946%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:41:48.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:48.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:48.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:48.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:49.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:49.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:49.121 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:41:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:41:49] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-08-29T15:41:49.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:49.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:49.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:49.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:49.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:49.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:49.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:49.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:49.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:49.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:49.470 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:41:49.470 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:41:49.498 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:49.511 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:41:49.512 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:41:49.524 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:49.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:49.542 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:41:49.542 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:41:49.576 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:41:49.576 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:41:49.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:49.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:49.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:49.606 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:41:49.607 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:41:49.639 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:41:49.640 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:41:49.654 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:49 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3036579554' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:49.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:49 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3036579554' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:49.670 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:41:49.670 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:41:49.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:49.705 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:41:49.705 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:41:49.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:49.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:49.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:49 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3036579554' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:41:49.841 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:49.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:49.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:49.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:50.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:50.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:50.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:50.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:50.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:50.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:50.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:50.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:50.396 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:50.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:50.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:50.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:50.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:50.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:50.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:50.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:50.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:50.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:50.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:50 smithi027 ceph-mon[93921]: pgmap v1570: 99 pgs: 1 activating+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 319 op/s; 29955/279309 objects degraded (10.725%); 13382/279309 objects misplaced (4.791%); 9.5 MiB/s, 152 objects/s recovering 2024-08-29T15:41:50.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:50 smithi027 ceph-mon[98970]: pgmap v1570: 99 pgs: 1 activating+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 319 op/s; 29955/279309 objects degraded (10.725%); 13382/279309 objects misplaced (4.791%); 9.5 MiB/s, 152 objects/s recovering 2024-08-29T15:41:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:50 smithi171 ceph-mon[90031]: pgmap v1570: 99 pgs: 1 activating+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 319 op/s; 29955/279309 objects degraded (10.725%); 13382/279309 objects misplaced (4.791%); 9.5 MiB/s, 152 objects/s recovering 2024-08-29T15:41:50.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:50.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:50.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:50.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:41:50.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:51.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:51.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:51.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:51.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:51.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:51.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:51.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:51.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:51.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:51.413 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:51.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:51.515 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:51.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:51.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:51.706 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:51.735 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:51.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:51.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:51.866 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:51.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:51.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:51.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:51.999 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:52.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:52.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:52.132 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:52.234 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:52.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:52.366 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:52.397 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:52.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:52.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:52.529 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:52.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:52.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:52.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:52 smithi171 ceph-mon[90031]: pgmap v1571: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 29654/275751 objects degraded (10.754%); 13185/275751 objects misplaced (4.781%); 10 MiB/s, 166 objects/s recovering 2024-08-29T15:41:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29654/275751 objects degraded (10.754%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:52.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:52.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:52.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:52 smithi027 ceph-mon[98970]: pgmap v1571: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 29654/275751 objects degraded (10.754%); 13185/275751 objects misplaced (4.781%); 10 MiB/s, 166 objects/s recovering 2024-08-29T15:41:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29654/275751 objects degraded (10.754%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:52.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:52 smithi027 ceph-mon[93921]: pgmap v1571: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 29654/275751 objects degraded (10.754%); 13185/275751 objects misplaced (4.781%); 10 MiB/s, 166 objects/s recovering 2024-08-29T15:41:52.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29654/275751 objects degraded (10.754%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:52.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:52.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:52.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:52.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:53.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:53.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:53.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:53.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:53.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:41:53.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:53.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:53.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:53.374 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:53.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:53.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:53.515 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:53.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:53.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:53.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:53.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:53.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:53.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:53.777 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:53.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:53.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:53.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:53.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:53.914 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:53.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:54.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:54.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:54.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:54.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:54.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:54.183 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:54.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:54.285 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:54.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:54.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:54.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:54.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:54.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:54.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:54.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:54.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:54.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:54.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:54.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:54.725 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:54.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:54.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:54 smithi171 ceph-mon[90031]: pgmap v1572: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 29330/271662 objects degraded (10.797%); 12812/271662 objects misplaced (4.716%); 13 MiB/s, 209 objects/s recovering 2024-08-29T15:41:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:41:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:54 smithi027 ceph-mon[98970]: pgmap v1572: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 29330/271662 objects degraded (10.797%); 12812/271662 objects misplaced (4.716%); 13 MiB/s, 209 objects/s recovering 2024-08-29T15:41:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:41:54.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:54 smithi027 ceph-mon[93921]: pgmap v1572: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 29330/271662 objects degraded (10.797%); 12812/271662 objects misplaced (4.716%); 13 MiB/s, 209 objects/s recovering 2024-08-29T15:41:54.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:41:54.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:54.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:54.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:54.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:54.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:54.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:55.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:55.101 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:55.140 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:55.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:55.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:55.400 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:55.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:55.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:55.533 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:55.568 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:55.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:55.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:55.701 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:55.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:55.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:55.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:55.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:55.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:55.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:55.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:56.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:56.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:56.098 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:56.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:56.199 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:56.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:56.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:56.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:56.361 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:56.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:56.463 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:41:56.494 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:56.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:56.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:41:56.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:56.630 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:56.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:56.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:56.732 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:56.766 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:56.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:56.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:56 smithi171 ceph-mon[90031]: pgmap v1573: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 29033/269160 objects degraded (10.787%); 12655/269160 objects misplaced (4.702%); 11 MiB/s, 175 objects/s recovering 2024-08-29T15:41:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:56 smithi027 ceph-mon[93921]: pgmap v1573: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 29033/269160 objects degraded (10.787%); 12655/269160 objects misplaced (4.702%); 11 MiB/s, 175 objects/s recovering 2024-08-29T15:41:56.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:56 smithi027 ceph-mon[98970]: pgmap v1573: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 29033/269160 objects degraded (10.787%); 12655/269160 objects misplaced (4.702%); 11 MiB/s, 175 objects/s recovering 2024-08-29T15:41:56.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:56.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:56.906 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:56.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:57.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:57.050 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:57.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:57.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:57.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:57.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:57.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:57.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:57.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:57.323 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:57.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:57.424 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:57.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:57.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:57.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:57.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:57.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:57.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:57.702 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:57.731 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:57.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:57.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 29033/269160 objects degraded (10.787%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:57.787 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:41:57.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 29033/269160 objects degraded (10.787%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:57.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 29033/269160 objects degraded (10.787%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:41:57.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:57.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:57.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:57.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:57.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:58.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:58.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:58.144 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:58.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:58.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:41:58.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.283 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:58.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:41:58.384 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:41:58.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:58.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:41:58.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:58.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:58.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:58.661 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:58 smithi027 ceph-mon[98970]: pgmap v1574: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 29033/266208 objects degraded (10.906%); 12655/266208 objects misplaced (4.754%); 9.5 MiB/s, 151 objects/s recovering 2024-08-29T15:41:58.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:58 smithi027 ceph-mon[93921]: pgmap v1574: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 29033/266208 objects degraded (10.906%); 12655/266208 objects misplaced (4.754%); 9.5 MiB/s, 151 objects/s recovering 2024-08-29T15:41:58.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:58 smithi171 ceph-mon[90031]: pgmap v1574: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 29033/266208 objects degraded (10.906%); 12655/266208 objects misplaced (4.754%); 9.5 MiB/s, 151 objects/s recovering 2024-08-29T15:41:58.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:41:58.842 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:58.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:41:58.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:58.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:58.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:58.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:59.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:41:59.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:59.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:59.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:41:59.217 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:41:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:41:59] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-08-29T15:41:59.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:41:59.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:41:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:59.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:41:59.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:59.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:59.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:59.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:59.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:41:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:59.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:59.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:59.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:41:59.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:59.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:41:59.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:41:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:59.787 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:59.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:41:59.810 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:41:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:41:59.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:41:59.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:41:59.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:41:59.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:00.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:00.060 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:00.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:00.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:00.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:00.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:00.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:00.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:00.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:00.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:00.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:00.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:00.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:00.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:00.614 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:42:00.614 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:42:00.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:42:00.614 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:42:00.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.614 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185356 8.c 8:3e65ed30:::benchmark_data_smithi027_306999_object58577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:42:00.614 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 262.78876571699999, 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.954196238, 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185356 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.615 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347621+0000", 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4959999999999999e-06 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347626+0000", 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7110000000000003e-06 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347636+0000", 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7690000000000008e-06 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347643+0000", 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9439999999999999e-06 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347708+0000", 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5822999999999996e-05 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347924+0000", 2024-08-29T15:42:00.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021531600000000001 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.348044+0000", 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120091 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359802+0000", 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011758006 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359853+0000", 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1329999999999998e-05 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390576+0000", 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030722635000000002 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390589+0000", 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2792000000000001e-05 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085219+0000", 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69463000399999997 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085233+0000", 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3964e-05 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.301754+0000", 2024-08-29T15:42:00.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2165213420000001 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.301816+0000", 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2014999999999995e-05 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185365 8.c 8:34b703f9:::benchmark_data_smithi027_306999_object58586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 262.78252280300001, 2024-08-29T15:42:00.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.948337904, 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185365 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353864+0000", 2024-08-29T15:42:00.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.136e-06 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353867+0000", 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4340000000000001e-06 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353876+0000", 2024-08-29T15:42:00.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8980000000000002e-06 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353882+0000", 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9800000000000003e-06 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359882+0000", 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059996040000000004 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359992+0000", 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110373 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.360098+0000", 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010622499999999999 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390623+0000", 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030524273000000001 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390631+0000", 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9530000000000005e-06 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085266+0000", 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69463474700000005 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085276+0000", 2024-08-29T15:42:00.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3610000000000007e-06 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302132+0000", 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2168561900000001 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302200+0000", 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8730000000000001e-05 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.629 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185370 8.c 8:321ac8e5:::benchmark_data_smithi027_306999_object58591:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 262.77688864200002, 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.942814085, 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.630 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185370 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359498+0000", 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7150000000000001e-06 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359504+0000", 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9279999999999998e-06 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359518+0000", 2024-08-29T15:42:00.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3708e-05 2024-08-29T15:42:00.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359526+0000", 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5100000000000001e-06 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.367931+0000", 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008405615 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.368066+0000", 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013519199999999999 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.368197+0000", 2024-08-29T15:42:00.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013018200000000001 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390661+0000", 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022464378 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390669+0000", 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1960000000000008e-06 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085313+0000", 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69464367800000004 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085321+0000", 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0919999999999998e-06 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302245+0000", 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2169240729999999 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302311+0000", 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5817999999999999e-05 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185373 8.c 8:3539b1da:::benchmark_data_smithi027_306999_object58594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:42:00.638 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 262.77317321599998, 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9786454089999999, 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185373 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363214+0000", 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4559999999999999e-06 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363221+0000", 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0700000000000001e-06 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363234+0000", 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3397999999999999e-05 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363244+0000", 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7629999999999995e-06 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390698+0000", 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027454427 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390706+0000", 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2250000000000003e-06 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085347+0000", 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69464127399999998 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085355+0000", 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3180000000000007e-06 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302347+0000", 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.216992074 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302464+0000", 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116505 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.303107+0000", 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064362000000000002 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329229+0000", 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.026121961999999999 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329277+0000", 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7749e-05 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338695+0000", 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094183740000000002 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338857+0000", 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016147700000000001 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.339098+0000", 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024068099999999999 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341858+0000", 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002760036 2024-08-29T15:42:00.646 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185396 8.c 8:37b7f43f:::benchmark_data_smithi027_306999_object58617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 262.72777806300002, 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9334865830000001, 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.647 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185396 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408609+0000", 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.114e-06 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408615+0000", 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9900000000000002e-06 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408631+0000", 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5426000000000001e-05 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408639+0000", 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3499999999999997e-06 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408716+0000", 2024-08-29T15:42:00.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6490999999999997e-05 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408726+0000", 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0570999999999999e-05 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085379+0000", 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.67665228200000005 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085386+0000", 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8730000000000007e-06 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319619+0000", 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2342329700000001 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319745+0000", 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125611 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319880+0000", 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134673 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329291+0000", 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094111300000000002 2024-08-29T15:42:00.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329493+0000", 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020221700000000001 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338735+0000", 2024-08-29T15:42:00.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092423220000000007 2024-08-29T15:42:00.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341916+0000", 2024-08-29T15:42:00.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031804369999999999 2024-08-29T15:42:00.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341975+0000", 2024-08-29T15:42:00.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9617000000000002e-05 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342094+0000", 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118509 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185406 8.c 8:388927f1:::benchmark_data_smithi027_306999_object58627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:42:00.657 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 262.70812376700002, 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9141273249999999, 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185406 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1699999999999997e-07 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428267+0000", 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1339999999999997e-06 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428278+0000", 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0927999999999999e-05 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428283+0000", 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1510000000000003e-06 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428329+0000", 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6029000000000003e-05 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428334+0000", 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3979999999999998e-06 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085411+0000", 2024-08-29T15:42:00.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.657076836 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085419+0000", 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.644e-06 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.323864+0000", 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2384456589999999 2024-08-29T15:42:00.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.323988+0000", 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012354099999999999 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.324118+0000", 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129987 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329297+0000", 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051789770000000004 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338774+0000", 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094772769999999992 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338812+0000", 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7799000000000002e-05 2024-08-29T15:42:00.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342228+0000", 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034166130000000002 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342282+0000", 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3149999999999998e-05 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342389+0000", 2024-08-29T15:42:00.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107385 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210331 8.c 8:3aae235d:::benchmark_data_smithi027_306999_object83552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.11298713100001, 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.042995135, 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.666 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210331 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023400+0000", 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.875e-06 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023406+0000", 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5149999999999997e-06 2024-08-29T15:42:00.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023422+0000", 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6733999999999999e-05 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023430+0000", 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7279999999999995e-06 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024594+0000", 2024-08-29T15:42:00.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011642289999999999 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024726+0000", 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013213600000000001 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024866+0000", 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013915999999999999 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.670 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.033244+0000", 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008378155 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.034487+0000", 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012431300000000001 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.035958+0000", 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001471539 2024-08-29T15:42:00.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.040150+0000", 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041912520000000003 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.040161+0000", 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1744e-05 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066310+0000", 2024-08-29T15:42:00.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.02614841 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066393+0000", 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3528000000000004e-05 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.673 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210341 8.c 8:3b7d0476:::benchmark_data_smithi027_306999_object83562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.09761059300001, 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0358083629999999, 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210341 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.674 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038776+0000", 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.409e-06 2024-08-29T15:42:00.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038781+0000", 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5680000000000001e-06 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038791+0000", 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0149000000000001e-05 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038799+0000", 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8630000000000008e-06 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.041098+0000", 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022990480000000002 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.041106+0000", 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0080000000000002e-06 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066885+0000", 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0257787739999999 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066990+0000", 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010541500000000001 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.067139+0000", 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014869899999999999 2024-08-29T15:42:00.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074258+0000", 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007119383 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074420+0000", 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016190599999999999 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074489+0000", 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8998000000000006e-05 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074542+0000", 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2697e-05 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074583+0000", 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1446000000000003e-05 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.680 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210361 8.c 8:36e2a08e:::benchmark_data_smithi027_306999_object83582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.075777469, 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.026884127, 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210361 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060609+0000", 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5700000000000002e-07 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060613+0000", 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0419999999999996e-06 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060624+0000", 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0821e-05 2024-08-29T15:42:00.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060630+0000", 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2890000000000003e-06 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060687+0000", 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7071000000000003e-05 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060695+0000", 2024-08-29T15:42:00.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.605e-06 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070807+0000", 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.010112353 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070906+0000", 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8680999999999999e-05 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070983+0000", 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7495000000000003e-05 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074476+0000", 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003492673 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074595+0000", 2024-08-29T15:42:00.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119187 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087278+0000", 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012683589 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087441+0000", 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016204500000000001 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087492+0000", 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1518999999999999e-05 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210366 8.c 8:3bfc388b:::benchmark_data_smithi027_306999_object83587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.06742901999999, 2024-08-29T15:42:00.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0187623729999999, 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210366 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:42:00.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068958+0000", 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3519999999999999e-06 2024-08-29T15:42:00.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068962+0000", 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7690000000000004e-06 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068975+0000", 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.217e-05 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068982+0000", 2024-08-29T15:42:00.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.069045+0000", 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2791000000000002e-05 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.069054+0000", 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5640000000000002e-06 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074624+0000", 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.005569739 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074808+0000", 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184026 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074951+0000", 2024-08-29T15:42:00.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014334600000000001 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087555+0000", 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012603130000000001 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087570+0000", 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5834000000000001e-05 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087625+0000", 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4398999999999999e-05 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087681+0000", 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6036999999999999e-05 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087719+0000", 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7907999999999997e-05 2024-08-29T15:42:00.695 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210405 8.c 8:3354f347:::benchmark_data_smithi027_306999_object83626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.023695683, 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98371990300000001, 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.696 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210405 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112691+0000", 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5740000000000001e-06 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112697+0000", 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5999999999999997e-06 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112714+0000", 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6898e-05 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112721+0000", 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6669999999999996e-06 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112782+0000", 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0148000000000001e-05 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112790+0000", 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1089999999999998e-06 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083315+0000", 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97052495400000005 2024-08-29T15:42:00.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083455+0000", 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139925 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083591+0000", 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013614500000000001 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.091643+0000", 2024-08-29T15:42:00.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080527510000000004 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.095910+0000", 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042662530000000002 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.095950+0000", 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0290000000000002e-05 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096361+0000", 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041099000000000002 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096410+0000", 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8599e-05 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210417 8.c 8:351de282:::benchmark_data_smithi027_306999_object83638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:42:00.703 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.00912069, 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97409499899999996, 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210417 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.704 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127266+0000", 2024-08-29T15:42:00.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.291e-06 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127271+0000", 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3689999999999998e-06 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127281+0000", 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9320000000000008e-06 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127289+0000", 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5730000000000002e-06 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127353+0000", 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4462999999999996e-05 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127362+0000", 2024-08-29T15:42:00.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1330000000000005e-06 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087778+0000", 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96041514900000002 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087926+0000", 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014825999999999999 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:42:00.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.088065+0000", 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013917199999999999 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096459+0000", 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008393546 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101179+0000", 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047199649999999996 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101267+0000", 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8187000000000001e-05 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101319+0000", 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2432999999999998e-05 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101360+0000", 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0525999999999998e-05 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236777 8.f 8:f2ec1397:::benchmark_data_smithi027_306999_object109998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.236625+0000", 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.89976000199999, 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0719445379999999, 2024-08-29T15:42:00.711 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236777 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236625+0000", 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236625+0000", 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236627+0000", 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5519999999999999e-06 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236632+0000", 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3489999999999999e-06 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236643+0000", 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1117e-05 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236649+0000", 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8340000000000004e-06 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.249513+0000", 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012863556999999999 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.249647+0000", 2024-08-29T15:42:00.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013372500000000001 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2,4", 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.249786+0000", 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013996700000000001 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260509+0000", 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010722109000000001 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260550+0000", 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1396999999999997e-05 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260587+0000", 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6711000000000001e-05 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278680+0000", 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018093046000000002 2024-08-29T15:42:00.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278688+0000", 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2740000000000007e-06 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.308488+0000", 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.029800271 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.308570+0000", 2024-08-29T15:42:00.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1629000000000001e-05 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236783 8.f 8:f3550e7b:::benchmark_data_smithi027_306999_object110004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.237275+0000", 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.89911069499999, 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.071859028, 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.719 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236783 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237275+0000", 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237275+0000", 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237276+0000", 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2900000000000002e-07 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237280+0000", 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6160000000000004e-06 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237291+0000", 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0828e-05 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237298+0000", 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8360000000000003e-06 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260617+0000", 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023319096000000001 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260753+0000", 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013595 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2,4", 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260899+0000", 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014577500000000001 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.264873+0000", 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039747280000000003 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278798+0000", 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013924153 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278808+0000", 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9899999999999992e-06 2024-08-29T15:42:00.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309071+0000", 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0302634500000001 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309134+0000", 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2676999999999996e-05 2024-08-29T15:42:00.725 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236829 8.f 8:f7cd12c4:::benchmark_data_smithi027_306999_object110050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.316115+0000", 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.82026998399999, 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0129513210000001, 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.726 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236829 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316115+0000", 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316115+0000", 2024-08-29T15:42:00.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316117+0000", 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.19e-06 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316121+0000", 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6380000000000003e-06 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316134+0000", 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2989e-05 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316141+0000", 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7569999999999998e-06 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316213+0000", 2024-08-29T15:42:00.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1662999999999995e-05 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316222+0000", 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6209999999999999e-06 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309160+0000", 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99293816300000004 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309252+0000", 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1131000000000006e-05 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309357+0000", 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105801 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.316947+0000", 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075896039999999998 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.317335+0000", 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038852800000000002 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.328895+0000", 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011559908000000001 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329020+0000", 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124146 2024-08-29T15:42:00.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329067+0000", 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7182000000000003e-05 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236830 8.f 8:f2d34e4d:::benchmark_data_smithi027_306999_object110051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.319225+0000", 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.81716022699999, 2024-08-29T15:42:00.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0100289629999999, 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236830 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319225+0000", 2024-08-29T15:42:00.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319225+0000", 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319226+0000", 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319232+0000", 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1630000000000004e-06 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319245+0000", 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3664e-05 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319253+0000", 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2409999999999998e-06 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319324+0000", 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1743999999999996e-05 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319333+0000", 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3550000000000005e-06 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.312729+0000", 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99339648999999997 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.312869+0000", 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013976799999999999 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.312996+0000", 2024-08-29T15:42:00.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127391 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.317317+0000", 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043208580000000003 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.317341+0000", 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3623999999999999e-05 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329111+0000", 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011769998 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329211+0000", 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100616 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329254+0000", 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2735000000000003e-05 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.740 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236837 8.f 8:ff15e796:::benchmark_data_smithi027_306999_object110058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.325967+0000", 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.810418392, 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0078366379999999, 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.741 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236837 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325967+0000", 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325967+0000", 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325969+0000", 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.592e-06 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325973+0000", 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8600000000000001e-06 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325989+0000", 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5569e-05 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325996+0000", 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9399999999999996e-06 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.326261+0000", 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026476199999999998 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.326269+0000", 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8040000000000002e-06 2024-08-29T15:42:00.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.320687+0000", 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99441820299999995 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.320787+0000", 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010012499999999999 2024-08-29T15:42:00.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.320868+0000", 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1366000000000007e-05 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329434+0000", 2024-08-29T15:42:00.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085656870000000006 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329509+0000", 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5117000000000006e-05 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333707+0000", 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041977300000000002 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333757+0000", 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0417999999999997e-05 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333804+0000", 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6465000000000003e-05 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.748 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236843 8.f 8:f7533c45:::benchmark_data_smithi027_306999_object110064:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.330976+0000", 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.80540911099999, 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0029207099999999, 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236843 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.749 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330976+0000", 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330976+0000", 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330978+0000", 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0109999999999999e-06 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330984+0000", 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4609999999999999e-06 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330996+0000", 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2041e-05 2024-08-29T15:42:00.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.331003+0000", 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4889999999999998e-06 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.334063+0000", 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030598380000000001 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.334071+0000", 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8679999999999999e-06 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.324739+0000", 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99066757900000002 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.324856+0000", 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117119 2024-08-29T15:42:00.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.324977+0000", 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012087900000000001 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329465+0000", 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044884139999999996 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329550+0000", 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4844000000000003e-05 2024-08-29T15:42:00.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333812+0000", 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004262493 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333857+0000", 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4994000000000003e-05 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333897+0000", 2024-08-29T15:42:00.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.968e-05 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236872 8.f 8:f57e9354:::benchmark_data_smithi027_306999_object110093:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.368172+0000", 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.76821293200001, 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97553822099999998, 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.756 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236872 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368172+0000", 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368172+0000", 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368175+0000", 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2390000000000001e-06 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368180+0000", 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7950000000000004e-06 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368195+0000", 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4482e-05 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368203+0000", 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5210000000000002e-06 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368322+0000", 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011834599999999999 2024-08-29T15:42:00.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368330+0000", 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0620000000000001e-06 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329581+0000", 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96125158899999996 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329686+0000", 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010479 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329803+0000", 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116474 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333943+0000", 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041398049999999999 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333956+0000", 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3508e-05 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343595+0000", 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096386920000000008 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343671+0000", 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6726999999999999e-05 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343711+0000", 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9190999999999998e-05 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.763 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236891 8.f 8:fb7fc2c9:::benchmark_data_smithi027_306999_object110112:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.392116+0000", 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 192.74426914200001, 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97105291000000005, 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236891 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.764 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392116+0000", 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392116+0000", 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392118+0000", 2024-08-29T15:42:00.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.325e-06 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392123+0000", 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1580000000000004e-06 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392135+0000", 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2327000000000001e-05 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392143+0000", 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9000000000000006e-06 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392219+0000", 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.593e-05 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392229+0000", 2024-08-29T15:42:00.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9410000000000002e-06 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343974+0000", 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95174541999999995 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.344082+0000", 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107692 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.344155+0000", 2024-08-29T15:42:00.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2698000000000001e-05 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.352435+0000", 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082806140000000004 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.352452+0000", 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6252999999999998e-05 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.362986+0000", 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010534109999999999 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.363124+0000", 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138052 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.363169+0000", 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.549e-05 2024-08-29T15:42:00.770 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.771 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:00.771 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:00.771 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:42:00.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:00.773 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:00.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:00.773 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:00 smithi171 ceph-mon[90031]: pgmap v1575: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 28671/263631 objects degraded (10.875%); 12295/263631 objects misplaced (4.664%); 12 MiB/s, 191 objects/s recovering 2024-08-29T15:42:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:00 smithi027 ceph-mon[93921]: pgmap v1575: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 28671/263631 objects degraded (10.875%); 12295/263631 objects misplaced (4.664%); 12 MiB/s, 191 objects/s recovering 2024-08-29T15:42:00.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:00 smithi027 ceph-mon[98970]: pgmap v1575: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 28671/263631 objects degraded (10.875%); 12295/263631 objects misplaced (4.664%); 12 MiB/s, 191 objects/s recovering 2024-08-29T15:42:00.874 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:00.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:00.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:00.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:01.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:01.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:01.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:01.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:01.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:01.203 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:01.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:01.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:01.279 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:42:01.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:01.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:01.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:01.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:01.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:01.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:01.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:01.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:01.637 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:01.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:01.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:01.739 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:01.789 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:01.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:01.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:01.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:01.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:01.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:01.985 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:02.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:02.052 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:02.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:02.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:02.201 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:02.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:02.302 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:02.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:02.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:02.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:02.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:02.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:02.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:02.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:02.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:02.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:02.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:02.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:02.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:02 smithi171 ceph-mon[90031]: pgmap v1576: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 28320/259482 objects degraded (10.914%); 12119/259482 objects misplaced (4.670%); 11 MiB/s, 173 objects/s recovering 2024-08-29T15:42:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28320/259482 objects degraded (10.914%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:42:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:02 smithi027 ceph-mon[93921]: pgmap v1576: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 28320/259482 objects degraded (10.914%); 12119/259482 objects misplaced (4.670%); 11 MiB/s, 173 objects/s recovering 2024-08-29T15:42:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28320/259482 objects degraded (10.914%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:42:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:02 smithi027 ceph-mon[98970]: pgmap v1576: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 28320/259482 objects degraded (10.914%); 12119/259482 objects misplaced (4.670%); 11 MiB/s, 173 objects/s recovering 2024-08-29T15:42:02.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28320/259482 objects degraded (10.914%), 3 pgs degraded (PG_DEGRADED) 2024-08-29T15:42:02.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:02.899 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:02.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:02.993 INFO:teuthology.orchestra.run.smithi027.stdout:moved 37 / 297 (12.4579%) 2024-08-29T15:42:02.993 INFO:teuthology.orchestra.run.smithi027.stdout:avg 37.125 2024-08-29T15:42:02.993 INFO:teuthology.orchestra.run.smithi027.stdout:stddev 1.45237 -> 7.67199 (expected baseline 5.69951) 2024-08-29T15:42:02.993 INFO:teuthology.orchestra.run.smithi027.stdout:min osd.2 with 35 -> 39 pgs (0.942761 -> 1.05051 * mean) 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:max osd.4 with 39 -> 45 pgs (1.05051 -> 1.21212 * mean) 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:oload 110 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:max_change 3 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:max_change_osds 4 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:average_utilization 0.0300 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:overload_utilization 0.0330 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 weight 0.8500 -> 0.5909 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 weight 0.8500 -> 0.6034 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 weight 0.8500 -> 0.6431 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 weight 0.8500 -> 0.6658 2024-08-29T15:42:02.994 INFO:teuthology.orchestra.run.smithi027.stderr:no change 2024-08-29T15:42:03.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:03.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:03.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:03.127 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:42:02 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:42:02.991+0000 7f7dea97a640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-08-29T15:42:03.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:03.166 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:03.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:03.303 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:03.404 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:03.438 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:03.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:03.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:42:03.654 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:42:03.655 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:42:03.655 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:42:03.655 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:42:03.655 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:42:03.655 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:42:03.655 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:42:03.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:03.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:03.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:03.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:03 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1285008520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:03.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:03.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:03 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1285008520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:03.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:03 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1285008520' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:03.840 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:03.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:03.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:03.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:03.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:03.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:04.076 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:04.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:04.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:04.215 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:04.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:04.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:04.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:04.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:04.392 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:04.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:04.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:04.494 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:04.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:04.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:04.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:04.629 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:04.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:04.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:04.730 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:42:04.730 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:42:04.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:04.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:04.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:04.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:04 smithi027 ceph-mon[98970]: pgmap v1577: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 28034/255321 objects degraded (10.980%); 11772/255321 objects misplaced (4.611%); 12 MiB/s, 185 objects/s recovering 2024-08-29T15:42:04.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:04 smithi027 ceph-mon[98970]: from='client.56557 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:42:04.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:04 smithi027 ceph-mon[93921]: pgmap v1577: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 28034/255321 objects degraded (10.980%); 11772/255321 objects misplaced (4.611%); 12 MiB/s, 185 objects/s recovering 2024-08-29T15:42:04.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:04 smithi027 ceph-mon[93921]: from='client.56557 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:42:04.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:04.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:04.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:04.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:04.967 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:04 smithi171 ceph-mon[90031]: pgmap v1577: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 28034/255321 objects degraded (10.980%); 11772/255321 objects misplaced (4.611%); 12 MiB/s, 185 objects/s recovering 2024-08-29T15:42:04.967 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:04 smithi171 ceph-mon[90031]: from='client.56557 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-29T15:42:05.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:05.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:05.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:05.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:05.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:05.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:05.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:05.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:05.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:05.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:05.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:05.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:05.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:05.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:05.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:05.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:05.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:05.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:05.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:05.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:05.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:05.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:05.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:05.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:05.937 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:05.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:06.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:06.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:06.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:06.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:06.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:06.217 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:06.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:06.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:06.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:06.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:06.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:06.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:06.462 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:06.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:06.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:06.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:06.597 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:06.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:06.633 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:06.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:06.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:06.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:06.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:06 smithi171 ceph-mon[90031]: pgmap v1578: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 28034/254862 objects degraded (11.000%); 11772/254862 objects misplaced (4.619%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:42:06.794 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:06 smithi027 ceph-mon[93921]: pgmap v1578: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 28034/254862 objects degraded (11.000%); 11772/254862 objects misplaced (4.619%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:42:06.794 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:06 smithi027 ceph-mon[98970]: pgmap v1578: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 28034/254862 objects degraded (11.000%); 11772/254862 objects misplaced (4.619%); 8.8 MiB/s, 140 objects/s recovering 2024-08-29T15:42:06.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:06.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:06.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:06.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:07.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:07.041 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:07.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:07.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:07.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:07.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:07.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:07.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:07.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:07.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:07.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:07.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:07.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:07.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:07.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:42:07.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:07.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:07.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:07.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:07.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:42:07.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:07.776 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:07.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:07.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:07.901 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:07.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:07.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:07.946 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:07.946 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:07 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:07.946 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:07 smithi171 ceph-mon[90031]: pgmap v1579: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 27709/249903 objects degraded (11.088%); 11611/249903 objects misplaced (4.646%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:42:07.946 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 28034/254862 objects degraded (11.000%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:07.946 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/524958851' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:07.947 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:07 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:07.977 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:42:08.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:08.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:08.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:08.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:08.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:08.206 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:08.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:08.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:08.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:08.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[93921]: pgmap v1579: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 27709/249903 objects degraded (11.088%); 11611/249903 objects misplaced (4.646%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 28034/254862 objects degraded (11.000%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/524958851' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[98970]: pgmap v1579: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 27709/249903 objects degraded (11.088%); 11611/249903 objects misplaced (4.646%); 8.9 MiB/s, 142 objects/s recovering 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 28034/254862 objects degraded (11.000%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/524958851' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:08.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:07 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:08.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:08.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:08.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:08.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:08.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:08.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:08.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:08.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:08.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:08.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:08.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:08.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:08.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:08.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:08.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:08.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:08.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:09.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:09.123 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:09.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:09.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:08 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:42:09.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:08 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:42:09.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:08 smithi027 ceph-mon[93921]: osdmap e448: 8 total, 8 up, 8 in 2024-08-29T15:42:09.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:08 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:42:09.206 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:08 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:42:09.206 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:08 smithi027 ceph-mon[98970]: osdmap e448: 8 total, 8 up, 8 in 2024-08-29T15:42:09.206 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:42:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:42:09] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-08-29T15:42:09.207 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:08 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:42:09.207 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:08 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:42:09.207 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:08 smithi171 ceph-mon[90031]: osdmap e448: 8 total, 8 up, 8 in 2024-08-29T15:42:09.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:09.287 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:09.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:09.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:09.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:09.530 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:09.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:09.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:09.669 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:09.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:09.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:09.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:09.854 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:09.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:09.960 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:09 smithi027 ceph-mon[93921]: pgmap v1581: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 27379/247518 objects degraded (11.061%); 11190/247518 objects misplaced (4.521%); 11 MiB/s, 175 objects/s recovering 2024-08-29T15:42:09.960 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:09 smithi027 ceph-mon[98970]: pgmap v1581: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 27379/247518 objects degraded (11.061%); 11190/247518 objects misplaced (4.521%); 11 MiB/s, 175 objects/s recovering 2024-08-29T15:42:09.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:09.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:09.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:10.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:10.082 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:42:10.082 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:42:10.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:10.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:10.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:10.207 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:10.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:10.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:10.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:10.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:09 smithi171 ceph-mon[90031]: pgmap v1581: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 27379/247518 objects degraded (11.061%); 11190/247518 objects misplaced (4.521%); 11 MiB/s, 175 objects/s recovering 2024-08-29T15:42:10.346 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:10.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:10.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:10.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:10.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:10.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:10.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:10.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:10.630 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:10.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:10.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:10.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:10.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:10.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:10.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:10.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:10.937 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:10.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:10.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:11.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:11.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:11.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:11.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:11.076 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:42:11.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:11.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:11.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:11.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:11.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:11.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:11.364 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:11.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:11.488 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:11.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:11.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:11.590 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:11.634 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:11.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:11.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:11.695 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:42:11.695 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:42:11.695 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-29T15:42:11.696 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2024-08-29T15:42:11.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 3 1 2024-08-29T15:42:11.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:42:11.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:11.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:11.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:11.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:11.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:11.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:11.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:11.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:12.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:12.072 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:12.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:12.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:12.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:12.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:12.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:12.287 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:12 smithi027 ceph-mon[93921]: pgmap v1582: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 27079/243633 objects degraded (11.115%); 11043/243633 objects misplaced (4.533%); 11 MiB/s, 169 objects/s recovering 2024-08-29T15:42:12.287 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3886374356' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:12.288 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:12 smithi027 ceph-mon[98970]: pgmap v1582: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 27079/243633 objects degraded (11.115%); 11043/243633 objects misplaced (4.533%); 11 MiB/s, 169 objects/s recovering 2024-08-29T15:42:12.288 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3886374356' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:12.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:12.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:12.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:12.362 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:42:12.362 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:42:12.362 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:42:12.362 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:42:12.362 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:42:12.363 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:42:12.364 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:42:12.364 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:42:12.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:12.484 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:12.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:12 smithi171 ceph-mon[90031]: pgmap v1582: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 27079/243633 objects degraded (11.115%); 11043/243633 objects misplaced (4.533%); 11 MiB/s, 169 objects/s recovering 2024-08-29T15:42:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3886374356' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:12.585 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:12.620 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:12.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:12.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:12.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:12.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:12.853 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:12.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:12.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:12.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:12.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:13.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:13.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:13.118 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:13.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:13.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:13.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:13.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:13 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 27079/243633 objects degraded (11.115%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:13.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:13.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:13 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 27079/243633 objects degraded (11.115%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:13.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:13 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 27079/243633 objects degraded (11.115%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:13.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:13.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:13.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:13.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:13.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:13.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:13.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:13.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:13.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:13.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:13.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:13.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:13.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:14.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:14.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:14.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:14.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:14.202 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:14.207 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:14.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:14.271 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:14.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:14.333 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:14 smithi171 ceph-mon[90031]: pgmap v1583: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 26757/239679 objects degraded (11.164%); 10533/239679 objects misplaced (4.395%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:14.333 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:14.333 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:185378 8.b 8:dcdc8026:::benchmark_data_smithi027_306999_object58599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:36.371763+0000", 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 277.98949087599999, 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8660286230000001, 2024-08-29T15:42:14.365 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 185378 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.371763+0000", 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.371763+0000", 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.371761+0000", 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.371768+0000", 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.494e-06 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.371779+0000", 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1396000000000001e-05 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.371787+0000", 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8960000000000003e-06 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.378877+0000", 2024-08-29T15:42:14.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070899709999999996 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.378888+0000", 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1222e-05 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:37.089377+0000", 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.71048891400000003 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:37.089393+0000", 2024-08-29T15:42:14.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5447999999999998e-05 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.220447+0000", 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1310545830000001 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.220558+0000", 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000110241 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2,7", 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.221082+0000", 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00052427099999999998 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.229292+0000", 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.008209994 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.229338+0000", 2024-08-29T15:42:14.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5688e-05 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.237581+0000", 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0082431710000000005 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.237671+0000", 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9758000000000004e-05 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.237723+0000", 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2258e-05 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.237792+0000", 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8860000000000001e-05 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:185403 8.b 8:da1f48d7:::benchmark_data_smithi027_306999_object58624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:42:14.373 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:36.421340+0000", 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 277.93991429200003, 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.816668342, 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 185403 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421340+0000", 2024-08-29T15:42:14.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421340+0000", 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421338+0000", 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421344+0000", 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.446e-06 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421354+0000", 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8949999999999993e-06 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421359+0000", 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2979999999999999e-06 2024-08-29T15:42:14.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421416+0000", 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6449000000000001e-05 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.421427+0000", 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.115e-05 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:37.089447+0000", 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.66802070899999999 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:37.089455+0000", 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9379999999999992e-06 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.225726+0000", 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.13627057 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.225866+0000", 2024-08-29T15:42:14.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013987 2024-08-29T15:42:14.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2,7", 2024-08-29T15:42:14.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.226006+0000", 2024-08-29T15:42:14.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:14.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014044700000000001 2024-08-29T15:42:14.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.229389+0000", 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033831629999999998 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.229592+0000", 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00020281100000000001 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.237635+0000", 2024-08-29T15:42:14.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0080431719999999995 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.237833+0000", 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000197701 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.237882+0000", 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9230000000000001e-05 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.238008+0000", 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012561399999999999 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:250200 8.b 8:dc713462:::benchmark_data_smithi027_306999_object123421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:39:06.758552+0000", 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.60270203900001, 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.138066179, 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:42:14.383 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 250200 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758552+0000", 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758552+0000", 2024-08-29T15:42:14.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758551+0000", 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758556+0000", 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1710000000000001e-06 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758569+0000", 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3037000000000001e-05 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758576+0000", 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8820000000000003e-06 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758629+0000", 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3054999999999999e-05 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758759+0000", 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012949899999999999 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2,7", 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.758877+0000", 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000118243 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.766443+0000", 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075660629999999996 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.766451+0000", 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1659999999999994e-06 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.772382+0000", 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0059312119999999999 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.775749+0000", 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033666049999999999 2024-08-29T15:42:14.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.775762+0000", 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2969000000000001e-05 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:07.896547+0000", 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1207852300000001 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:07.896618+0000", 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0915999999999997e-05 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:250201 8.b 8:d4152181:::benchmark_data_smithi027_306999_object123422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e401)", 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:39:06.761692+0000", 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.59956235300001, 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1352913179999999, 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.390 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 250201 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.761692+0000", 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.761692+0000", 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.761690+0000", 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.761696+0000", 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4419999999999997e-06 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.761715+0000", 2024-08-29T15:42:14.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9545999999999999e-05 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.761722+0000", 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7920000000000004e-06 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.762608+0000", 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00088610200000000005 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.762723+0000", 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011459300000000001 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,2,7", 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.762835+0000", 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011237499999999999 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.766478+0000", 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036429959999999999 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.766716+0000", 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00023774700000000001 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.775791+0000", 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0090756550000000002 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:06.775806+0000", 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4175999999999999e-05 2024-08-29T15:42:14.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:07.896935+0000", 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.121129431 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:39:07.896983+0000", 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7772000000000003e-05 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.396 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:100518 9.d e444/435 9:b314caf6:::benchmark_data_smithi027_341149_object100517:head v 444'6316, mlcod=430'1982)", 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:25.359401+0000", 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 49.001852833000001, 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033145372999999999, 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.359401+0000", 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.359401+0000", 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.359400+0000", 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.359486+0000", 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5447999999999999e-05 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.359499+0000", 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2639999999999999e-05 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.359505+0000", 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2580000000000001e-06 2024-08-29T15:42:14.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.362364+0000", 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028590199999999999 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.362392+0000", 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8055000000000002e-05 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.392498+0000", 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030105726999999999 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:25.392546+0000", 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8846e-05 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:102283 9.d e444/435 9:bf579404:::benchmark_data_smithi027_341149_object102282:head v 444'6435, mlcod=430'2076)", 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:27.703005+0000", 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 46.658249060000003, 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036621616000000003, 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703005+0000", 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703005+0000", 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703004+0000", 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703031+0000", 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7047999999999999e-05 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703051+0000", 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0292000000000002e-05 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703060+0000", 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5169999999999999e-06 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703136+0000", 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6320000000000001e-05 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.703165+0000", 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8731999999999999e-05 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.739594+0000", 2024-08-29T15:42:14.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036429050999999997 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.739626+0000", 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2765999999999998e-05 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:102297 9.d e444/435 9:ba0a9520:::benchmark_data_smithi027_341149_object102296:head v 444'6437, mlcod=430'2076)", 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:27.710823+0000", 2024-08-29T15:42:14.406 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 46.650430427000003, 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039749620999999999, 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.710823+0000", 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.710823+0000", 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.710822+0000", 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.710846+0000", 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3648000000000001e-05 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.710864+0000", 2024-08-29T15:42:14.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8538000000000001e-05 2024-08-29T15:42:14.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.710874+0000", 2024-08-29T15:42:14.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.798e-06 2024-08-29T15:42:14.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.716451+0000", 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0055767450000000001 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.716478+0000", 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7291999999999999e-05 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.750538+0000", 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034060048000000002 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.750573+0000", 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4857999999999997e-05 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:102300 9.d e444/435 9:bfaeee15:::benchmark_data_smithi027_341149_object102299:head v 444'6438, mlcod=430'2076)", 2024-08-29T15:42:14.411 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:27.714725+0000", 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 46.646528433, 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035871683000000001, 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.714725+0000", 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.714725+0000", 2024-08-29T15:42:14.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.714724+0000", 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.714757+0000", 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3577000000000001e-05 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.413 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.714781+0000", 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3198999999999999e-05 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.714791+0000", 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0583e-05 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.723288+0000", 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0084971200000000004 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.723317+0000", 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8317e-05 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.750575+0000", 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027258879 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:27.750597+0000", 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1637e-05 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.415 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:102963 9.e 9:7e72a073:::benchmark_data_smithi027_341149_object533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:29.768803+0000", 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 44.592450661999997, 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098211375000000004, 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.416 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 102963 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.768803+0000", 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.768803+0000", 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.768801+0000", 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.768814+0000", 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3284e-05 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.768836+0000", 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1929999999999998e-05 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.418 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.768845+0000", 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6540000000000005e-06 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.768906+0000", 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1184000000000002e-05 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.769063+0000", 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015720400000000001 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.419 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.769190+0000", 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012645899999999999 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.772946+0000", 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037564040000000001 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.773385+0000", 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000438632 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.420 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.866920+0000", 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093535400000000005 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.866971+0000", 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0946999999999999e-05 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:29.867015+0000", 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3390000000000003e-05 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.421 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:105774 9.e 9:775aad82:::benchmark_data_smithi027_341149_object3344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:34.491575+0000", 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 39.869679063, 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.037516938999999999, 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 105774 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.422 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491575+0000", 2024-08-29T15:42:14.423 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.423 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.423 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.423 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.423 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491575+0000", 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491573+0000", 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.425 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491583+0000", 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7450000000000007e-06 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491603+0000", 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0613999999999999e-05 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491611+0000", 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7680000000000008e-06 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.426 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491668+0000", 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7209000000000001e-05 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491765+0000", 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6286999999999995e-05 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491848+0000", 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3166999999999998e-05 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.427 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.497299+0000", 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0054509279999999998 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.497324+0000", 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5516000000000001e-05 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.528952+0000", 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031627685000000003 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.428 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.529052+0000", 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9893000000000001e-05 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.529092+0000", 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0055000000000002e-05 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.429 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:105775 9.e 9:745d3c26:::benchmark_data_smithi027_341149_object3345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:34.491915+0000", 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 39.869338911, 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038637868999999998, 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 105775 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491915+0000", 2024-08-29T15:42:14.430 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491915+0000", 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491914+0000", 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.431 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491919+0000", 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2849999999999996e-06 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491926+0000", 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6120000000000001e-06 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.491931+0000", 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0470000000000001e-06 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.432 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.494529+0000", 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025975439999999998 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.494651+0000", 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000122378 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.494744+0000", 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2509999999999993e-05 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.433 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.497463+0000", 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027191170000000001 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.497607+0000", 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014427799999999999 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.434 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.530446+0000", 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032839067 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.530518+0000", 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2015999999999996e-05 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:34.530553+0000", 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4721000000000002e-05 2024-08-29T15:42:14.435 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:116374 9.e 9:764df65b:::benchmark_data_smithi027_341149_object13944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e447)", 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:54.832917+0000", 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 19.528337316999998, 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.068666780999999996, 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 116374 2024-08-29T15:42:14.436 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.832917+0000", 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.832917+0000", 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.437 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.832915+0000", 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.832924+0000", 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3270000000000007e-06 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.832941+0000", 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6773999999999999e-05 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.438 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.832949+0000", 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1899999999999995e-06 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.833003+0000", 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3928999999999998e-05 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.833100+0000", 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7313999999999997e-05 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.439 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.833187+0000", 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6717000000000003e-05 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.837166+0000", 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003978812 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.838246+0000", 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001080244 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.440 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.901491+0000", 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.063244773000000004 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.901540+0000", 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8724999999999999e-05 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:54.901583+0000", 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3791999999999997e-05 2024-08-29T15:42:14.441 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:118516 9.e 9:707612d7:::benchmark_data_smithi027_341149_object16086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e447)", 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:58.717556+0000", 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.643697977, 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033774825000000001, 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.442 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 118516 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717556+0000", 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717556+0000", 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.443 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717554+0000", 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717563+0000", 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1749999999999994e-06 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717578+0000", 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5364000000000001e-05 2024-08-29T15:42:14.444 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717585+0000", 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6610000000000003e-06 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717638+0000", 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2930000000000003e-05 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717789+0000", 2024-08-29T15:42:14.445 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015063399999999999 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.717930+0000", 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014110500000000001 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.722031+0000", 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041014090000000003 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.446 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.723251+0000", 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001220194 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.751245+0000", 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027993207999999999 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.751296+0000", 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0974999999999999e-05 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.751331+0000", 2024-08-29T15:42:14.447 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5169000000000002e-05 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:118600 9.c e447/420 9:3e05b2c4:::benchmark_data_smithi027_341149_object16170:head v 447'7593, mlcod=447'7591)", 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:58.864648+0000", 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.496605606999999, 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041585807000000002, 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864648+0000", 2024-08-29T15:42:14.448 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864648+0000", 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864648+0000", 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864655+0000", 2024-08-29T15:42:14.449 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3699999999999997e-06 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864663+0000", 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4279999999999996e-06 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864668+0000", 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4120000000000003e-06 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864717+0000", 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9089999999999999e-05 2024-08-29T15:42:14.450 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.864738+0000", 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1287000000000001e-05 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.906186+0000", 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041447997 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.906234+0000", 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7988999999999997e-05 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.451 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:118604 9.c e447/420 9:397d9345:::benchmark_data_smithi027_341149_object16174:head v 447'7594, mlcod=447'7591)", 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:58.874161+0000", 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.48709264, 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035878925999999998, 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.874161+0000", 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.452 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.874161+0000", 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.874160+0000", 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.874170+0000", 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0027000000000001e-05 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.453 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.899340+0000", 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025169596999999998 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.899355+0000", 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5299e-05 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.899788+0000", 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000432779 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.454 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.899808+0000", 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0083000000000001e-05 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.909854+0000", 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010046332999999999 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:58.910040+0000", 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018615600000000001 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.455 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "MOSDPGPush(9.d 447/435 [PushOp(9:b8c46091:::benchmark_data_smithi027_341149_object69528:head, version: 431'4349, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:b8c46091:::benchmark_data_smithi027_341149_object69528:head@431'4349, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:59.331563+0000", 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.029691346, 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085994403999999997, 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.456 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331563+0000", 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331563+0000", 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331562+0000", 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.457 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331597+0000", 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5437e-05 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331615+0000", 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7807000000000001e-05 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331629+0000", 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3593e-05 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.458 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331696+0000", 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7209e-05 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331713+0000", 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7022000000000001e-05 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417557+0000", 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085844119999999996 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.459 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:118844 9.e 9:7c0d10f9:::benchmark_data_smithi027_341149_object16414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e447)", 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:59.331834+0000", 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.029419814000001, 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.097179557999999999, 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 118844 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.460 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331834+0000", 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331834+0000", 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331833+0000", 2024-08-29T15:42:14.461 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331838+0000", 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7589999999999997e-06 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331846+0000", 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3450000000000006e-06 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331853+0000", 2024-08-29T15:42:14.462 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.754e-06 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331884+0000", 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1529999999999998e-05 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.331968+0000", 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3131000000000001e-05 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417690+0000", 2024-08-29T15:42:14.463 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.085721980000000003 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.425547+0000", 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078575620000000002 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.425557+0000", 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.859e-06 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.464 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.428924+0000", 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033669450000000001 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.428973+0000", 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9304000000000001e-05 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.429014+0000", 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0349999999999998e-05 2024-08-29T15:42:14.465 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:118848 9.e 9:7581e3fd:::benchmark_data_smithi027_341149_object16418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e447)", 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:59.345420+0000", 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 15.015833800999999, 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.083761642999999997, 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 118848 2024-08-29T15:42:14.466 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.345420+0000", 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.345420+0000", 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.467 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.345418+0000", 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.345428+0000", 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3829999999999998e-06 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417565+0000", 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.072136932000000001 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.468 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417578+0000", 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.273e-05 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.421491+0000", 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039133630000000004 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.421679+0000", 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000187858 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.469 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.421838+0000", 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015913799999999999 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.425774+0000", 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039361700000000001 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.427873+0000", 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0020989860000000002 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.470 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.429118+0000", 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012447459999999999 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.429153+0000", 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5463000000000002e-05 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.429182+0000", 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8493999999999999e-05 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.471 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:118857 9.c e447/420 9:38f91847:::benchmark_data_smithi027_341149_object16427:head v 447'7611, mlcod=447'7609)", 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:59.378904+0000", 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 14.982350145, 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042684673999999999, 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.378904+0000", 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.472 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.378904+0000", 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.378902+0000", 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.378912+0000", 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7049999999999994e-06 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.473 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417563+0000", 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038650447999999997 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417575+0000", 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2342e-05 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417617+0000", 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1912000000000001e-05 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.474 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.417638+0000", 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0798e-05 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.421454+0000", 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038166089999999999 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:59.421588+0000", 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013419300000000001 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.475 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:126354 9.e 9:74c83925:::benchmark_data_smithi027_341149_object23924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:13.610625+0000", 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.75062879299999996, 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034787804999999998, 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:42:14.476 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 126354 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610625+0000", 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610625+0000", 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:14.477 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610623+0000", 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610630+0000", 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0589999999999997e-06 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:14.478 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610645+0000", 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4107999999999999e-05 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610653+0000", 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3450000000000006e-06 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610693+0000", 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9860999999999997e-05 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:14.479 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610799+0000", 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010663299999999999 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.610879+0000", 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9896999999999998e-05 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.614380+0000", 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00350118 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.614838+0000", 2024-08-29T15:42:14.480 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00045716700000000002 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.645353+0000", 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030515845 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.645388+0000", 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4066999999999998e-05 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:13.645413+0000", 2024-08-29T15:42:14.481 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5337000000000002e-05 2024-08-29T15:42:14.482 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.482 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.482 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.482 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:14.482 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:14.482 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:42:14.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:14.482 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:14.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:14 smithi027 ceph-mon[98970]: pgmap v1583: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 26757/239679 objects degraded (11.164%); 10533/239679 objects misplaced (4.395%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:14 smithi027 ceph-mon[93921]: pgmap v1583: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 26757/239679 objects degraded (11.164%); 10533/239679 objects misplaced (4.395%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:14.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:14.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:14.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:14.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:14.752 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:14.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:14.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:14.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:14.889 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:14.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:14.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:15.022 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:15.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:15.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:15.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:15.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:15.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:15.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:42:15.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:15.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:15.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:15.556 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:15.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:15.658 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:15.672 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:15.703 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:15.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:15.839 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:15.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:15.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:15.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:16.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:16.109 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:16.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:16.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:16.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:16.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:16.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:16.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:16.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:16.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:16.472 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:16.496 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:16 smithi027 ceph-mon[98970]: pgmap v1584: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 26757/239169 objects degraded (11.187%); 10533/239169 objects misplaced (4.404%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:16.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:16 smithi027 ceph-mon[93921]: pgmap v1584: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 26757/239169 objects degraded (11.187%); 10533/239169 objects misplaced (4.404%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:16.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:16.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:16 smithi171 ceph-mon[90031]: pgmap v1584: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 26757/239169 objects degraded (11.187%); 10533/239169 objects misplaced (4.404%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:16.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:16.634 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:16.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:16.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:16.766 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:16.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:16.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:16.809 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:16.867 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:16.893 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:16.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:16.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:16.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:17.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:17.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:17.123 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:17.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:17.202 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 26757/239169 objects degraded (11.187%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:17.202 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 26757/239169 objects degraded (11.187%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:17.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:17.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 26757/239169 objects degraded (11.187%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:17.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:17.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:17.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.440 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:17.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:17.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:17.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:17.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:17.709 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:17.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:17.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:17.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:17.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:17.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:17.967 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:42:17.968 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:42:17.968 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:42:17.968 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:42:17.994 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:17.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:18.096 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:18.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:18.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:18.228 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:18.264 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:18.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:18.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:18.366 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:18.401 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:18.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:18.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:18 smithi171 ceph-mon[90031]: pgmap v1585: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 26415/234240 objects degraded (11.277%); 10364/234240 objects misplaced (4.425%); 12 MiB/s, 189 objects/s recovering 2024-08-29T15:42:18.529 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:18 smithi027 ceph-mon[98970]: pgmap v1585: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 26415/234240 objects degraded (11.277%); 10364/234240 objects misplaced (4.425%); 12 MiB/s, 189 objects/s recovering 2024-08-29T15:42:18.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:18 smithi027 ceph-mon[93921]: pgmap v1585: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 26415/234240 objects degraded (11.277%); 10364/234240 objects misplaced (4.425%); 12 MiB/s, 189 objects/s recovering 2024-08-29T15:42:18.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:18.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:18.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:18.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:18.663 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:18.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:18.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:18.765 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:18.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:18.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:18.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:18.891 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:18.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:18.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:19.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:19.059 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:19.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:19.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:19.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:19.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:19.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:19.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:42:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:42:19] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.51.0" 2024-08-29T15:42:19.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:19.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:19.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:42:19.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:19.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:19.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:19.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:19.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:19.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:19.698 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:19.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:19.822 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:19.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:42:19.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:19.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:19.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:19.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:20.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:20.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:20.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:20.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:20.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:20.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:20.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:20.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:20.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:20.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:20.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:20.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:20.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:20.481 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:20.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:20 smithi171 ceph-mon[90031]: pgmap v1586: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 26109/231681 objects degraded (11.269%); 9754/231681 objects misplaced (4.210%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:20 smithi171 ceph-mon[90031]: osdmap e449: 8 total, 8 up, 8 in 2024-08-29T15:42:20.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:20 smithi027 ceph-mon[98970]: pgmap v1586: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 26109/231681 objects degraded (11.269%); 9754/231681 objects misplaced (4.210%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:20.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:20 smithi027 ceph-mon[98970]: osdmap e449: 8 total, 8 up, 8 in 2024-08-29T15:42:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:20 smithi027 ceph-mon[93921]: pgmap v1586: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 26109/231681 objects degraded (11.269%); 9754/231681 objects misplaced (4.210%); 12 MiB/s, 187 objects/s recovering 2024-08-29T15:42:20.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:20 smithi027 ceph-mon[93921]: osdmap e449: 8 total, 8 up, 8 in 2024-08-29T15:42:20.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:20.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:20.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:20.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:20.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:20.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:20.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:20.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:20.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:20.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:21.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:21.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:21.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:21.166 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:21.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:21.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:21.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:21 smithi027 ceph-mon[93921]: osdmap e450: 8 total, 8 up, 8 in 2024-08-29T15:42:21.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:21 smithi027 ceph-mon[98970]: osdmap e450: 8 total, 8 up, 8 in 2024-08-29T15:42:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:21.319 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:21.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:21.413 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:21 smithi171 ceph-mon[90031]: osdmap e450: 8 total, 8 up, 8 in 2024-08-29T15:42:21.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:21.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:21.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:21.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:21.548 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:42:21.549 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:42:21.549 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:42:21.549 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:42:21.549 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:42:21.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:21.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:21.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:21.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:21.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:21.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:21.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:21.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:21.900 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:21.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:21.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:22.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:22.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:22.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:22.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:22.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:22.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:22.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:22.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:22 smithi027 ceph-mon[98970]: pgmap v1589: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 21273/227904 objects degraded (9.334%); 13982/227904 objects misplaced (6.135%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:42:22.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:42:22.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:22.333 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:22.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:22.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:22.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:22.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:22.504 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 21273/227904 objects degraded (9.334%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:22.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:22 smithi027 ceph-mon[93921]: pgmap v1589: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 21273/227904 objects degraded (9.334%); 13982/227904 objects misplaced (6.135%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:42:22.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 21273/227904 objects degraded (9.334%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:22 smithi171 ceph-mon[90031]: pgmap v1589: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 93 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 21273/227904 objects degraded (9.334%); 13982/227904 objects misplaced (6.135%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:42:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 21273/227904 objects degraded (9.334%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:22.559 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:22.573 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:22.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:22.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:22.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:22.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:22.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:22.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:22.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:22.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:22.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:23.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:23.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:23.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:23.175 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:23.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:23.277 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:23.326 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:23.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:23.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:23.452 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:23.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:23.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:23.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:23.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:23.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:23.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:23.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:23.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:23.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:23.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:23.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:23.985 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:24.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:24.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:24.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:24.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:24.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:24.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:24.305 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:24.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:24.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:24.439 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:24 smithi027 ceph-mon[93921]: pgmap v1590: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 20981/224571 objects degraded (9.343%); 13791/224571 objects misplaced (6.141%); 15 MiB/s, 235 objects/s recovering 2024-08-29T15:42:24.439 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:24 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:24.439 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:24 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:24.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:24 smithi027 ceph-mon[98970]: pgmap v1590: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 20981/224571 objects degraded (9.343%); 13791/224571 objects misplaced (6.141%); 15 MiB/s, 235 objects/s recovering 2024-08-29T15:42:24.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:24 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:24.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:24 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:24.474 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:24 smithi171 ceph-mon[90031]: pgmap v1590: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 20981/224571 objects degraded (9.343%); 13791/224571 objects misplaced (6.141%); 15 MiB/s, 235 objects/s recovering 2024-08-29T15:42:24.474 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:24 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:24.474 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:24 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:24.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:24.547 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:24.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:24.649 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:24.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:24.779 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:24.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:24.804 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:180121 8.6 e372/344 8:6f0f00f8:::benchmark_data_smithi027_306999_object53342:head v 372'11323, mlcod=372'11319)", 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:30.556359+0000", 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 293.91582051799998, 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049309257000000002, 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.805 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556359+0000", 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556359+0000", 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556358+0000", 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.806 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556363+0000", 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.395e-06 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556378+0000", 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4881e-05 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556385+0000", 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8499999999999996e-06 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.807 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556471+0000", 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5739e-05 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556489+0000", 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8725000000000001e-05 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605522+0000", 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049032294999999997 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.808 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605668+0000", 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014615599999999999 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:180122 8.6 e372/344 8:64fe0a2b:::benchmark_data_smithi027_306999_object53343:head v 372'11324, mlcod=372'11321)", 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 293.908846719, 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042496968000000003, 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.809 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.810 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563337+0000", 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4129999999999997e-06 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563345+0000", 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2910000000000002e-06 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563349+0000", 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6029999999999998e-06 2024-08-29T15:42:24.811 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.570758+0000", 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007409044 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.570777+0000", 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.842e-05 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.812 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605672+0000", 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034895109000000001 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605829+0000", 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015755599999999999 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:180138 8.6 e372/344 8:6a769c0c:::benchmark_data_smithi027_306999_object53359:head v 372'11325, mlcod=372'11321)", 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:30.564453+0000", 2024-08-29T15:42:24.813 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 293.90772626699999, 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041501364999999998, 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564453+0000", 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564453+0000", 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.814 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564452+0000", 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564458+0000", 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8590000000000001e-06 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564469+0000", 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0536e-05 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.815 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564475+0000", 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9830000000000001e-06 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.575560+0000", 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011084946 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.575605+0000", 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4928000000000001e-05 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.816 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605834+0000", 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030229096 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605954+0000", 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000120667 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.817 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:236681 8.6 e393/344 8:68c8102a:::benchmark_data_smithi027_306999_object109902:head v 393'14929, mlcod=393'14926)", 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:46.096960+0000", 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 218.37521905400001, 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036404883999999998, 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096960+0000", 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.818 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096960+0000", 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096959+0000", 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096971+0000", 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2201000000000001e-05 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096988+0000", 2024-08-29T15:42:24.819 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7407000000000001e-05 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096998+0000", 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0107e-05 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.097070+0000", 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1593000000000004e-05 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.097094+0000", 2024-08-29T15:42:24.820 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3966999999999999e-05 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.133154+0000", 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036059846 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.133365+0000", 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000211239 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.821 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:236689 8.6 e393/344 8:60741c53:::benchmark_data_smithi027_306999_object109910:head v 393'14931, mlcod=393'14926)", 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:46.101218+0000", 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 218.37096144700001, 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042136763000000001, 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101218+0000", 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101218+0000", 2024-08-29T15:42:24.822 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101217+0000", 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101225+0000", 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0909999999999993e-06 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101240+0000", 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5225000000000001e-05 2024-08-29T15:42:24.823 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101247+0000", 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4499999999999998e-06 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.114281+0000", 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013033916 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.114305+0000", 2024-08-29T15:42:24.824 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3921999999999999e-05 2024-08-29T15:42:24.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.825 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.825 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.825 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143195+0000", 2024-08-29T15:42:24.825 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028890078 2024-08-29T15:42:24.825 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.826 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.826 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.826 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143355+0000", 2024-08-29T15:42:24.826 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000159149 2024-08-29T15:42:24.826 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.826 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:236691 8.6 e393/344 8:672d18b6:::benchmark_data_smithi027_306999_object109912:head v 393'14932, mlcod=393'14926)", 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:46.104680+0000", 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 218.36749965300001, 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038806661999999999, 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.827 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104680+0000", 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104680+0000", 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104678+0000", 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.828 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104690+0000", 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1099e-05 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104709+0000", 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9902999999999999e-05 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104718+0000", 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5059999999999995e-06 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.829 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.121620+0000", 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016901596000000001 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.121667+0000", 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7104999999999997e-05 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143359+0000", 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021692052 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.830 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143486+0000", 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012745399999999999 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:75304 9.b e432/420 9:d35b9d25:::benchmark_data_smithi027_341149_object75303:head v 432'4759, mlcod=432'4755)", 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:40:38.926391+0000", 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 105.545788457, 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017142794999999999, 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.831 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926391+0000", 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926391+0000", 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926390+0000", 2024-08-29T15:42:24.832 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926430+0000", 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9685999999999999e-05 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926450+0000", 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0497e-05 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.833 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926458+0000", 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1799999999999996e-06 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926515+0000", 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6993000000000001e-05 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926536+0000", 2024-08-29T15:42:24.834 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0548e-05 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.942032+0000", 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015496535000000001 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.943534+0000", 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015011720000000001 2024-08-29T15:42:24.835 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:88608 9.b e432/420 9:d936d0b4:::benchmark_data_smithi027_341149_object88607:head v 432'5577, mlcod=432'5574)", 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:40:49.829521+0000", 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 94.642658355999998, 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014284933, 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.836 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.829521+0000", 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.829521+0000", 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.829520+0000", 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.837 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.829602+0000", 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1693000000000005e-05 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.829641+0000", 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9227999999999998e-05 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.829648+0000", 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5480000000000004e-06 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.838 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.832145+0000", 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0024968780000000001 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.832168+0000", 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3051e-05 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.843665+0000", 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011496515000000001 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.839 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:49.843806+0000", 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000140872 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:90597 9.1 e432/420 9:8fcb4ce9:::benchmark_data_smithi027_341149_object90596:head v 432'5659, mlcod=432'5656)", 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:40:51.469906+0000", 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 93.002273591000005, 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016407099000000001, 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.840 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.469906+0000", 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.469906+0000", 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.841 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.469905+0000", 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.470024+0000", 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000118823 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.470037+0000", 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3277e-05 2024-08-29T15:42:24.842 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.470044+0000", 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4839999999999998e-06 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.471459+0000", 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014141970000000001 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.843 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.471506+0000", 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7055999999999998e-05 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.486274+0000", 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01476889 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.486313+0000", 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8219000000000001e-05 2024-08-29T15:42:24.844 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:91087 9.b e432/420 9:dca8a3ae:::benchmark_data_smithi027_341149_object91086:head v 432'5740, mlcod=432'5737)", 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:40:51.929665+0000", 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 92.542513823999997, 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014391549, 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.845 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929665+0000", 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929665+0000", 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929665+0000", 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.846 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929790+0000", 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012555699999999999 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929806+0000", 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5766000000000001e-05 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929813+0000", 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.674e-06 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.847 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929861+0000", 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8364e-05 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.929878+0000", 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7431999999999998e-05 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.943952+0000", 2024-08-29T15:42:24.848 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014073353 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.944057+0000", 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010519 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:114831 9.1 e447/420 9:8308c35c:::benchmark_data_smithi027_341149_object12401:head v 447'7168, mlcod=447'7166)", 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:51.926509+0000", 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 32.545670585000003, 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033561088000000003, 2024-08-29T15:42:24.849 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926509+0000", 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926509+0000", 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.850 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926508+0000", 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926516+0000", 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8510000000000005e-06 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926530+0000", 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3522e-05 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926536+0000", 2024-08-29T15:42:24.851 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3199999999999996e-06 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926575+0000", 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9122999999999998e-05 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926592+0000", 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6526000000000001e-05 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.959986+0000", 2024-08-29T15:42:24.852 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033394090000000001 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.960070+0000", 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3647999999999996e-05 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:114832 9.0 e447/446 9:06227d88:::benchmark_data_smithi027_341149_object12402:head v 447'6974, mlcod=429'517)", 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:51.927353+0000", 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 32.544826407999999, 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028675895, 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.853 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927353+0000", 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927353+0000", 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927352+0000", 2024-08-29T15:42:24.854 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927359+0000", 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.199e-06 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927371+0000", 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1878e-05 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927377+0000", 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4710000000000004e-06 2024-08-29T15:42:24.855 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927426+0000", 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8463e-05 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927448+0000", 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2396e-05 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.856 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.955983+0000", 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028535151000000002 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956029+0000", 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5198999999999999e-05 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.857 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:114834 9.b e447/420 9:d707837b:::benchmark_data_smithi027_341149_object12404:head v 447'7158, mlcod=447'7156)", 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:51.931036+0000", 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 32.541142856999997, 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029067564000000001, 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931036+0000", 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.858 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931036+0000", 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931035+0000", 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931043+0000", 2024-08-29T15:42:24.859 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7700000000000001e-06 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956560+0000", 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025517387999999998 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956571+0000", 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0098000000000001e-05 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.860 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956615+0000", 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3946999999999998e-05 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956635+0000", 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0295000000000001e-05 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.960078+0000", 2024-08-29T15:42:24.861 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003442826 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.960104+0000", 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6239000000000001e-05 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119373 9.0 e447/446 9:00090137:::benchmark_data_smithi027_341149_object16943:head v 447'7240, mlcod=429'517)", 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.436935+0000", 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 24.035244090999999, 2024-08-29T15:42:24.862 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018171396999999999, 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436935+0000", 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436935+0000", 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.863 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436934+0000", 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436944+0000", 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9399999999999997e-06 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436964+0000", 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0052e-05 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.864 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436973+0000", 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6690000000000001e-06 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437141+0000", 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000168018 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437213+0000", 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1418e-05 2024-08-29T15:42:24.865 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455035+0000", 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017822489 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455106+0000", 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1141000000000002e-05 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.866 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119378 9.0 e447/446 9:052ba21d:::benchmark_data_smithi027_341149_object16948:head v 447'7241, mlcod=429'517)", 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.437597+0000", 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 24.034582328999999, 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017570830999999999, 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437597+0000", 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437597+0000", 2024-08-29T15:42:24.867 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437596+0000", 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437603+0000", 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5549999999999999e-06 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437616+0000", 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3407999999999999e-05 2024-08-29T15:42:24.868 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437622+0000", 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.02e-06 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.441222+0000", 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035998969999999999 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.869 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.441247+0000", 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5188e-05 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455110+0000", 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013862670000000001 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455168+0000", 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7708999999999999e-05 2024-08-29T15:42:24.870 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119456 9.b e447/420 9:d0fbc3a5:::benchmark_data_smithi027_341149_object17026:head v 447'7429, mlcod=447'7427)", 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.589097+0000", 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 23.883082046999998, 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015616016999999999, 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589097+0000", 2024-08-29T15:42:24.871 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589097+0000", 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589096+0000", 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589106+0000", 2024-08-29T15:42:24.872 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7240000000000004e-06 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589122+0000", 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6192e-05 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589129+0000", 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3860000000000001e-06 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589193+0000", 2024-08-29T15:42:24.873 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3207000000000007e-05 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589218+0000", 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5778999999999998e-05 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604639+0000", 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015420496000000001 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.874 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604713+0000", 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4313999999999994e-05 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119461 9.b e447/420 9:de9ccd52:::benchmark_data_smithi027_341149_object17031:head v 447'7430, mlcod=447'7427)", 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.589711+0000", 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 23.882468137, 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015031308, 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.875 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589711+0000", 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589711+0000", 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.876 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589710+0000", 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589717+0000", 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0269999999999998e-06 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589730+0000", 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2801e-05 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.877 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589737+0000", 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4810000000000003e-06 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.592962+0000", 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032251810000000001 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.593010+0000", 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7922e-05 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604722+0000", 2024-08-29T15:42:24.878 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011712482999999999 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604742+0000", 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0250000000000001e-05 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119673 9.b e447/420 9:d3d729fb:::benchmark_data_smithi027_341149_object17243:head v 447'7443, mlcod=447'7441)", 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.891670+0000", 2024-08-29T15:42:24.879 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 23.580509383999999, 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015279612999999999, 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891670+0000", 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891670+0000", 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.880 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891668+0000", 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891676+0000", 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6389999999999992e-06 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891692+0000", 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5679e-05 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.881 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891699+0000", 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2450000000000001e-06 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891756+0000", 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6913999999999998e-05 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891777+0000", 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1030999999999999e-05 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.882 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.906904+0000", 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015127108 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.906949+0000", 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5419e-05 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:126706 9.0 e448/446 9:042b3f23:::benchmark_data_smithi027_341149_object24276:head v 448'7671, mlcod=429'517)", 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:14.367025+0000", 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 10.105154434999999, 2024-08-29T15:42:24.883 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011792702, 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367025+0000", 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367025+0000", 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.884 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367023+0000", 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367035+0000", 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1493e-05 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367055+0000", 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9797999999999999e-05 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.885 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367063+0000", 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9130000000000003e-06 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367144+0000", 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1687999999999994e-05 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.367176+0000", 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1145000000000001e-05 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.886 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.378764+0000", 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011588259 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:14.378817+0000", 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3671000000000002e-05 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:127126 9.b e448/420 9:df1eb09b:::benchmark_data_smithi027_341149_object24696:head v 448'7926, mlcod=448'7923)", 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:15.134751+0000", 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 9.3374279829999995, 2024-08-29T15:42:24.887 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013041763, 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.134751+0000", 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.134751+0000", 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.888 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.134750+0000", 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.134758+0000", 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6809999999999998e-06 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.134775+0000", 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6918999999999998e-05 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.889 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.134783+0000", 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4749999999999993e-06 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.138109+0000", 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033252809999999998 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.138170+0000", 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1304999999999997e-05 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:42:24.890 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.147628+0000", 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0094577759999999993 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:15.147793+0000", 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016526600000000001 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:24.891 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:42:24.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:24.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:24.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:25.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:25.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:25.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:25.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:25.197 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:25.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:25.242 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-08-29T15:42:25.284 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1477074293' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1477074293' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:42:25.285 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:25.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:25.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:25.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:25.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:25.490 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:25.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1477074293' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:42:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:25 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:42:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:42:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:42:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:25.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:25.625 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:25.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:25.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:25.727 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:25.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:25.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:25.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:25.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:25.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:25.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:26.007 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:26.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:42:26.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:26.148 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:26.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:26.264 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:42:26.291 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:26.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:26.350 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:26.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:26.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:26.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:26.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:26 smithi171 ceph-mon[90031]: pgmap v1591: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 20981/224169 objects degraded (9.359%); 13791/224169 objects misplaced (6.152%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:42:26.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:26 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:42:26.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:26 smithi171 ceph-mon[90031]: osdmap e451: 8 total, 8 up, 8 in 2024-08-29T15:42:26.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2230448836' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:42:26.477 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:26 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:42:26.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:26.563 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:26.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[98970]: pgmap v1591: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 20981/224169 objects degraded (9.359%); 13791/224169 objects misplaced (6.152%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:42:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:42:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[98970]: osdmap e451: 8 total, 8 up, 8 in 2024-08-29T15:42:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2230448836' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:42:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:42:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[93921]: pgmap v1591: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 20981/224169 objects degraded (9.359%); 13791/224169 objects misplaced (6.152%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:42:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:42:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[93921]: osdmap e451: 8 total, 8 up, 8 in 2024-08-29T15:42:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2230448836' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:42:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:26 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:42:26.665 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:26.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:26.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:26.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:26.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:26.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:26.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:26.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:26.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:26.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:27.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:27.100 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:27.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:27.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:27.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:27.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:27.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:27.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:27.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:27.383 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:27.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:27.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:27.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:27.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:27.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:27 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:42:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:27 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:42:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:27 smithi171 ceph-mon[90031]: osdmap e452: 8 total, 8 up, 8 in 2024-08-29T15:42:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 20981/224169 objects degraded (9.359%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:42:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:42:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[98970]: osdmap e452: 8 total, 8 up, 8 in 2024-08-29T15:42:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 20981/224169 objects degraded (9.359%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:42:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:42:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[93921]: osdmap e452: 8 total, 8 up, 8 in 2024-08-29T15:42:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 20981/224169 objects degraded (9.359%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:27.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:27.651 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:27.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:27.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:27.752 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:27.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:27.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:27.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:27.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:27.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:28.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:28.045 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:28.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:28.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:133702 9.8 e452/435 9:118a239e:::benchmark_data_smithi027_341149_object31272:head v 452'8414, mlcod=430'1971)", 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:28.276350+0000", 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.002559838, 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002586372, 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:42:28.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276350+0000", 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276350+0000", 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:42:28.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276350+0000", 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276356+0000", 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3690000000000003e-06 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276368+0000", 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1711e-05 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276377+0000", 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1079999999999999e-06 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276417+0000", 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0031999999999999e-05 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:28.276433+0000", 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6569e-05 2024-08-29T15:42:28.284 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:28.285 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:42:28.285 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:28.285 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:42:28.285 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:42:28.285 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 1 2024-08-29T15:42:28.285 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:42:28.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:28.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:28.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:28.475 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:28.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:28 smithi171 ceph-mon[90031]: pgmap v1594: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20679/219513 objects degraded (9.420%); 13520/219513 objects misplaced (6.159%); 7.6 MiB/s, 121 objects/s recovering 2024-08-29T15:42:28.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:28 smithi027 ceph-mon[98970]: pgmap v1594: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20679/219513 objects degraded (9.420%); 13520/219513 objects misplaced (6.159%); 7.6 MiB/s, 121 objects/s recovering 2024-08-29T15:42:28.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:28 smithi027 ceph-mon[93921]: pgmap v1594: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20679/219513 objects degraded (9.420%); 13520/219513 objects misplaced (6.159%); 7.6 MiB/s, 121 objects/s recovering 2024-08-29T15:42:28.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:28.608 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:28.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:28.709 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:28.735 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:28.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:28.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:28.861 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:28.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:28.986 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:28.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:28.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:29.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:42:29.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:29.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:29.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:29.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:29.265 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:42:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:42:29] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-29T15:42:29.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:29.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:29.333 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:29.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:29.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:29.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:29.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:29.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:29.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:29.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:29.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:29.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:29.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:29.788 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:29.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:29.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:29.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:29.948 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:29.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:29.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:29.950 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:29.950 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:29.950 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:29 smithi027 ceph-mon[98970]: pgmap v1595: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 20343/217002 objects degraded (9.375%); 13520/217002 objects misplaced (6.230%); 7.2 MiB/s, 115 objects/s recovering 2024-08-29T15:42:29.950 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:29.950 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:29.950 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:29 smithi027 ceph-mon[93921]: pgmap v1595: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 20343/217002 objects degraded (9.375%); 13520/217002 objects misplaced (6.230%); 7.2 MiB/s, 115 objects/s recovering 2024-08-29T15:42:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:42:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:29 smithi171 ceph-mon[90031]: pgmap v1595: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 20343/217002 objects degraded (9.375%); 13520/217002 objects misplaced (6.230%); 7.2 MiB/s, 115 objects/s recovering 2024-08-29T15:42:30.035 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:42:30.035 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:42:30.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:30.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:30.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:30.074 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:42:30.074 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:42:30.109 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:42:30.109 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:42:30.140 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:42:30.141 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:42:30.173 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:42:30.173 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:42:30.174 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:30.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:30.214 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:42:30.214 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:42:30.254 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:42:30.254 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:42:30.290 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:42:30.290 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:42:30.301 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:30.325 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:30.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:30.459 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:30.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:30.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:30.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:30.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:30.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:30.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:30.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:30.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:30.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:31.015 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/377736408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/377736408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/377736408' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:31.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:31.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:31.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:31.217 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:31.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:31.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:31.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:42:31.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:31.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:31.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:31.429 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:42:31.429 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:42:31.430 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-08-29T15:42:31.430 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:42:31.430 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:42:31.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-08-29T15:42:31.471 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:31.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:31.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:31.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:31.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:31.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:31.719 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:42:31.719 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:42:31.719 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:31.752 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:31 smithi027 ceph-mon[93921]: pgmap v1596: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 20085/212871 objects degraded (9.435%); 13189/212871 objects misplaced (6.196%); 8.0 MiB/s, 127 objects/s recovering 2024-08-29T15:42:31.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:31.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:31.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:31.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:31.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:32.009 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:32.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:31 smithi171 ceph-mon[90031]: pgmap v1596: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 20085/212871 objects degraded (9.435%); 13189/212871 objects misplaced (6.196%); 8.0 MiB/s, 127 objects/s recovering 2024-08-29T15:42:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:31 smithi027 ceph-mon[98970]: pgmap v1596: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 20085/212871 objects degraded (9.435%); 13189/212871 objects misplaced (6.196%); 8.0 MiB/s, 127 objects/s recovering 2024-08-29T15:42:32.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:32.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:32.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:32.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:32.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:32.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:32.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:32.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:32.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:32.412 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:32.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:32.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:32.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:32.562 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:32.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:32.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:32.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:32.748 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:32.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:32.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:32.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:42:32.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:32.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:32.894 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:32.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:32.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 20085/212871 objects degraded (9.435%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:32.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 20085/212871 objects degraded (9.435%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:32.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 20085/212871 objects degraded (9.435%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:33.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:33.030 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:33.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:33.095 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:33.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:33.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:42:33.158 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:33.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:33.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:33.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:33.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:33.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:33.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:33.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:33.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:33.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:33.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:33.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:33.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:33.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:33.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:33.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:33.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:33.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:33.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:33.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:33.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:33.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:33.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:34.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:34.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:33 smithi171 ceph-mon[90031]: pgmap v1597: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 19767/208887 objects degraded (9.463%); 13189/208887 objects misplaced (6.314%); 9.2 MiB/s, 147 objects/s recovering 2024-08-29T15:42:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:33 smithi027 ceph-mon[98970]: pgmap v1597: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 19767/208887 objects degraded (9.463%); 13189/208887 objects misplaced (6.314%); 9.2 MiB/s, 147 objects/s recovering 2024-08-29T15:42:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:33 smithi027 ceph-mon[93921]: pgmap v1597: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 19767/208887 objects degraded (9.463%); 13189/208887 objects misplaced (6.314%); 9.2 MiB/s, 147 objects/s recovering 2024-08-29T15:42:34.104 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:34.128 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:34.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:34.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:34.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:34.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:34.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:34.380 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:34.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:34.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:34.518 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:34.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:34.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:34.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:34.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.785 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:34.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:34.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:34.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:34.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:34.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:34.989 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:42:34.990 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:42:35.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:35.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:35.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:35.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:35.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:35.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:35.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:35.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:35.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:35.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:35.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:35.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:35.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:35.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:35.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:35.573 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:35.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:35.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:35.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:35.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:42:35.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:35.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:35.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:35.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:35.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:35.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:35.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:35.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:35.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:36.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:36.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:36.054 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:36.131 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:36.172 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:36.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:36.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:36.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:36 smithi027 ceph-mon[93921]: pgmap v1598: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 19767/208443 objects degraded (9.483%); 13189/208443 objects misplaced (6.327%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:42:36.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:36 smithi027 ceph-mon[98970]: pgmap v1598: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 19767/208443 objects degraded (9.483%); 13189/208443 objects misplaced (6.327%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:42:36.343 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:36 smithi171 ceph-mon[90031]: pgmap v1598: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 19767/208443 objects degraded (9.483%); 13189/208443 objects misplaced (6.327%); 5.6 MiB/s, 89 objects/s recovering 2024-08-29T15:42:36.402 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:36.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:36.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:36.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:36.556 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:36.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:36.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:36.691 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:36.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:36.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:36.937 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:36.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:36.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:36.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:37.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:37.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:37.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:37.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:37.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:37.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:37.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:37.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:37.377 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:37.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:37.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:37.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:37.503 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:37.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:37.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:37.605 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:37.632 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:37.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:37.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:37.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:37.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:37.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:37.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:37.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:37.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:37.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:38.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:38.040 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:38.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:38.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:38.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:38.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:38.220 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:42:38.220 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:42:38.282 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:38.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:38.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:38.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:38 smithi027 ceph-mon[98970]: pgmap v1599: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 19521/203256 objects degraded (9.604%); 12854/203256 objects misplaced (6.324%); 6.9 MiB/s, 110 objects/s recovering 2024-08-29T15:42:38.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:38 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 19521/203256 objects degraded (9.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:38 smithi027 ceph-mon[93921]: pgmap v1599: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 19521/203256 objects degraded (9.604%); 12854/203256 objects misplaced (6.324%); 6.9 MiB/s, 110 objects/s recovering 2024-08-29T15:42:38.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:38 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 19521/203256 objects degraded (9.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:38.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:38.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:38.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:38 smithi171 ceph-mon[90031]: pgmap v1599: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 19521/203256 objects degraded (9.604%); 12854/203256 objects misplaced (6.324%); 6.9 MiB/s, 110 objects/s recovering 2024-08-29T15:42:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:38 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 19521/203256 objects degraded (9.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:38.553 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:38.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:38.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:38.689 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:38.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:38.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:38.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:38.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:38.861 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:38.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:38.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:38.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:42:39.002 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:39.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:39.103 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:39.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:39.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:39.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:39.260 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:39.291 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:42:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:42:39] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-29T15:42:39.392 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:39.418 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:39.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:39.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:39.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:39.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:39.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:39.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:39.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.827 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:39.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:39.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:39.973 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:39.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:39.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:40.075 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:40.100 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:40.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:40.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:40.233 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:40.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:40.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:40 smithi027 ceph-mon[93921]: pgmap v1600: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 19215/200994 objects degraded (9.560%); 12854/200994 objects misplaced (6.395%); 7.0 MiB/s, 111 objects/s recovering 2024-08-29T15:42:40.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:40 smithi027 ceph-mon[98970]: pgmap v1600: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 19215/200994 objects degraded (9.560%); 12854/200994 objects misplaced (6.395%); 7.0 MiB/s, 111 objects/s recovering 2024-08-29T15:42:40.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:40.378 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:40.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:40.432 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:40 smithi171 ceph-mon[90031]: pgmap v1600: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 19215/200994 objects degraded (9.560%); 12854/200994 objects misplaced (6.395%); 7.0 MiB/s, 111 objects/s recovering 2024-08-29T15:42:40.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:40.510 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:40.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:40.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:40.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:40.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:40.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:40.762 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:40.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:40.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:40.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:40.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:40.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:40.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:40.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:41.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:41.068 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:42:41.069 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:42:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:41.071 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:41.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:41.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:41.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:41.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:41.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:41.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:41.356 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:41.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:41.450 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:42:41.450 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:42:41.451 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:42:41.458 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:41.482 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:41.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:41.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:41.617 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:41.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:41.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:41.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:41.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:41.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:41.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:42:41.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:41.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:41.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:41.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:42.021 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:42.087 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:42:42.088 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:42:42.123 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:42.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:42.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:42.280 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:42.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:42 smithi171 ceph-mon[90031]: pgmap v1601: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 18926/197073 objects degraded (9.604%); 12515/197073 objects misplaced (6.350%); 8.5 MiB/s, 136 objects/s recovering 2024-08-29T15:42:42.368 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1291975124' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:42.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:42.411 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:42.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:42.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:42.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:42 smithi027 ceph-mon[98970]: pgmap v1601: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 18926/197073 objects degraded (9.604%); 12515/197073 objects misplaced (6.350%); 8.5 MiB/s, 136 objects/s recovering 2024-08-29T15:42:42.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1291975124' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:42 smithi027 ceph-mon[93921]: pgmap v1601: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 18926/197073 objects degraded (9.604%); 12515/197073 objects misplaced (6.350%); 8.5 MiB/s, 136 objects/s recovering 2024-08-29T15:42:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1291975124' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:42:42.640 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:42.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:42.767 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:42.797 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:42.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:42.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:42.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:43.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:43.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:43.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:43.165 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:43.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:43.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:43 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:42:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1291975124' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:42:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:43 smithi171 ceph-mon[90031]: osdmap e453: 8 total, 8 up, 8 in 2024-08-29T15:42:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:43 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 18926/197073 objects degraded (9.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:43.296 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:43.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:43.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:43.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:43.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:43.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:43.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:43.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:42:43.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1291975124' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:42:43.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[98970]: osdmap e453: 8 total, 8 up, 8 in 2024-08-29T15:42:43.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 18926/197073 objects degraded (9.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:43.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:42:43.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1291975124' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:42:43.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[93921]: osdmap e453: 8 total, 8 up, 8 in 2024-08-29T15:42:43.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:43 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 18926/197073 objects degraded (9.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:43.611 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:43.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:43.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:43.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:43.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:43.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:43.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:43.886 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:43.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:43.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:43.959 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:42:43.959 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:42:43.988 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.019 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:44.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:44.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:44.162 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:44.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:44.264 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:44 smithi171 ceph-mon[90031]: pgmap v1603: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 18578/193119 objects degraded (9.620%); 12515/193119 objects misplaced (6.480%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:42:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:44.290 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:44.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:44.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:44.432 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:44.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:44.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:44 smithi027 ceph-mon[98970]: pgmap v1603: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 18578/193119 objects degraded (9.620%); 12515/193119 objects misplaced (6.480%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:42:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:44 smithi027 ceph-mon[93921]: pgmap v1603: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 18578/193119 objects degraded (9.620%); 12515/193119 objects misplaced (6.480%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:42:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:44.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:44.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:44.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:44.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:44.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:42:44.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:44.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:44.943 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:44.966 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:44.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:44.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:45.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:45.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:45.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:45.122 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:45.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:45.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:45.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:45.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:45.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:45.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:45.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:45.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:45.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:45.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:45.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:45.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:45.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:45.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:45.789 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:45.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:45.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:45.927 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:45.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:45.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:45.941 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:46.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:46.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:46.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:46.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:46.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:46.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:46.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:46.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:46 smithi027 ceph-mon[93921]: pgmap v1604: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 18578/192624 objects degraded (9.645%); 12515/192624 objects misplaced (6.497%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:42:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:46 smithi027 ceph-mon[98970]: pgmap v1604: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 18578/192624 objects degraded (9.645%); 12515/192624 objects misplaced (6.497%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:42:46.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:46.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:46.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:46.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:46.474 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:46.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:46 smithi171 ceph-mon[90031]: pgmap v1604: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 18578/192624 objects degraded (9.645%); 12515/192624 objects misplaced (6.497%); 7.6 MiB/s, 122 objects/s recovering 2024-08-29T15:42:46.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:46.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:46.629 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:46.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:46.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:42:46.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:46.769 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:46.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:46.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:46.894 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:46.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:46.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:47.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:47.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:47.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:47.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:47.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:47.318 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:47.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:47.420 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:47.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:47.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:47.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:47.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:47.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:47.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:47.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:47.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:47.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:47.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:47.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:47.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:47.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:48.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:48.102 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:48.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:48.159 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:42:48.159 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:42:48.159 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:42:48.159 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:42:48.159 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:42:48.159 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:42:48.160 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:42:48.169 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:42:48.169 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:42:48.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:48.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:48.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:48 smithi027 ceph-mon[98970]: pgmap v1605: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 18302/187821 objects degraded (9.744%); 12159/187821 objects misplaced (6.474%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:42:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:48 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 18302/187821 objects degraded (9.744%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1487200690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:48 smithi027 ceph-mon[93921]: pgmap v1605: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 18302/187821 objects degraded (9.744%); 12159/187821 objects misplaced (6.474%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:42:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:48 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 18302/187821 objects degraded (9.744%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1487200690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:48.329 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:48.355 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:48 smithi171 ceph-mon[90031]: pgmap v1605: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 18302/187821 objects degraded (9.744%); 12159/187821 objects misplaced (6.474%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:42:48.355 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:48 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 18302/187821 objects degraded (9.744%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:48.355 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1487200690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:42:48.358 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:48.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:48.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:48.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:48.503 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:48.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:48.605 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:48.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:48.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:48.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:48.769 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:48.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:48.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:48.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:48.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:48.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:48.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:49.007 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:49.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:49.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:49.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:49.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:49.284 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:42:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:42:49] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-29T15:42:49.386 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:49.412 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:49.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:49.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:49.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:49.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:49.778 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:49.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:49.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:49.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:49.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:50.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:50.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:50.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:50.161 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:42:50.161 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:42:50.162 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-08-29T15:42:50.162 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:42:50.177 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:50.263 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:50.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:50.365 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:50.389 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:50.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:50.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:50.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:50.513 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:50.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:50.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:50 smithi171 ceph-mon[90031]: pgmap v1606: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 17930/185331 objects degraded (9.675%); 12159/185331 objects misplaced (6.561%); 8.1 MiB/s, 129 objects/s recovering 2024-08-29T15:42:50.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:50 smithi027 ceph-mon[93921]: pgmap v1606: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 17930/185331 objects degraded (9.675%); 12159/185331 objects misplaced (6.561%); 8.1 MiB/s, 129 objects/s recovering 2024-08-29T15:42:50.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:50 smithi027 ceph-mon[98970]: pgmap v1606: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 17930/185331 objects degraded (9.675%); 12159/185331 objects misplaced (6.561%); 8.1 MiB/s, 129 objects/s recovering 2024-08-29T15:42:50.616 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:50.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:50.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:50.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:50.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:50.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:50.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:50.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:50.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:50.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:50.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:51.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:51.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:51.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:51.170 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:51.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:51.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:51.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:51.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:51.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:51.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:51.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:51.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:51.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:51.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:51.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.859 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:51.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:51.960 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:51.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:51.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:51.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:52.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:52.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:52.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:52.215 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:52.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:52.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:52.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:52.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:52.373 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:52.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:52.475 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:52.499 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:52.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:52.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:52 smithi171 ceph-mon[90031]: pgmap v1607: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 17635/181341 objects degraded (9.725%); 11767/181341 objects misplaced (6.489%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:42:52.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:52 smithi027 ceph-mon[98970]: pgmap v1607: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 17635/181341 objects degraded (9.725%); 11767/181341 objects misplaced (6.489%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:42:52.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:52 smithi027 ceph-mon[93921]: pgmap v1607: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 17635/181341 objects degraded (9.725%); 11767/181341 objects misplaced (6.489%); 8.4 MiB/s, 133 objects/s recovering 2024-08-29T15:42:52.602 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:52.631 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:52.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:52.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:52.745 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:52.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:52.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:52.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:52.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:52.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:52.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:53.064 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:53.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:53.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:53.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:53.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:53.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:53.299 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:53 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 17635/181341 objects degraded (9.725%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:53.299 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:53 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 17635/181341 objects degraded (9.725%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:53.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:53.335 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:53.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:53.373 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:53.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:53.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:53 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 17635/181341 objects degraded (9.725%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:53.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:53.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:53.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:53.672 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:53.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:53.773 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:53.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:53.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:53.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:53.929 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:53.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:54.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:54.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:54.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:54.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:54.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:54.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:54.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:54.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:54.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:54.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:54.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:54 smithi027 ceph-mon[93921]: pgmap v1608: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 17321/177360 objects degraded (9.766%); 11767/177360 objects misplaced (6.635%); 8.5 MiB/s, 136 objects/s recovering 2024-08-29T15:42:54.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:54 smithi027 ceph-mon[98970]: pgmap v1608: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 17321/177360 objects degraded (9.766%); 11767/177360 objects misplaced (6.635%); 8.5 MiB/s, 136 objects/s recovering 2024-08-29T15:42:54.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:54.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:54.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:54 smithi171 ceph-mon[90031]: pgmap v1608: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 17321/177360 objects degraded (9.766%); 11767/177360 objects misplaced (6.635%); 8.5 MiB/s, 136 objects/s recovering 2024-08-29T15:42:54.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:54.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:54.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:54.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:42:54.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:54.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:54.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:54.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:54.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:54.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:55.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:55.061 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:55.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:55.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:55.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:55.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:55.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:55.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:55.347 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:55.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:55.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:55.448 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:42:55.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:55.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:42:55.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:55.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:55.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:55.733 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:55.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:55.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:55.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:55.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:55.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:55.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:55.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:55.994 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:56.036 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:56.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:56.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:56.241 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:42:56.241 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:42:56.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.241 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(9.7 453/446 [PushOp(9:e290f4e5:::benchmark_data_smithi027_341149_object46063:head, version: 430'2848, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:e290f4e5:::benchmark_data_smithi027_341149_object46063:head@430'2848, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:56.054718+0000", 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.008279949, 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083304829999999996, 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.054718+0000", 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.054718+0000", 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.054719+0000", 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2780000000000001e-06 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.054775+0000", 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5244999999999997e-05 2024-08-29T15:42:56.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.054804+0000", 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9431999999999999e-05 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.054838+0000", 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3850000000000003e-05 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.055014+0000", 2024-08-29T15:42:56.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000176534 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.055059+0000", 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4833000000000003e-05 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:148495 9.2 e453/440 9:46e820b8:::benchmark_data_smithi027_341149_object46065:head v 453'9248, mlcod=430'2886)", 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:56.058107+0000", 2024-08-29T15:42:56.245 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0048910790000000004, 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049933419999999996, 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058107+0000", 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:42:56.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058107+0000", 2024-08-29T15:42:56.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:42:56.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:42:56.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058109+0000", 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.838e-06 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058118+0000", 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5699999999999999e-06 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:42:56.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058139+0000", 2024-08-29T15:42:56.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0913000000000001e-05 2024-08-29T15:42:56.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058150+0000", 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0285000000000001e-05 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058239+0000", 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9398e-05 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:42:56.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:56.058272+0000", 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3509000000000001e-05 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 2 2024-08-29T15:42:56.251 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:42:56.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:56.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:56.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:56 smithi027 ceph-mon[93921]: pgmap v1609: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 17321/176841 objects degraded (9.795%); 11767/176841 objects misplaced (6.654%); 6.9 MiB/s, 110 objects/s recovering 2024-08-29T15:42:56.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:56 smithi027 ceph-mon[98970]: pgmap v1609: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 17321/176841 objects degraded (9.795%); 11767/176841 objects misplaced (6.654%); 6.9 MiB/s, 110 objects/s recovering 2024-08-29T15:42:56.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:56.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:42:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:56 smithi171 ceph-mon[90031]: pgmap v1609: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 17321/176841 objects degraded (9.795%); 11767/176841 objects misplaced (6.654%); 6.9 MiB/s, 110 objects/s recovering 2024-08-29T15:42:56.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:56.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:56.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:56.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:56.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:56.831 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:56.933 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:56.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:56.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:56.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:57.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:57.089 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:57.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:57.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:57.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:57.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:57.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:57.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:57.372 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:57.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:57.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:57.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:57.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:57.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:57.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:57.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:57.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:42:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:57.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:57.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:57.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:57.794 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:57.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:57.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:57.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:57.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:58.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:58.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:58.106 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:58.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:58.150 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:58 smithi027 ceph-mon[98970]: pgmap v1610: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 17050/171504 objects degraded (9.941%); 11382/171504 objects misplaced (6.637%); 9.5 MiB/s, 151 objects/s recovering 2024-08-29T15:42:58.150 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:58 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 17050/171504 objects degraded (9.941%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:58.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:58 smithi027 ceph-mon[93921]: pgmap v1610: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 17050/171504 objects degraded (9.941%); 11382/171504 objects misplaced (6.637%); 9.5 MiB/s, 151 objects/s recovering 2024-08-29T15:42:58.150 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:58 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 17050/171504 objects degraded (9.941%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:58.207 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:58.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:58.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:58.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:58.311 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:42:58.314 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:42:58.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:58.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:58.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:58.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:58 smithi171 ceph-mon[90031]: pgmap v1610: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 17050/171504 objects degraded (9.941%); 11382/171504 objects misplaced (6.637%); 9.5 MiB/s, 151 objects/s recovering 2024-08-29T15:42:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:58 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 17050/171504 objects degraded (9.941%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:42:58.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:58.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:58.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:58.632 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:58.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:58.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:58.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:58.764 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:42:58.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:58.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:42:58.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:58.944 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:58.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:59.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:59.063 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1610,"stamp":"2024-08-29T15:42:57.029137+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3749451773,"num_objects":57168,"num_object_clones":0,"num_object_copies":171504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17050,"num_objects_misplaced":11382,"num_objects_unfound":0,"num_objects_dirty":57168,"num_whiteouts":0,"num_read":5988,"num_read_kb":13001,"num_write":148624,"num_write_kb":6567954,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16885,"num_bytes_recovered":1103700075,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127489,"ondisk_log_size":127489,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18074916,"kb_used_data":12641188,"kb_used_omap":43,"kb_used_meta":5433620,"kb_avail":731886300,"statfs":{"total":767960285184,"available":749451571200,"internally_reserved":0,"allocated":12944576512,"data_stored":12921966070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":5564027372},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,7,8,4,2,4,3,1,0,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-461373440,"num_objects":-7040,"num_object_clones":0,"num_object_copies":-21120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1528,"num_objects_misplaced":-1133,"num_objects_unfound":0,"num_objects_dirty":-7040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7040,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1819,"num_bytes_recovered":119209984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.007498"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":819,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105380+0000","last_change":"2024-08-29T15:41:16.221956+0000","last_active":"2024-08-29T15:42:42.105380+0000","last_peered":"2024-08-29T15:42:42.105380+0000","last_clean":"2024-08-29T15:42:42.105380+0000","last_became_active":"2024-08-29T15:41:16.221533+0000","last_became_peered":"2024-08-29T15:41:16.221533+0000","last_unstale":"2024-08-29T15:42:42.105380+0000","last_undegraded":"2024-08-29T15:42:42.105380+0000","last_fullsized":"2024-08-29T15:42:42.105380+0000","mapping_epoch":439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:26:34.947614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":245,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.268048+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:42:26.268048+0000","last_peered":"2024-08-29T15:42:26.268048+0000","last_clean":"2024-08-29T15:42:26.268048+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:42:26.268048+0000","last_undegraded":"2024-08-29T15:42:26.268048+0000","last_fullsized":"2024-08-29T15:42:26.268048+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":40,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.257781+0000","last_change":"2024-08-29T15:39:53.258205+0000","last_active":"2024-08-29T15:42:26.257781+0000","last_peered":"2024-08-29T15:42:26.257781+0000","last_clean":"2024-08-29T15:42:26.257781+0000","last_became_active":"2024-08-29T15:39:53.257784+0000","last_became_peered":"2024-08-29T15:39:53.257784+0000","last_unstale":"2024-08-29T15:42:26.257781+0000","last_undegraded":"2024-08-29T15:42:26.257781+0000","last_fullsized":"2024-08-29T15:42:26.257781+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:30:42.270982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":778,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.091764+0000","last_change":"2024-08-29T15:39:21.000890+0000","last_active":"2024-08-29T15:42:42.091764+0000","last_peered":"2024-08-29T15:42:42.091764+0000","last_clean":"2024-08-29T15:42:42.091764+0000","last_became_active":"2024-08-29T15:39:21.000547+0000","last_became_peered":"2024-08-29T15:39:21.000547+0000","last_unstale":"2024-08-29T15:42:42.091764+0000","last_undegraded":"2024-08-29T15:42:42.091764+0000","last_fullsized":"2024-08-29T15:42:42.091764+0000","mapping_epoch":410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:43:30.808747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":770,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.092941+0000","last_change":"2024-08-29T15:41:44.447039+0000","last_active":"2024-08-29T15:42:42.092941+0000","last_peered":"2024-08-29T15:42:42.092941+0000","last_clean":"2024-08-29T15:42:42.092941+0000","last_became_active":"2024-08-29T15:41:44.446631+0000","last_became_peered":"2024-08-29T15:41:44.446631+0000","last_unstale":"2024-08-29T15:42:42.092941+0000","last_undegraded":"2024-08-29T15:42:42.092941+0000","last_fullsized":"2024-08-29T15:42:42.092941+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:02:25.609598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":238,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267720+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:42:26.267720+0000","last_peered":"2024-08-29T15:42:26.267720+0000","last_clean":"2024-08-29T15:42:26.267720+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:42:26.267720+0000","last_undegraded":"2024-08-29T15:42:26.267720+0000","last_fullsized":"2024-08-29T15:42:26.267720+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":519,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104767+0000","last_change":"2024-08-29T15:41:44.469235+0000","last_active":"2024-08-29T15:42:42.104767+0000","last_peered":"2024-08-29T15:42:42.104767+0000","last_clean":"2024-08-29T15:42:42.104767+0000","last_became_active":"2024-08-29T15:41:44.468857+0000","last_became_peered":"2024-08-29T15:41:44.468857+0000","last_unstale":"2024-08-29T15:42:42.104767+0000","last_undegraded":"2024-08-29T15:42:42.104767+0000","last_fullsized":"2024-08-29T15:42:42.104767+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:22:54.347105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":208,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.099481+0000","last_change":"2024-08-29T15:41:00.757152+0000","last_active":"2024-08-29T15:42:42.099481+0000","last_peered":"2024-08-29T15:42:42.099481+0000","last_clean":"2024-08-29T15:42:42.099481+0000","last_became_active":"2024-08-29T15:41:00.748843+0000","last_became_peered":"2024-08-29T15:41:00.748843+0000","last_unstale":"2024-08-29T15:42:42.099481+0000","last_undegraded":"2024-08-29T15:42:42.099481+0000","last_fullsized":"2024-08-29T15:42:42.099481+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:52:45.532884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.5","version":"453'9391","reported_seq":18544,"reported_epoch":453,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-29T15:42:55.234195+0000","last_change":"2024-08-29T15:41:02.070908+0000","last_active":"2024-08-29T15:42:55.234195+0000","last_peered":"2024-08-29T15:42:55.234195+0000","last_clean":"2024-08-29T15:40:59.706274+0000","last_became_active":"2024-08-29T15:41:01.969325+0000","last_became_peered":"2024-08-29T15:41:01.969325+0000","last_unstale":"2024-08-29T15:42:55.234195+0000","last_undegraded":"2024-08-29T15:42:55.234195+0000","last_fullsized":"2024-08-29T15:41:01.757718+0000","mapping_epoch":435,"log_start":"429'1400","ondisk_log_start":"429'1400","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7991,"log_dups_size":0,"ondisk_log_size":7991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236519424,"num_objects":3609,"num_object_clones":0,"num_object_copies":10827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1793,"num_objects_unfound":0,"num_objects_dirty":3609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9391,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4536,"num_bytes_recovered":297271296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":1816},{"shards":"0,5,7","objects":1793}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":269,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.268392+0000","last_change":"2024-08-29T15:37:26.322453+0000","last_active":"2024-08-29T15:42:26.268392+0000","last_peered":"2024-08-29T15:42:26.268392+0000","last_clean":"2024-08-29T15:42:26.268392+0000","last_became_active":"2024-08-29T15:37:26.308813+0000","last_became_peered":"2024-08-29T15:37:26.308813+0000","last_unstale":"2024-08-29T15:42:26.268392+0000","last_undegraded":"2024-08-29T15:42:26.268392+0000","last_fullsized":"2024-08-29T15:42:26.268392+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:05:55.778279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":40,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.256179+0000","last_change":"2024-08-29T15:39:53.258473+0000","last_active":"2024-08-29T15:42:26.256179+0000","last_peered":"2024-08-29T15:42:26.256179+0000","last_clean":"2024-08-29T15:42:26.256179+0000","last_became_active":"2024-08-29T15:39:53.258055+0000","last_became_peered":"2024-08-29T15:39:53.258055+0000","last_unstale":"2024-08-29T15:42:26.256179+0000","last_undegraded":"2024-08-29T15:42:26.256179+0000","last_fullsized":"2024-08-29T15:42:26.256179+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:38:54.441868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"453'9314","reported_seq":9391,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:55.928980+0000","last_change":"2024-08-29T15:39:37.186986+0000","last_active":"2024-08-29T15:42:55.928980+0000","last_peered":"2024-08-29T15:42:55.928980+0000","last_clean":"2024-08-29T15:42:55.928980+0000","last_became_active":"2024-08-29T15:39:37.186669+0000","last_became_peered":"2024-08-29T15:39:37.186669+0000","last_unstale":"2024-08-29T15:42:55.928980+0000","last_undegraded":"2024-08-29T15:42:55.928980+0000","last_fullsized":"2024-08-29T15:42:55.928980+0000","mapping_epoch":420,"log_start":"430'1900","ondisk_log_start":"430'1900","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7414,"log_dups_size":0,"ondisk_log_size":7414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:33:01.221948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225181696,"num_objects":3436,"num_object_clones":0,"num_object_copies":10308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9314,"num_write_kb":408000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":243,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.268490+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:42:26.268490+0000","last_peered":"2024-08-29T15:42:26.268490+0000","last_clean":"2024-08-29T15:42:26.268490+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:42:26.268490+0000","last_undegraded":"2024-08-29T15:42:26.268490+0000","last_fullsized":"2024-08-29T15:42:26.268490+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":275,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267575+0000","last_change":"2024-08-29T15:41:44.436101+0000","last_active":"2024-08-29T15:42:26.267575+0000","last_peered":"2024-08-29T15:42:26.267575+0000","last_clean":"2024-08-29T15:42:26.267575+0000","last_became_active":"2024-08-29T15:41:44.435769+0000","last_became_peered":"2024-08-29T15:41:44.435769+0000","last_unstale":"2024-08-29T15:42:26.267575+0000","last_undegraded":"2024-08-29T15:42:26.267575+0000","last_fullsized":"2024-08-29T15:42:26.267575+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:20:19.269808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"453'9072","reported_seq":15635,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:42:52.290989+0000","last_change":"2024-08-29T15:41:45.701312+0000","last_active":"2024-08-29T15:42:52.290989+0000","last_peered":"2024-08-29T15:42:52.290989+0000","last_clean":"2024-08-29T15:41:03.137460+0000","last_became_active":"2024-08-29T15:41:45.687083+0000","last_became_peered":"2024-08-29T15:41:45.687083+0000","last_unstale":"2024-08-29T15:42:52.290989+0000","last_undegraded":"2024-08-29T15:41:45.573091+0000","last_fullsized":"2024-08-29T15:41:45.435192+0000","mapping_epoch":446,"log_start":"429'900","ondisk_log_start":"429'900","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:21:14.334460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237633536,"num_objects":3626,"num_object_clones":0,"num_object_copies":10878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8203,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9072,"num_write_kb":406336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2168,"num_bytes_recovered":142016512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"2,3,5","objects":159},{"shards":"3,5,7","objects":3467}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":44,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.257415+0000","last_change":"2024-08-29T15:39:53.262167+0000","last_active":"2024-08-29T15:42:26.257415+0000","last_peered":"2024-08-29T15:42:26.257415+0000","last_clean":"2024-08-29T15:42:26.257415+0000","last_became_active":"2024-08-29T15:39:53.258406+0000","last_became_peered":"2024-08-29T15:39:53.258406+0000","last_unstale":"2024-08-29T15:42:26.257415+0000","last_undegraded":"2024-08-29T15:42:26.257415+0000","last_fullsized":"2024-08-29T15:42:26.257415+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:05:19.731559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":399,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.112728+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:42:42.112728+0000","last_peered":"2024-08-29T15:42:42.112728+0000","last_clean":"2024-08-29T15:42:42.112728+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:42:42.112728+0000","last_undegraded":"2024-08-29T15:42:42.112728+0000","last_fullsized":"2024-08-29T15:42:42.112728+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"453'9077","reported_seq":9120,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:52.294069+0000","last_change":"2024-08-29T15:39:37.376667+0000","last_active":"2024-08-29T15:42:52.294069+0000","last_peered":"2024-08-29T15:42:52.294069+0000","last_clean":"2024-08-29T15:42:52.294069+0000","last_became_active":"2024-08-29T15:39:37.376372+0000","last_became_peered":"2024-08-29T15:39:37.376372+0000","last_unstale":"2024-08-29T15:42:52.294069+0000","last_undegraded":"2024-08-29T15:42:52.294069+0000","last_fullsized":"2024-08-29T15:42:52.294069+0000","mapping_epoch":420,"log_start":"429'900","ondisk_log_start":"429'900","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":8177,"log_dups_size":0,"ondisk_log_size":8177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:52:02.092878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240713728,"num_objects":3673,"num_object_clones":0,"num_object_copies":11019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9077,"num_write_kb":408000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":347,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.257579+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:42:26.257579+0000","last_peered":"2024-08-29T15:42:26.257579+0000","last_clean":"2024-08-29T15:42:26.257579+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:42:26.257579+0000","last_undegraded":"2024-08-29T15:42:26.257579+0000","last_fullsized":"2024-08-29T15:42:26.257579+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":778,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.094596+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:42:42.094596+0000","last_peered":"2024-08-29T15:42:42.094596+0000","last_clean":"2024-08-29T15:42:42.094596+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:42:42.094596+0000","last_undegraded":"2024-08-29T15:42:42.094596+0000","last_fullsized":"2024-08-29T15:42:42.094596+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"453'9347","reported_seq":9425,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:56.642677+0000","last_change":"2024-08-29T15:39:37.311470+0000","last_active":"2024-08-29T15:42:56.642677+0000","last_peered":"2024-08-29T15:42:56.642677+0000","last_clean":"2024-08-29T15:42:56.642677+0000","last_became_active":"2024-08-29T15:39:37.310913+0000","last_became_peered":"2024-08-29T15:39:37.310913+0000","last_unstale":"2024-08-29T15:42:56.642677+0000","last_undegraded":"2024-08-29T15:42:56.642677+0000","last_fullsized":"2024-08-29T15:42:56.642677+0000","mapping_epoch":420,"log_start":"430'1943","ondisk_log_start":"430'1943","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:39.550827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230490112,"num_objects":3517,"num_object_clones":0,"num_object_copies":10551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9347,"num_write_kb":411648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":901,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104179+0000","last_change":"2024-08-29T15:37:18.101543+0000","last_active":"2024-08-29T15:42:42.104179+0000","last_peered":"2024-08-29T15:42:42.104179+0000","last_clean":"2024-08-29T15:42:42.104179+0000","last_became_active":"2024-08-29T15:37:01.308140+0000","last_became_peered":"2024-08-29T15:37:01.308140+0000","last_unstale":"2024-08-29T15:42:42.104179+0000","last_undegraded":"2024-08-29T15:42:42.104179+0000","last_fullsized":"2024-08-29T15:42:42.104179+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:18.101472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:18.101472+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:06:29.187841+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":795,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.106359+0000","last_change":"2024-08-29T15:37:26.340871+0000","last_active":"2024-08-29T15:42:42.106359+0000","last_peered":"2024-08-29T15:42:42.106359+0000","last_clean":"2024-08-29T15:42:42.106359+0000","last_became_active":"2024-08-29T15:37:26.328280+0000","last_became_peered":"2024-08-29T15:37:26.328280+0000","last_unstale":"2024-08-29T15:42:42.106359+0000","last_undegraded":"2024-08-29T15:42:42.106359+0000","last_fullsized":"2024-08-29T15:42:42.106359+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:47:01.239361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":239,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267999+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:42:26.267999+0000","last_peered":"2024-08-29T15:42:26.267999+0000","last_clean":"2024-08-29T15:42:26.267999+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:42:26.267999+0000","last_undegraded":"2024-08-29T15:42:26.267999+0000","last_fullsized":"2024-08-29T15:42:26.267999+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"453'9020","reported_seq":9099,"reported_epoch":453,"state":"active+recovery_wait+remapped","last_fresh":"2024-08-29T15:42:55.922694+0000","last_change":"2024-08-29T15:41:45.679775+0000","last_active":"2024-08-29T15:42:55.922694+0000","last_peered":"2024-08-29T15:42:55.922694+0000","last_clean":"2024-08-29T15:41:43.386876+0000","last_became_active":"2024-08-29T15:41:45.666366+0000","last_became_peered":"2024-08-29T15:41:45.666366+0000","last_unstale":"2024-08-29T15:42:55.922694+0000","last_undegraded":"2024-08-29T15:42:55.922694+0000","last_fullsized":"2024-08-29T15:42:55.922694+0000","mapping_epoch":446,"log_start":"430'1700","ondisk_log_start":"430'1700","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7320,"log_dups_size":0,"ondisk_log_size":7320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228589568,"num_objects":3488,"num_object_clones":0,"num_object_copies":10464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6254,"num_objects_unfound":0,"num_objects_dirty":3488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9020,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":-2249}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":876,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.103515+0000","last_change":"2024-08-29T15:41:03.142904+0000","last_active":"2024-08-29T15:42:42.103515+0000","last_peered":"2024-08-29T15:42:42.103515+0000","last_clean":"2024-08-29T15:42:42.103515+0000","last_became_active":"2024-08-29T15:41:03.142474+0000","last_became_peered":"2024-08-29T15:41:03.142474+0000","last_unstale":"2024-08-29T15:42:42.103515+0000","last_undegraded":"2024-08-29T15:42:42.103515+0000","last_fullsized":"2024-08-29T15:42:42.103515+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:40:21.404385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:40:21.404385+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:57:43.184842+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":234,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267764+0000","last_change":"2024-08-29T15:41:44.443161+0000","last_active":"2024-08-29T15:42:26.267764+0000","last_peered":"2024-08-29T15:42:26.267764+0000","last_clean":"2024-08-29T15:42:26.267764+0000","last_became_active":"2024-08-29T15:41:44.442573+0000","last_became_peered":"2024-08-29T15:41:44.442573+0000","last_unstale":"2024-08-29T15:42:26.267764+0000","last_undegraded":"2024-08-29T15:42:26.267764+0000","last_fullsized":"2024-08-29T15:42:26.267764+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:31:08.897327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.268508+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:42:26.268508+0000","last_peered":"2024-08-29T15:42:26.268508+0000","last_clean":"2024-08-29T15:42:26.268508+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:42:26.268508+0000","last_undegraded":"2024-08-29T15:42:26.268508+0000","last_fullsized":"2024-08-29T15:42:26.268508+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"453'9329","reported_seq":9407,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:56.639383+0000","last_change":"2024-08-29T15:39:37.258693+0000","last_active":"2024-08-29T15:42:56.639383+0000","last_peered":"2024-08-29T15:42:56.639383+0000","last_clean":"2024-08-29T15:42:56.639383+0000","last_became_active":"2024-08-29T15:39:37.258303+0000","last_became_peered":"2024-08-29T15:39:37.258303+0000","last_unstale":"2024-08-29T15:42:56.639383+0000","last_undegraded":"2024-08-29T15:42:56.639383+0000","last_fullsized":"2024-08-29T15:42:56.639383+0000","mapping_epoch":420,"log_start":"430'1954","ondisk_log_start":"430'1954","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:53:38.911068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9329,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":239,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267656+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:42:26.267656+0000","last_peered":"2024-08-29T15:42:26.267656+0000","last_clean":"2024-08-29T15:42:26.267656+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:42:26.267656+0000","last_undegraded":"2024-08-29T15:42:26.267656+0000","last_fullsized":"2024-08-29T15:42:26.267656+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":472,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.087493+0000","last_change":"2024-08-29T15:41:00.747697+0000","last_active":"2024-08-29T15:42:42.087493+0000","last_peered":"2024-08-29T15:42:42.087493+0000","last_clean":"2024-08-29T15:42:42.087493+0000","last_became_active":"2024-08-29T15:41:00.747350+0000","last_became_peered":"2024-08-29T15:41:00.747350+0000","last_unstale":"2024-08-29T15:42:42.087493+0000","last_undegraded":"2024-08-29T15:42:42.087493+0000","last_fullsized":"2024-08-29T15:42:42.087493+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:13:16.221880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"453'9245","reported_seq":11084,"reported_epoch":453,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-29T15:42:55.954557+0000","last_change":"2024-08-29T15:42:18.569141+0000","last_active":"2024-08-29T15:42:55.954557+0000","last_peered":"2024-08-29T15:42:55.954557+0000","last_clean":"2024-08-29T15:41:02.137059+0000","last_became_active":"2024-08-29T15:41:17.378848+0000","last_became_peered":"2024-08-29T15:41:17.378848+0000","last_unstale":"2024-08-29T15:42:55.954557+0000","last_undegraded":"2024-08-29T15:41:17.245306+0000","last_fullsized":"2024-08-29T15:41:17.211749+0000","mapping_epoch":440,"log_start":"430'1900","ondisk_log_start":"430'1900","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7345,"log_dups_size":0,"ondisk_log_size":7345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227868672,"num_objects":3477,"num_object_clones":0,"num_object_copies":10431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2005,"num_objects_misplaced":3335,"num_objects_unfound":0,"num_objects_dirty":3477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9245,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":880,"num_bytes_recovered":57671680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":-1863},{"shards":"1,2,3","objects":3335}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":808,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.257378+0000","last_change":"2024-08-29T15:37:24.681545+0000","last_active":"2024-08-29T15:42:26.257378+0000","last_peered":"2024-08-29T15:42:26.257378+0000","last_clean":"2024-08-29T15:42:26.257378+0000","last_became_active":"2024-08-29T15:37:01.312787+0000","last_became_peered":"2024-08-29T15:37:01.312787+0000","last_unstale":"2024-08-29T15:42:26.257378+0000","last_undegraded":"2024-08-29T15:42:26.257378+0000","last_fullsized":"2024-08-29T15:42:26.257378+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:24.681392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:24.681392+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:01:45.435539+0000","scrub_duration":0.0064019280000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":304,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.111418+0000","last_change":"2024-08-29T15:41:44.435967+0000","last_active":"2024-08-29T15:42:42.111418+0000","last_peered":"2024-08-29T15:42:42.111418+0000","last_clean":"2024-08-29T15:42:42.111418+0000","last_became_active":"2024-08-29T15:41:44.435570+0000","last_became_peered":"2024-08-29T15:41:44.435570+0000","last_unstale":"2024-08-29T15:42:42.111418+0000","last_undegraded":"2024-08-29T15:42:42.111418+0000","last_fullsized":"2024-08-29T15:42:42.111418+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:57:19.832668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":768,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104858+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:42:42.104858+0000","last_peered":"2024-08-29T15:42:42.104858+0000","last_clean":"2024-08-29T15:42:42.104858+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:42:42.104858+0000","last_undegraded":"2024-08-29T15:42:42.104858+0000","last_fullsized":"2024-08-29T15:42:42.104858+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"453'9285","reported_seq":17897,"reported_epoch":453,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:42:55.221078+0000","last_change":"2024-08-29T15:41:24.685591+0000","last_active":"2024-08-29T15:42:55.221078+0000","last_peered":"2024-08-29T15:42:55.221078+0000","last_clean":"2024-08-29T15:40:59.730315+0000","last_became_active":"2024-08-29T15:41:24.597966+0000","last_became_peered":"2024-08-29T15:41:24.597966+0000","last_unstale":"2024-08-29T15:42:55.221078+0000","last_undegraded":"2024-08-29T15:41:19.482778+0000","last_fullsized":"2024-08-29T15:41:19.476039+0000","mapping_epoch":435,"log_start":"429'1300","ondisk_log_start":"429'1300","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7985,"log_dups_size":0,"ondisk_log_size":7985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235995136,"num_objects":3601,"num_object_clones":0,"num_object_copies":10803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6842,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9285,"num_write_kb":412352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2843,"num_bytes_recovered":186318848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,1","objects":180},{"shards":"0,3,5","objects":3421}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":893,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105965+0000","last_change":"2024-08-29T15:37:19.170088+0000","last_active":"2024-08-29T15:42:42.105965+0000","last_peered":"2024-08-29T15:42:42.105965+0000","last_clean":"2024-08-29T15:42:42.105965+0000","last_became_active":"2024-08-29T15:36:54.189617+0000","last_became_peered":"2024-08-29T15:36:54.189617+0000","last_unstale":"2024-08-29T15:42:42.105965+0000","last_undegraded":"2024-08-29T15:42:42.105965+0000","last_fullsized":"2024-08-29T15:42:42.105965+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":360,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.170014+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.170014+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:55:52.939920+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":932,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.085949+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:42:42.085949+0000","last_peered":"2024-08-29T15:42:42.085949+0000","last_clean":"2024-08-29T15:42:42.085949+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:42:42.085949+0000","last_undegraded":"2024-08-29T15:42:42.085949+0000","last_fullsized":"2024-08-29T15:42:42.085949+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104375+0000","last_change":"2024-08-29T15:37:26.313495+0000","last_active":"2024-08-29T15:42:42.104375+0000","last_peered":"2024-08-29T15:42:42.104375+0000","last_clean":"2024-08-29T15:42:42.104375+0000","last_became_active":"2024-08-29T15:37:26.313188+0000","last_became_peered":"2024-08-29T15:37:26.313188+0000","last_unstale":"2024-08-29T15:42:42.104375+0000","last_undegraded":"2024-08-29T15:42:42.104375+0000","last_fullsized":"2024-08-29T15:42:42.104375+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:39.580490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":206,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.087620+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:42:42.087620+0000","last_peered":"2024-08-29T15:42:42.087620+0000","last_clean":"2024-08-29T15:42:42.087620+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:42:42.087620+0000","last_undegraded":"2024-08-29T15:42:42.087620+0000","last_fullsized":"2024-08-29T15:42:42.087620+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"453'9254","reported_seq":9332,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:55.228851+0000","last_change":"2024-08-29T15:39:37.311691+0000","last_active":"2024-08-29T15:42:55.228851+0000","last_peered":"2024-08-29T15:42:55.228851+0000","last_clean":"2024-08-29T15:42:55.228851+0000","last_became_active":"2024-08-29T15:39:37.311136+0000","last_became_peered":"2024-08-29T15:39:37.311136+0000","last_unstale":"2024-08-29T15:42:55.228851+0000","last_undegraded":"2024-08-29T15:42:55.228851+0000","last_fullsized":"2024-08-29T15:42:55.228851+0000","mapping_epoch":420,"log_start":"429'1300","ondisk_log_start":"429'1300","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7954,"log_dups_size":0,"ondisk_log_size":7954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:40:42.703845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232783872,"num_objects":3552,"num_object_clones":0,"num_object_copies":10656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9254,"num_write_kb":409792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":900,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.103310+0000","last_change":"2024-08-29T15:37:19.656177+0000","last_active":"2024-08-29T15:42:42.103310+0000","last_peered":"2024-08-29T15:42:42.103310+0000","last_clean":"2024-08-29T15:42:42.103310+0000","last_became_active":"2024-08-29T15:37:01.322473+0000","last_became_peered":"2024-08-29T15:37:01.322473+0000","last_unstale":"2024-08-29T15:42:42.103310+0000","last_undegraded":"2024-08-29T15:42:42.103310+0000","last_fullsized":"2024-08-29T15:42:42.103310+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.654978+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.654978+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:21:16.623191+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1242,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.111316+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:42:42.111316+0000","last_peered":"2024-08-29T15:42:42.111316+0000","last_clean":"2024-08-29T15:42:42.111316+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:42:42.111316+0000","last_undegraded":"2024-08-29T15:42:42.111316+0000","last_fullsized":"2024-08-29T15:42:42.111316+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":801,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.091835+0000","last_change":"2024-08-29T15:41:44.432130+0000","last_active":"2024-08-29T15:42:42.091835+0000","last_peered":"2024-08-29T15:42:42.091835+0000","last_clean":"2024-08-29T15:42:42.091835+0000","last_became_active":"2024-08-29T15:41:44.431835+0000","last_became_peered":"2024-08-29T15:41:44.431835+0000","last_unstale":"2024-08-29T15:42:42.091835+0000","last_undegraded":"2024-08-29T15:42:42.091835+0000","last_fullsized":"2024-08-29T15:42:42.091835+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:07.995950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":139,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.092106+0000","last_change":"2024-08-29T15:38:27.108492+0000","last_active":"2024-08-29T15:42:42.092106+0000","last_peered":"2024-08-29T15:42:42.092106+0000","last_clean":"2024-08-29T15:42:42.092106+0000","last_became_active":"2024-08-29T15:38:27.108046+0000","last_became_peered":"2024-08-29T15:38:27.108046+0000","last_unstale":"2024-08-29T15:42:42.092106+0000","last_undegraded":"2024-08-29T15:42:42.092106+0000","last_fullsized":"2024-08-29T15:42:42.092106+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:53:22.265093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"453'9201","reported_seq":9241,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:52.303950+0000","last_change":"2024-08-29T15:39:53.258131+0000","last_active":"2024-08-29T15:42:52.303950+0000","last_peered":"2024-08-29T15:42:52.303950+0000","last_clean":"2024-08-29T15:42:52.303950+0000","last_became_active":"2024-08-29T15:39:53.257746+0000","last_became_peered":"2024-08-29T15:39:53.257746+0000","last_unstale":"2024-08-29T15:42:52.303950+0000","last_undegraded":"2024-08-29T15:42:52.303950+0000","last_fullsized":"2024-08-29T15:42:52.303950+0000","mapping_epoch":424,"log_start":"429'1000","ondisk_log_start":"429'1000","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":8201,"log_dups_size":0,"ondisk_log_size":8201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:48.746732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237699072,"num_objects":3627,"num_object_clones":0,"num_object_copies":10881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9201,"num_write_kb":410496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":918,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.091672+0000","last_change":"2024-08-29T15:41:03.142940+0000","last_active":"2024-08-29T15:42:42.091672+0000","last_peered":"2024-08-29T15:42:42.091672+0000","last_clean":"2024-08-29T15:42:42.091672+0000","last_became_active":"2024-08-29T15:41:03.142499+0000","last_became_peered":"2024-08-29T15:41:03.142499+0000","last_unstale":"2024-08-29T15:42:42.091672+0000","last_undegraded":"2024-08-29T15:42:42.091672+0000","last_fullsized":"2024-08-29T15:42:42.091672+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:18.154828+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:18.154828+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:05.027604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":810,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104415+0000","last_change":"2024-08-29T15:37:29.345555+0000","last_active":"2024-08-29T15:42:42.104415+0000","last_peered":"2024-08-29T15:42:42.104415+0000","last_clean":"2024-08-29T15:42:42.104415+0000","last_became_active":"2024-08-29T15:37:29.344738+0000","last_became_peered":"2024-08-29T15:37:29.344738+0000","last_unstale":"2024-08-29T15:42:42.104415+0000","last_undegraded":"2024-08-29T15:42:42.104415+0000","last_fullsized":"2024-08-29T15:42:42.104415+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:14.103195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":192,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:29.618234+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:42:29.618234+0000","last_peered":"2024-08-29T15:42:29.618234+0000","last_clean":"2024-08-29T15:42:29.618234+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:42:29.618234+0000","last_undegraded":"2024-08-29T15:42:29.618234+0000","last_fullsized":"2024-08-29T15:42:29.618234+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":356,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.258004+0000","last_change":"2024-08-29T15:41:00.745651+0000","last_active":"2024-08-29T15:42:26.258004+0000","last_peered":"2024-08-29T15:42:26.258004+0000","last_clean":"2024-08-29T15:42:26.258004+0000","last_became_active":"2024-08-29T15:41:00.745279+0000","last_became_peered":"2024-08-29T15:41:00.745279+0000","last_unstale":"2024-08-29T15:42:26.258004+0000","last_undegraded":"2024-08-29T15:42:26.258004+0000","last_fullsized":"2024-08-29T15:42:26.258004+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:30:17.790638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"453'9183","reported_seq":9261,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:55.155636+0000","last_change":"2024-08-29T15:39:37.258623+0000","last_active":"2024-08-29T15:42:55.155636+0000","last_peered":"2024-08-29T15:42:55.155636+0000","last_clean":"2024-08-29T15:42:55.155636+0000","last_became_active":"2024-08-29T15:39:37.258321+0000","last_became_peered":"2024-08-29T15:39:37.258321+0000","last_unstale":"2024-08-29T15:42:55.155636+0000","last_undegraded":"2024-08-29T15:42:55.155636+0000","last_fullsized":"2024-08-29T15:42:55.155636+0000","mapping_epoch":420,"log_start":"429'1200","ondisk_log_start":"429'1200","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7983,"log_dups_size":0,"ondisk_log_size":7983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:09:19.337857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233504768,"num_objects":3563,"num_object_clones":0,"num_object_copies":10689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9183,"num_write_kb":407872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":807,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.256308+0000","last_change":"2024-08-29T15:37:23.716571+0000","last_active":"2024-08-29T15:42:26.256308+0000","last_peered":"2024-08-29T15:42:26.256308+0000","last_clean":"2024-08-29T15:42:26.256308+0000","last_became_active":"2024-08-29T15:37:01.308514+0000","last_became_peered":"2024-08-29T15:37:01.308514+0000","last_unstale":"2024-08-29T15:42:26.256308+0000","last_undegraded":"2024-08-29T15:42:26.256308+0000","last_fullsized":"2024-08-29T15:42:26.256308+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:23.716435+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:23.716435+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:18:15.213865+0000","scrub_duration":0.022267417000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":400,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:37.300620+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:42:37.300620+0000","last_peered":"2024-08-29T15:42:37.300620+0000","last_clean":"2024-08-29T15:42:37.300620+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:42:37.300620+0000","last_undegraded":"2024-08-29T15:42:37.300620+0000","last_fullsized":"2024-08-29T15:42:37.300620+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":240,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.087182+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:42:42.087182+0000","last_peered":"2024-08-29T15:42:42.087182+0000","last_clean":"2024-08-29T15:42:42.087182+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:42:42.087182+0000","last_undegraded":"2024-08-29T15:42:42.087182+0000","last_fullsized":"2024-08-29T15:42:42.087182+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":307,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.085810+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:42:42.085810+0000","last_peered":"2024-08-29T15:42:42.085810+0000","last_clean":"2024-08-29T15:42:42.085810+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:42:42.085810+0000","last_undegraded":"2024-08-29T15:42:42.085810+0000","last_fullsized":"2024-08-29T15:42:42.085810+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"453'9172","reported_seq":9215,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:52.288311+0000","last_change":"2024-08-29T15:39:37.185853+0000","last_active":"2024-08-29T15:42:52.288311+0000","last_peered":"2024-08-29T15:42:52.288311+0000","last_clean":"2024-08-29T15:42:52.288311+0000","last_became_active":"2024-08-29T15:39:37.185573+0000","last_became_peered":"2024-08-29T15:39:37.185573+0000","last_unstale":"2024-08-29T15:42:52.288311+0000","last_undegraded":"2024-08-29T15:42:52.288311+0000","last_fullsized":"2024-08-29T15:42:52.288311+0000","mapping_epoch":420,"log_start":"429'1000","ondisk_log_start":"429'1000","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:12.924373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648192,"num_objects":3672,"num_object_clones":0,"num_object_copies":11016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9172,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":853,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.257949+0000","last_change":"2024-08-29T15:41:19.243130+0000","last_active":"2024-08-29T15:42:26.257949+0000","last_peered":"2024-08-29T15:42:26.257949+0000","last_clean":"2024-08-29T15:42:26.257949+0000","last_became_active":"2024-08-29T15:41:19.242679+0000","last_became_peered":"2024-08-29T15:41:19.242679+0000","last_unstale":"2024-08-29T15:42:26.257949+0000","last_undegraded":"2024-08-29T15:42:26.257949+0000","last_fullsized":"2024-08-29T15:42:26.257949+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:16.688893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:16.688893+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:50.927792+0000","scrub_duration":0.016432537000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":332,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.086131+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:42:42.086131+0000","last_peered":"2024-08-29T15:42:42.086131+0000","last_clean":"2024-08-29T15:42:42.086131+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:42:42.086131+0000","last_undegraded":"2024-08-29T15:42:42.086131+0000","last_fullsized":"2024-08-29T15:42:42.086131+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1206,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105699+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:42:42.105699+0000","last_peered":"2024-08-29T15:42:42.105699+0000","last_clean":"2024-08-29T15:42:42.105699+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:42:42.105699+0000","last_undegraded":"2024-08-29T15:42:42.105699+0000","last_fullsized":"2024-08-29T15:42:42.105699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":84,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.257200+0000","last_change":"2024-08-29T15:41:16.222377+0000","last_active":"2024-08-29T15:42:26.257200+0000","last_peered":"2024-08-29T15:42:26.257200+0000","last_clean":"2024-08-29T15:42:26.257200+0000","last_became_active":"2024-08-29T15:41:16.221775+0000","last_became_peered":"2024-08-29T15:41:16.221775+0000","last_unstale":"2024-08-29T15:42:26.257200+0000","last_undegraded":"2024-08-29T15:42:26.257200+0000","last_fullsized":"2024-08-29T15:42:26.257200+0000","mapping_epoch":439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:43:52.584991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"453'9157","reported_seq":9199,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:54.685594+0000","last_change":"2024-08-29T15:39:37.258863+0000","last_active":"2024-08-29T15:42:54.685594+0000","last_peered":"2024-08-29T15:42:54.685594+0000","last_clean":"2024-08-29T15:42:54.685594+0000","last_became_active":"2024-08-29T15:39:37.258296+0000","last_became_peered":"2024-08-29T15:39:37.258296+0000","last_unstale":"2024-08-29T15:42:54.685594+0000","last_undegraded":"2024-08-29T15:42:54.685594+0000","last_fullsized":"2024-08-29T15:42:54.685594+0000","mapping_epoch":420,"log_start":"429'1400","ondisk_log_start":"429'1400","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7757,"log_dups_size":0,"ondisk_log_size":7757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:32:01.990004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234815488,"num_objects":3583,"num_object_clones":0,"num_object_copies":10749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9157,"num_write_kb":407680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":912,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.092157+0000","last_change":"2024-08-29T15:37:40.133670+0000","last_active":"2024-08-29T15:42:42.092157+0000","last_peered":"2024-08-29T15:42:42.092157+0000","last_clean":"2024-08-29T15:42:42.092157+0000","last_became_active":"2024-08-29T15:37:40.132923+0000","last_became_peered":"2024-08-29T15:37:40.132923+0000","last_unstale":"2024-08-29T15:42:42.092157+0000","last_undegraded":"2024-08-29T15:42:42.092157+0000","last_fullsized":"2024-08-29T15:42:42.092157+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:15.111260+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:15.111260+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:42:27.886225+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":788,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105595+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:42:42.105595+0000","last_peered":"2024-08-29T15:42:42.105595+0000","last_clean":"2024-08-29T15:42:42.105595+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:42:42.105595+0000","last_undegraded":"2024-08-29T15:42:42.105595+0000","last_fullsized":"2024-08-29T15:42:42.105595+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":912,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:29.618259+0000","last_change":"2024-08-29T15:28:39.046816+0000","last_active":"2024-08-29T15:42:29.618259+0000","last_peered":"2024-08-29T15:42:29.618259+0000","last_clean":"2024-08-29T15:42:29.618259+0000","last_became_active":"2024-08-29T15:19:46.805425+0000","last_became_peered":"2024-08-29T15:19:46.805425+0000","last_unstale":"2024-08-29T15:42:29.618259+0000","last_undegraded":"2024-08-29T15:42:29.618259+0000","last_fullsized":"2024-08-29T15:42:29.618259+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:24:29.899045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":780,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.094456+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:42:42.094456+0000","last_peered":"2024-08-29T15:42:42.094456+0000","last_clean":"2024-08-29T15:42:42.094456+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:42:42.094456+0000","last_undegraded":"2024-08-29T15:42:42.094456+0000","last_fullsized":"2024-08-29T15:42:42.094456+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"453'9208","reported_seq":21828,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:52.314833+0000","last_change":"2024-08-29T15:42:20.219072+0000","last_active":"2024-08-29T15:42:52.314833+0000","last_peered":"2024-08-29T15:42:52.314833+0000","last_clean":"2024-08-29T15:42:52.314833+0000","last_became_active":"2024-08-29T15:42:20.200986+0000","last_became_peered":"2024-08-29T15:42:20.200986+0000","last_unstale":"2024-08-29T15:42:52.314833+0000","last_undegraded":"2024-08-29T15:42:52.314833+0000","last_fullsized":"2024-08-29T15:42:52.314833+0000","mapping_epoch":449,"log_start":"429'1000","ondisk_log_start":"429'1000","created":420,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":8208,"log_dups_size":0,"ondisk_log_size":8208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:18.829519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241238040,"num_objects":3682,"num_object_clones":0,"num_object_copies":11046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3682,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9208,"num_write_kb":412417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6273,"num_bytes_recovered":411107328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":824,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.268179+0000","last_change":"2024-08-29T15:39:14.809022+0000","last_active":"2024-08-29T15:42:26.268179+0000","last_peered":"2024-08-29T15:42:26.268179+0000","last_clean":"2024-08-29T15:42:26.268179+0000","last_became_active":"2024-08-29T15:39:14.808645+0000","last_became_peered":"2024-08-29T15:39:14.808645+0000","last_unstale":"2024-08-29T15:42:26.268179+0000","last_undegraded":"2024-08-29T15:42:26.268179+0000","last_fullsized":"2024-08-29T15:42:26.268179+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:19.644669+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:19.644669+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:51.600506+0000","scrub_duration":0.0030306299999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":773,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.106005+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:42:42.106005+0000","last_peered":"2024-08-29T15:42:42.106005+0000","last_clean":"2024-08-29T15:42:42.106005+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:42:42.106005+0000","last_undegraded":"2024-08-29T15:42:42.106005+0000","last_fullsized":"2024-08-29T15:42:42.106005+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1425,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.103786+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:42:42.103786+0000","last_peered":"2024-08-29T15:42:42.103786+0000","last_clean":"2024-08-29T15:42:42.103786+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:42:42.103786+0000","last_undegraded":"2024-08-29T15:42:42.103786+0000","last_fullsized":"2024-08-29T15:42:42.103786+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.103401+0000","last_change":"2024-08-29T15:41:44.452195+0000","last_active":"2024-08-29T15:42:42.103401+0000","last_peered":"2024-08-29T15:42:42.103401+0000","last_clean":"2024-08-29T15:42:42.103401+0000","last_became_active":"2024-08-29T15:41:44.451619+0000","last_became_peered":"2024-08-29T15:41:44.451619+0000","last_unstale":"2024-08-29T15:42:42.103401+0000","last_undegraded":"2024-08-29T15:42:42.103401+0000","last_fullsized":"2024-08-29T15:42:42.103401+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:31:24.464258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"453'9461","reported_seq":9539,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:56.624158+0000","last_change":"2024-08-29T15:39:37.259666+0000","last_active":"2024-08-29T15:42:56.624158+0000","last_peered":"2024-08-29T15:42:56.624158+0000","last_clean":"2024-08-29T15:42:56.624158+0000","last_became_active":"2024-08-29T15:39:37.259380+0000","last_became_peered":"2024-08-29T15:39:37.259380+0000","last_unstale":"2024-08-29T15:42:56.624158+0000","last_undegraded":"2024-08-29T15:42:56.624158+0000","last_fullsized":"2024-08-29T15:42:56.624158+0000","mapping_epoch":420,"log_start":"430'2102","ondisk_log_start":"430'2102","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7359,"log_dups_size":0,"ondisk_log_size":7359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:02:18.132955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232718336,"num_objects":3551,"num_object_clones":0,"num_object_copies":10653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9461,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":932,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104352+0000","last_change":"2024-08-29T15:39:56.274024+0000","last_active":"2024-08-29T15:42:42.104352+0000","last_peered":"2024-08-29T15:42:42.104352+0000","last_clean":"2024-08-29T15:42:42.104352+0000","last_became_active":"2024-08-29T15:39:56.273578+0000","last_became_peered":"2024-08-29T15:39:56.273578+0000","last_unstale":"2024-08-29T15:42:42.104352+0000","last_undegraded":"2024-08-29T15:42:42.104352+0000","last_fullsized":"2024-08-29T15:42:42.104352+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:20.077258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:20.077258+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:34:20.595923+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":185,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.092247+0000","last_change":"2024-08-29T15:37:26.328428+0000","last_active":"2024-08-29T15:42:42.092247+0000","last_peered":"2024-08-29T15:42:42.092247+0000","last_clean":"2024-08-29T15:42:42.092247+0000","last_became_active":"2024-08-29T15:37:26.313757+0000","last_became_peered":"2024-08-29T15:37:26.313757+0000","last_unstale":"2024-08-29T15:42:42.092247+0000","last_undegraded":"2024-08-29T15:42:42.092247+0000","last_fullsized":"2024-08-29T15:42:42.092247+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:29:31.366546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":64,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105652+0000","last_change":"2024-08-29T15:41:00.761163+0000","last_active":"2024-08-29T15:42:42.105652+0000","last_peered":"2024-08-29T15:42:42.105652+0000","last_clean":"2024-08-29T15:42:42.105652+0000","last_became_active":"2024-08-29T15:41:00.749422+0000","last_became_peered":"2024-08-29T15:41:00.749422+0000","last_unstale":"2024-08-29T15:42:42.105652+0000","last_undegraded":"2024-08-29T15:42:42.105652+0000","last_fullsized":"2024-08-29T15:42:42.105652+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:21.012113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":240,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.087058+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:42:42.087058+0000","last_peered":"2024-08-29T15:42:42.087058+0000","last_clean":"2024-08-29T15:42:42.087058+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:42:42.087058+0000","last_undegraded":"2024-08-29T15:42:42.087058+0000","last_fullsized":"2024-08-29T15:42:42.087058+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":238,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.268083+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:42:26.268083+0000","last_peered":"2024-08-29T15:42:26.268083+0000","last_clean":"2024-08-29T15:42:26.268083+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:42:26.268083+0000","last_undegraded":"2024-08-29T15:42:26.268083+0000","last_fullsized":"2024-08-29T15:42:26.268083+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":248,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.268082+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:42:26.268082+0000","last_peered":"2024-08-29T15:42:26.268082+0000","last_clean":"2024-08-29T15:42:26.268082+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:42:26.268082+0000","last_undegraded":"2024-08-29T15:42:26.268082+0000","last_fullsized":"2024-08-29T15:42:26.268082+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.091569+0000","last_change":"2024-08-29T15:41:00.747464+0000","last_active":"2024-08-29T15:42:42.091569+0000","last_peered":"2024-08-29T15:42:42.091569+0000","last_clean":"2024-08-29T15:42:42.091569+0000","last_became_active":"2024-08-29T15:41:00.747118+0000","last_became_peered":"2024-08-29T15:41:00.747118+0000","last_unstale":"2024-08-29T15:42:42.091569+0000","last_undegraded":"2024-08-29T15:42:42.091569+0000","last_fullsized":"2024-08-29T15:42:42.091569+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:39:50.063552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":536,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105953+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:42:42.105953+0000","last_peered":"2024-08-29T15:42:42.105953+0000","last_clean":"2024-08-29T15:42:42.105953+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:42:42.105953+0000","last_undegraded":"2024-08-29T15:42:42.105953+0000","last_fullsized":"2024-08-29T15:42:42.105953+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":328,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.257961+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:42:26.257961+0000","last_peered":"2024-08-29T15:42:26.257961+0000","last_clean":"2024-08-29T15:42:26.257961+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:42:26.257961+0000","last_undegraded":"2024-08-29T15:42:26.257961+0000","last_fullsized":"2024-08-29T15:42:26.257961+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":736,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104274+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:42:42.104274+0000","last_peered":"2024-08-29T15:42:42.104274+0000","last_clean":"2024-08-29T15:42:42.104274+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:42:42.104274+0000","last_undegraded":"2024-08-29T15:42:42.104274+0000","last_fullsized":"2024-08-29T15:42:42.104274+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":789,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.092589+0000","last_change":"2024-08-29T15:17:40.713746+0000","last_active":"2024-08-29T15:42:42.092589+0000","last_peered":"2024-08-29T15:42:42.092589+0000","last_clean":"2024-08-29T15:42:42.092589+0000","last_became_active":"2024-08-29T15:17:40.711638+0000","last_became_peered":"2024-08-29T15:17:40.711638+0000","last_unstale":"2024-08-29T15:42:42.092589+0000","last_undegraded":"2024-08-29T15:42:42.092589+0000","last_fullsized":"2024-08-29T15:42:42.092589+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:07.145766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":788,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104054+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:42:42.104054+0000","last_peered":"2024-08-29T15:42:42.104054+0000","last_clean":"2024-08-29T15:42:42.104054+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:42:42.104054+0000","last_undegraded":"2024-08-29T15:42:42.104054+0000","last_fullsized":"2024-08-29T15:42:42.104054+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":310,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.087201+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:42:42.087201+0000","last_peered":"2024-08-29T15:42:42.087201+0000","last_clean":"2024-08-29T15:42:42.087201+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:42:42.087201+0000","last_undegraded":"2024-08-29T15:42:42.087201+0000","last_fullsized":"2024-08-29T15:42:42.087201+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":26,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.103303+0000","last_change":"2024-08-29T15:41:44.451915+0000","last_active":"2024-08-29T15:42:42.103303+0000","last_peered":"2024-08-29T15:42:42.103303+0000","last_clean":"2024-08-29T15:42:42.103303+0000","last_became_active":"2024-08-29T15:41:44.451203+0000","last_became_peered":"2024-08-29T15:41:44.451203+0000","last_unstale":"2024-08-29T15:42:42.103303+0000","last_undegraded":"2024-08-29T15:42:42.103303+0000","last_fullsized":"2024-08-29T15:42:42.103303+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:05:30.128036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":794,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.092354+0000","last_change":"2024-08-29T15:41:16.224398+0000","last_active":"2024-08-29T15:42:42.092354+0000","last_peered":"2024-08-29T15:42:42.092354+0000","last_clean":"2024-08-29T15:42:42.092354+0000","last_became_active":"2024-08-29T15:41:16.223643+0000","last_became_peered":"2024-08-29T15:41:16.223643+0000","last_unstale":"2024-08-29T15:42:42.092354+0000","last_undegraded":"2024-08-29T15:42:42.092354+0000","last_fullsized":"2024-08-29T15:42:42.092354+0000","mapping_epoch":439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:59:32.406120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":768,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.103050+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:42:42.103050+0000","last_peered":"2024-08-29T15:42:42.103050+0000","last_clean":"2024-08-29T15:42:42.103050+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:42:42.103050+0000","last_undegraded":"2024-08-29T15:42:42.103050+0000","last_fullsized":"2024-08-29T15:42:42.103050+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":772,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105404+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:42:42.105404+0000","last_peered":"2024-08-29T15:42:42.105404+0000","last_clean":"2024-08-29T15:42:42.105404+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:42:42.105404+0000","last_undegraded":"2024-08-29T15:42:42.105404+0000","last_fullsized":"2024-08-29T15:42:42.105404+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":238,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267816+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:42:26.267816+0000","last_peered":"2024-08-29T15:42:26.267816+0000","last_clean":"2024-08-29T15:42:26.267816+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:42:26.267816+0000","last_undegraded":"2024-08-29T15:42:26.267816+0000","last_fullsized":"2024-08-29T15:42:26.267816+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":813,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105800+0000","last_change":"2024-08-29T15:39:11.832387+0000","last_active":"2024-08-29T15:42:42.105800+0000","last_peered":"2024-08-29T15:42:42.105800+0000","last_clean":"2024-08-29T15:42:42.105800+0000","last_became_active":"2024-08-29T15:39:11.832056+0000","last_became_peered":"2024-08-29T15:39:11.832056+0000","last_unstale":"2024-08-29T15:42:42.105800+0000","last_undegraded":"2024-08-29T15:42:42.105800+0000","last_fullsized":"2024-08-29T15:42:42.105800+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:47:42.049015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":138,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.094171+0000","last_change":"2024-08-29T15:38:27.106157+0000","last_active":"2024-08-29T15:42:42.094171+0000","last_peered":"2024-08-29T15:42:42.094171+0000","last_clean":"2024-08-29T15:42:42.094171+0000","last_became_active":"2024-08-29T15:38:27.105635+0000","last_became_peered":"2024-08-29T15:38:27.105635+0000","last_unstale":"2024-08-29T15:42:42.094171+0000","last_undegraded":"2024-08-29T15:42:42.094171+0000","last_fullsized":"2024-08-29T15:42:42.094171+0000","mapping_epoch":389,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:27.182762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":265,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267889+0000","last_change":"2024-08-29T15:41:44.446288+0000","last_active":"2024-08-29T15:42:26.267889+0000","last_peered":"2024-08-29T15:42:26.267889+0000","last_clean":"2024-08-29T15:42:26.267889+0000","last_became_active":"2024-08-29T15:41:44.445958+0000","last_became_peered":"2024-08-29T15:41:44.445958+0000","last_unstale":"2024-08-29T15:42:26.267889+0000","last_undegraded":"2024-08-29T15:42:26.267889+0000","last_fullsized":"2024-08-29T15:42:26.267889+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:47:53.474043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":371,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.085997+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:42:42.085997+0000","last_peered":"2024-08-29T15:42:42.085997+0000","last_clean":"2024-08-29T15:42:42.085997+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:42:42.085997+0000","last_undegraded":"2024-08-29T15:42:42.085997+0000","last_fullsized":"2024-08-29T15:42:42.085997+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":238,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267842+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:42:26.267842+0000","last_peered":"2024-08-29T15:42:26.267842+0000","last_clean":"2024-08-29T15:42:26.267842+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:42:26.267842+0000","last_undegraded":"2024-08-29T15:42:26.267842+0000","last_fullsized":"2024-08-29T15:42:26.267842+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":371,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.086342+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:42:42.086342+0000","last_peered":"2024-08-29T15:42:42.086342+0000","last_clean":"2024-08-29T15:42:42.086342+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:42:42.086342+0000","last_undegraded":"2024-08-29T15:42:42.086342+0000","last_fullsized":"2024-08-29T15:42:42.086342+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":804,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.104540+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:42:42.104540+0000","last_peered":"2024-08-29T15:42:42.104540+0000","last_clean":"2024-08-29T15:42:42.104540+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:42:42.104540+0000","last_undegraded":"2024-08-29T15:42:42.104540+0000","last_fullsized":"2024-08-29T15:42:42.104540+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":806,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.105769+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:42:42.105769+0000","last_peered":"2024-08-29T15:42:42.105769+0000","last_clean":"2024-08-29T15:42:42.105769+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:42:42.105769+0000","last_undegraded":"2024-08-29T15:42:42.105769+0000","last_fullsized":"2024-08-29T15:42:42.105769+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":238,"reported_epoch":452,"state":"active+clean","last_fresh":"2024-08-29T15:42:26.267553+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:42:26.267553+0000","last_peered":"2024-08-29T15:42:26.267553+0000","last_clean":"2024-08-29T15:42:26.267553+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:42:26.267553+0000","last_undegraded":"2024-08-29T15:42:26.267553+0000","last_fullsized":"2024-08-29T15:42:26.267553+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":158,"reported_epoch":453,"state":"active+clean","last_fresh":"2024-08-29T15:42:42.092448+0000","last_change":"2024-08-29T15:39:11.853131+0000","last_active":"2024-08-29T15:42:42.092448+0000","last_peered":"2024-08-29T15:42:42.092448+0000","last_clean":"2024-08-29T15:42:42.092448+0000","last_became_active":"2024-08-29T15:39:11.836033+0000","last_became_peered":"2024-08-29T15:39:11.836033+0000","last_unstale":"2024-08-29T15:42:42.092448+0000","last_undegraded":"2024-08-29T15:42:42.092448+0000","last_fullsized":"2024-08-29T15:42:42.092448+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:51.036468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3744661528,"num_objects":57140,"num_object_clones":0,"num_object_copies":171420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17050,"num_objects_misplaced":11382,"num_objects_unfound":0,"num_objects_dirty":57140,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147716,"num_write_kb":6555329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16700,"num_bytes_recovered":1094385664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12821278720,"data_stored":12821266504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124817,"ondisk_log_size":124817,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":7736,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234579,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3194900,"kb_used_data":2441868,"kb_used_omap":1,"kb_used_meta":753022,"kb_avail":90550252,"statfs":{"total":95995035648,"available":92723458048,"internally_reserved":0,"allocated":2500472832,"data_stored":2497633793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":771094987},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496210,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1221836,"kb_used_data":691140,"kb_used_omap":1,"kb_used_meta":530686,"kb_avail":92523316,"statfs":{"total":95995035648,"available":94743875584,"internally_reserved":0,"allocated":707727360,"data_stored":704896703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":543422919},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426796,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2787400,"kb_used_data":2295040,"kb_used_omap":8,"kb_used_meta":492343,"kb_avail":90957752,"statfs":{"total":95995035648,"available":93140738048,"internally_reserved":0,"allocated":2350120960,"data_stored":2347290191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":504160138},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019489,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1639120,"kb_used_data":932040,"kb_used_omap":1,"kb_used_meta":707070,"kb_avail":92106032,"statfs":{"total":95995035648,"available":94316576768,"internally_reserved":0,"allocated":954408960,"data_stored":951573744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":724040134},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435674,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3139840,"kb_used_data":2305784,"kb_used_omap":4,"kb_used_meta":834043,"kb_avail":90605312,"statfs":{"total":95995035648,"available":92779839488,"internally_reserved":0,"allocated":2361122816,"data_stored":2358299977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":854060297},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903525,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285820,"kb_used_data":707060,"kb_used_omap":10,"kb_used_meta":578741,"kb_avail":92459332,"statfs":{"total":95995035648,"available":94678355968,"internally_reserved":0,"allocated":724029440,"data_stored":721210043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":592631312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762727,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3185784,"kb_used_data":2337776,"kb_used_omap":7,"kb_used_meta":847992,"kb_avail":90559368,"statfs":{"total":95995035648,"available":92732792832,"internally_reserved":0,"allocated":2393882624,"data_stored":2391066217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":868344162},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,1,3,2,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926250,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1620216,"kb_used_data":930480,"kb_used_omap":7,"kb_used_meta":689720,"kb_avail":92124936,"statfs":{"total":95995035648,"available":94335934464,"internally_reserved":0,"allocated":952811520,"data_stored":949995402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":706273423},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,0,0,1,0,1,0,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":938803200,"data_stored":938803200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2379612160,"data_stored":2379612160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":714735616,"data_stored":714735616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2342912000,"data_stored":2342912000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":935268352,"data_stored":935264280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2331512832,"data_stored":2331508760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":695664640,"data_stored":695664640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2482769920,"data_stored":2482765848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:42:59.069 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-29T15:42:59.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool set .mgr pg_num 21 2024-08-29T15:42:59.071 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:59.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:59.173 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:42:59.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:59.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:42:59.221 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:42:59.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:42:59.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[93921]: from='client.56590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:42:59.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:59.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2073759718' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:42:59.321 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:42:59.322 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:42:59] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-29T15:42:59.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[98970]: from='client.56590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:42:59.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:59.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2073759718' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:42:59.322 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:42:59 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:42:59.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:59.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:42:59.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:59.475 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:59.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:59.479 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:42:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:59 smithi171 ceph-mon[90031]: from='client.56590 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:42:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:42:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2073759718' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:42:59.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:42:59 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:42:59.577 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:42:59.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:59.600 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:59.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:42:59.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:42:59.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:42:59.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:59.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:42:59.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:42:59.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:42:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:42:59.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:42:59.986 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:42:59.990 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:00.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:00.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:00.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:00.240 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[93921]: pgmap v1611: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 16706/168813 objects degraded (9.896%); 11382/168813 objects misplaced (6.742%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:43:00.240 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:00.240 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:43:00.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[93921]: osdmap e454: 8 total, 8 up, 8 in 2024-08-29T15:43:00.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[98970]: pgmap v1611: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 16706/168813 objects degraded (9.896%); 11382/168813 objects misplaced (6.742%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:43:00.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:00.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:43:00.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:00 smithi027 ceph-mon[98970]: osdmap e454: 8 total, 8 up, 8 in 2024-08-29T15:43:00.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:00.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:00.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:00.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:00.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:00 smithi171 ceph-mon[90031]: pgmap v1611: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 16706/168813 objects degraded (9.896%); 11382/168813 objects misplaced (6.742%); 7.9 MiB/s, 126 objects/s recovering 2024-08-29T15:43:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:00 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:00 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:43:00.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:00 smithi171 ceph-mon[90031]: osdmap e454: 8 total, 8 up, 8 in 2024-08-29T15:43:00.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:00.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:00.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:00.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.731 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:00.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:00.832 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:00.857 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:00.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:00.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:00.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:00.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:00.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:01.096 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:01.121 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:01.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:01.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:01.254 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:01.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:01.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:01.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:01.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:01.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:01.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:01.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:01.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:01.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:01.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:01.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:01.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:01.785 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:01.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:01.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:01.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:01.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:01.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:01.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:43:02.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:02.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:02.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:02.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:02.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:02.196 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:02.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:02.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:02.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:02 smithi027 ceph-mon[93921]: pgmap v1613: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16438/164775 objects degraded (9.976%); 11001/164775 objects misplaced (6.676%); 8.2 MiB/s, 131 objects/s recovering 2024-08-29T15:43:02.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:02 smithi027 ceph-mon[98970]: pgmap v1613: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16438/164775 objects degraded (9.976%); 11001/164775 objects misplaced (6.676%); 8.2 MiB/s, 131 objects/s recovering 2024-08-29T15:43:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:02.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:02.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:02.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:02.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:02.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:02.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:02 smithi171 ceph-mon[90031]: pgmap v1613: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16438/164775 objects degraded (9.976%); 11001/164775 objects misplaced (6.676%); 8.2 MiB/s, 131 objects/s recovering 2024-08-29T15:43:02.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:02.586 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:02.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:02.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:02.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:02.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:02.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:02.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:02.880 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:02.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:02.982 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:03.008 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:03.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:03.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:03.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:03.175 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:03.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:03.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:03.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:03.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:03.393 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:03 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 16438/164775 objects degraded (9.976%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:03.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:03 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 16438/164775 objects degraded (9.976%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:03.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:03.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:03.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:03.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:03 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 16438/164775 objects degraded (9.976%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:03.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:03.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:03.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:03.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:03.728 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:03.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:03.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:03.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:03.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:03.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:03.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:04.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:04.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:04.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:04.156 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:04.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:04 smithi027 ceph-mon[98970]: pgmap v1614: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16092/160794 objects degraded (10.008%); 11001/160794 objects misplaced (6.842%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:43:04.196 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3931368932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:04.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:04 smithi027 ceph-mon[93921]: pgmap v1614: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16092/160794 objects degraded (10.008%); 11001/160794 objects misplaced (6.842%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:43:04.196 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3931368932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:04.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:04.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:04.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:04.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:04.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:04.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:04 smithi171 ceph-mon[90031]: pgmap v1614: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16092/160794 objects degraded (10.008%); 11001/160794 objects misplaced (6.842%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:43:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3931368932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:04.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:04.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:04.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:04.726 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:04.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:04.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:04.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:04.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:04.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:04.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:05.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:05.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:05.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:05.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:05.192 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:43:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:43:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:43:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:43:05.193 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:43:05.228 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:05.253 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:05.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:05.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:05.393 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:05.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:05.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:05.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:05.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:05.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:05.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:05.654 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:05.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:05.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:05.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:05.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:05.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:05.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:05.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:05.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:06.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:06.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:06.070 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:06.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:06.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:06.212 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:06.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:06.313 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:06.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:06.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:06.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:06.438 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:06.472 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:06.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:06 smithi171 ceph-mon[90031]: pgmap v1615: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16092/160287 objects degraded (10.039%); 11001/160287 objects misplaced (6.863%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:43:06.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:06 smithi027 ceph-mon[93921]: pgmap v1615: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16092/160287 objects degraded (10.039%); 11001/160287 objects misplaced (6.863%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:43:06.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:06 smithi027 ceph-mon[98970]: pgmap v1615: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16092/160287 objects degraded (10.039%); 11001/160287 objects misplaced (6.863%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:43:06.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:06.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:06.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:06.718 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:06.743 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:06.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:06.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:06.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:06.875 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:06.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:06.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:07.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:07.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:07.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:07.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:07.280 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:07.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:07.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:07.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:07.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.558 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:07.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:07.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:07.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:07.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:07.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:07.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:07.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:07.995 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:07.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:08.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:08.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:08.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:08.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:08.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:08.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:08.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:08.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:08.412 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:08 smithi171 ceph-mon[90031]: pgmap v1616: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 15802/155712 objects degraded (10.148%); 10581/155712 objects misplaced (6.795%); 8.6 MiB/s, 136 objects/s recovering 2024-08-29T15:43:08.412 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:08 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 15802/155712 objects degraded (10.148%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:08.420 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:08.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:08.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:08.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:08.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:08.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:08.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:08 smithi027 ceph-mon[93921]: pgmap v1616: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 15802/155712 objects degraded (10.148%); 10581/155712 objects misplaced (6.795%); 8.6 MiB/s, 136 objects/s recovering 2024-08-29T15:43:08.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:08 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 15802/155712 objects degraded (10.148%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:08.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:08 smithi027 ceph-mon[98970]: pgmap v1616: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 15802/155712 objects degraded (10.148%); 10581/155712 objects misplaced (6.795%); 8.6 MiB/s, 136 objects/s recovering 2024-08-29T15:43:08.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:08 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 15802/155712 objects degraded (10.148%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:08.655 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:08.682 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:08.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:08.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:08.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:08.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:08.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:08.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:08.920 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:08.961 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:08.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:09.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:09.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:09.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:09.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:09.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:09.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:43:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:43:09] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-29T15:43:09.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:09.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:09.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:09.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:09.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:09.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:09.607 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:09.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:09.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:09.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:09.784 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:09.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:09.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:09.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:09.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:09.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:10.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:10.062 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:10.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:10.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:10.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:10.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:10.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:10.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:10.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:10.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:10.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:10.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:10 smithi171 ceph-mon[90031]: pgmap v1617: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 15506/153375 objects degraded (10.110%); 10581/153375 objects misplaced (6.899%); 8.4 MiB/s, 134 objects/s recovering 2024-08-29T15:43:10.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:10 smithi027 ceph-mon[93921]: pgmap v1617: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 15506/153375 objects degraded (10.110%); 10581/153375 objects misplaced (6.899%); 8.4 MiB/s, 134 objects/s recovering 2024-08-29T15:43:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:10 smithi027 ceph-mon[98970]: pgmap v1617: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 15506/153375 objects degraded (10.110%); 10581/153375 objects misplaced (6.899%); 8.4 MiB/s, 134 objects/s recovering 2024-08-29T15:43:10.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:10.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:10.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:10.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:10.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:10.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:10.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:10.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:10.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.011 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:11.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:11.066 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:43:11.066 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:43:11.066 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:43:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:43:11.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:11.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:11.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:11.266 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:11.368 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:11.399 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:11.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:11.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:11.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:11.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:11.664 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:11.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:11.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:11.823 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:11.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:11.952 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:11.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:11.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:12.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:12.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:12.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:12.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:12.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:12.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:12.347 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:12.374 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:12.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:12.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:12.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:43:12.433 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:12 smithi027 ceph-mon[93921]: pgmap v1618: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 15156/150732 objects degraded (10.055%); 10327/150732 objects misplaced (6.851%); 9.3 MiB/s, 149 objects/s recovering 2024-08-29T15:43:12.434 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:12 smithi027 ceph-mon[98970]: pgmap v1618: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 15156/150732 objects degraded (10.055%); 10327/150732 objects misplaced (6.851%); 9.3 MiB/s, 149 objects/s recovering 2024-08-29T15:43:12.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:12 smithi171 ceph-mon[90031]: pgmap v1618: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 506 op/s; 15156/150732 objects degraded (10.055%); 10327/150732 objects misplaced (6.851%); 9.3 MiB/s, 149 objects/s recovering 2024-08-29T15:43:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:12.520 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:12.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:12.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:12.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:12.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:12.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:12.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:12.812 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:12.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:12.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:12.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:12.949 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:12.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:13.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:13.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:13.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:13.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:13.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:13.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:13.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:13.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:13.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:13.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:13.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:13.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:13.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:13.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:13.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:13 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 15156/150732 objects degraded (10.055%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:13.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:13 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 15156/150732 objects degraded (10.055%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:13.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:13 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 15156/150732 objects degraded (10.055%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:13.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:13.634 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:13.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:13.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:13.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:13.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:13.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:13.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:13.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:13.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:13.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:13.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:14.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:14.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:14.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:14.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:14.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:14.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:14.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:14.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:14.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:14.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:14.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:14 smithi027 ceph-mon[93921]: pgmap v1619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 14821/144651 objects degraded (10.246%); 10166/144651 objects misplaced (7.028%); 8.3 MiB/s, 133 objects/s recovering 2024-08-29T15:43:14.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:14.505 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:14 smithi027 ceph-mon[98970]: pgmap v1619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 14821/144651 objects degraded (10.246%); 10166/144651 objects misplaced (7.028%); 8.3 MiB/s, 133 objects/s recovering 2024-08-29T15:43:14.505 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:14 smithi171 ceph-mon[90031]: pgmap v1619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 14821/144651 objects degraded (10.246%); 10166/144651 objects misplaced (7.028%); 8.3 MiB/s, 133 objects/s recovering 2024-08-29T15:43:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:14.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:14.584 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:14.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:14.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:14.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:14.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:14.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:14.975 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:14.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:15.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:15.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:15.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:15.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:15.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:43:15.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:15.287 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:15.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:15.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:15.389 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:15.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:15.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:15.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:15.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:15.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:15.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:15.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:15.714 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:15.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:15.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:15.855 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:15.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:15.957 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:15.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:15.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:15.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:16.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:16.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:16.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:16.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:16.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:16.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:16.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:16.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:16.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:16.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:16.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:16 smithi171 ceph-mon[90031]: pgmap v1620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 14821/144201 objects degraded (10.278%); 10166/144201 objects misplaced (7.050%); 7.4 MiB/s, 118 objects/s recovering 2024-08-29T15:43:16.549 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:16 smithi027 ceph-mon[93921]: pgmap v1620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 14821/144201 objects degraded (10.278%); 10166/144201 objects misplaced (7.050%); 7.4 MiB/s, 118 objects/s recovering 2024-08-29T15:43:16.549 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:16 smithi027 ceph-mon[98970]: pgmap v1620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 460 op/s; 14821/144201 objects degraded (10.278%); 10166/144201 objects misplaced (7.050%); 7.4 MiB/s, 118 objects/s recovering 2024-08-29T15:43:16.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:16.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:16.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:16.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:16.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:16.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:16.777 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:16.815 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:16.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:16.917 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:16.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:16.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:16.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:17.009 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:17.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:17.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:17.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:17.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:17.196 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:43:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:17.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:17.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:17.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:17 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4271733648' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:17.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:17 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:17.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 14821/144201 objects degraded (10.278%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:17.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:17 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4271733648' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:17.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:17 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:17.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 14821/144201 objects degraded (10.278%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:17.337 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:17.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:17.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:17.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:17.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:17.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:17.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:17 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4271733648' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:17 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:17.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 14821/144201 objects degraded (10.278%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:17.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:17.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:17.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:17.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:17.764 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:17.789 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:17.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:17.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:17.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:17.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:17.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:17.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:18.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:18.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:18.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:18.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:18.225 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:18.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:18.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:18.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:18.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:18.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:18.468 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:18.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[93921]: pgmap v1621: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 14482/138723 objects degraded (10.440%); 9713/138723 objects misplaced (7.002%); 11 MiB/s, 168 objects/s recovering 2024-08-29T15:43:18.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:43:18.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:43:18.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[93921]: osdmap e455: 8 total, 8 up, 8 in 2024-08-29T15:43:18.476 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[98970]: pgmap v1621: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 14482/138723 objects degraded (10.440%); 9713/138723 objects misplaced (7.002%); 11 MiB/s, 168 objects/s recovering 2024-08-29T15:43:18.477 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:43:18.477 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:43:18.477 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:18 smithi027 ceph-mon[98970]: osdmap e455: 8 total, 8 up, 8 in 2024-08-29T15:43:18.491 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:18.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:18 smithi171 ceph-mon[90031]: pgmap v1621: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 14482/138723 objects degraded (10.440%); 9713/138723 objects misplaced (7.002%); 11 MiB/s, 168 objects/s recovering 2024-08-29T15:43:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:18 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:43:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:18 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:43:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:18 smithi171 ceph-mon[90031]: osdmap e455: 8 total, 8 up, 8 in 2024-08-29T15:43:18.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:18.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:18.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:18.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:18.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:18.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:18.877 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:18.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:18.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:19.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:19.044 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:19.146 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:19.175 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:19.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:19.240 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:43:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:43:19] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.51.0" 2024-08-29T15:43:19.277 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:19.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:19.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:19.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.455 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:19.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:19.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:19.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:19.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:19.699 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:19.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:19.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:19.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:19.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:19.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:19.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:19.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:20.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:20.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:20.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:20.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:20.282 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:20.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:20.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:20.407 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:20.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:20.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:20 smithi171 ceph-mon[90031]: pgmap v1623: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 14166/136101 objects degraded (10.408%); 9713/136101 objects misplaced (7.137%); 9.4 MiB/s, 149 objects/s recovering 2024-08-29T15:43:20.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:20.543 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:20.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:20.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:20 smithi027 ceph-mon[98970]: pgmap v1623: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 14166/136101 objects degraded (10.408%); 9713/136101 objects misplaced (7.137%); 9.4 MiB/s, 149 objects/s recovering 2024-08-29T15:43:20.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:20 smithi027 ceph-mon[93921]: pgmap v1623: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 14166/136101 objects degraded (10.408%); 9713/136101 objects misplaced (7.137%); 9.4 MiB/s, 149 objects/s recovering 2024-08-29T15:43:20.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:20.670 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:20.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:20.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:20.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:20.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:20.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:20.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:20.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:20.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:20.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:21.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:21.090 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:21.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:21.192 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:21.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:21.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:21.321 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:21.348 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:21.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:21.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:21.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:21.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:21.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:21.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:21.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:21.614 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:21.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:21.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:21.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:21.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:21.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:21.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:21.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:21.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:21.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:21.986 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:22.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:22.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:22.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:22.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:22.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:22.268 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:22.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:22.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:22.341 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:22.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:22.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:22.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:22 smithi171 ceph-mon[90031]: pgmap v1624: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 13822/133482 objects degraded (10.355%); 9411/133482 objects misplaced (7.050%); 9.6 MiB/s, 154 objects/s recovering 2024-08-29T15:43:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 13822/133482 objects degraded (10.355%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:22 smithi027 ceph-mon[93921]: pgmap v1624: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 13822/133482 objects degraded (10.355%); 9411/133482 objects misplaced (7.050%); 9.6 MiB/s, 154 objects/s recovering 2024-08-29T15:43:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 13822/133482 objects degraded (10.355%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:22.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:22 smithi027 ceph-mon[98970]: pgmap v1624: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 13822/133482 objects degraded (10.355%); 9411/133482 objects misplaced (7.050%); 9.6 MiB/s, 154 objects/s recovering 2024-08-29T15:43:22.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 13822/133482 objects degraded (10.355%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:22.574 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:22.600 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:22.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:22.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:22.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:22.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:22.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:22.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:22.902 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:22.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:23.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:23.037 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:23.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:23.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:23.139 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:23.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:23.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:23.238 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:43:23.238 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:43:23.241 INFO:teuthology.orchestra.run.smithi027.stderr:set pool 1 pg_num to 21 2024-08-29T15:43:23.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:23.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:23.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:23.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:23.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:23.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2196040798' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-29T15:43:23.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-29T15:43:23.564 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2196040798' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-29T15:43:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-29T15:43:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2196040798' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-29T15:43:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:23 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-29T15:43:23.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:23.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:23.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:23.770 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:23.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:23.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:23.905 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:23.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:24.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:24.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:24.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:24.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:24.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:24.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:24.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:24.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:24.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:24.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:24.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:43:24.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:24.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:24.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:24.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:24 smithi171 ceph-mon[90031]: pgmap v1625: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 13466/127851 objects degraded (10.533%); 9250/127851 objects misplaced (7.235%); 9.8 MiB/s, 156 objects/s recovering 2024-08-29T15:43:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:24 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-08-29T15:43:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:24 smithi171 ceph-mon[90031]: osdmap e456: 8 total, 8 up, 8 in 2024-08-29T15:43:24.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:24 smithi027 ceph-mon[98970]: pgmap v1625: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 13466/127851 objects degraded (10.533%); 9250/127851 objects misplaced (7.235%); 9.8 MiB/s, 156 objects/s recovering 2024-08-29T15:43:24.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:24 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-08-29T15:43:24.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:24 smithi027 ceph-mon[98970]: osdmap e456: 8 total, 8 up, 8 in 2024-08-29T15:43:24.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:24 smithi027 ceph-mon[93921]: pgmap v1625: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 13466/127851 objects degraded (10.533%); 9250/127851 objects misplaced (7.235%); 9.8 MiB/s, 156 objects/s recovering 2024-08-29T15:43:24.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:24 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-08-29T15:43:24.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:24 smithi027 ceph-mon[93921]: osdmap e456: 8 total, 8 up, 8 in 2024-08-29T15:43:24.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:24.589 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:24.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:24.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:24.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:24.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:24.837 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:24.863 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:24.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:24.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:24.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:25.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:25.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:25.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:25.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:25.179 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:25.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:25.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:25.281 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:25.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:25.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:25.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:25.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:25.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:25.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:43:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:25 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-29T15:43:25.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:43:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:25 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-29T15:43:25.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:43:25.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:25 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-29T15:43:25.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:25.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:25.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:25.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:25.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:25.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:25.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:25.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:25.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:25.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:26.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:26.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:26.136 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:26.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:26.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:26.267 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:26.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:26.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:26.411 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:26.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:26 smithi171 ceph-mon[90031]: pgmap v1627: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 13466/127314 objects degraded (10.577%); 9250/127314 objects misplaced (7.266%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:43:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:26 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-08-29T15:43:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:26 smithi171 ceph-mon[90031]: osdmap e457: 8 total, 8 up, 8 in 2024-08-29T15:43:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:26 smithi171 ceph-mon[90031]: osdmap e458: 8 total, 8 up, 8 in 2024-08-29T15:43:26.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:26.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[93921]: pgmap v1627: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 13466/127314 objects degraded (10.577%); 9250/127314 objects misplaced (7.266%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:43:26.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-08-29T15:43:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[93921]: osdmap e457: 8 total, 8 up, 8 in 2024-08-29T15:43:26.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[93921]: osdmap e458: 8 total, 8 up, 8 in 2024-08-29T15:43:26.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[98970]: pgmap v1627: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 13466/127314 objects degraded (10.577%); 9250/127314 objects misplaced (7.266%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:43:26.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-08-29T15:43:26.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[98970]: osdmap e457: 8 total, 8 up, 8 in 2024-08-29T15:43:26.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:26 smithi027 ceph-mon[98970]: osdmap e458: 8 total, 8 up, 8 in 2024-08-29T15:43:26.644 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:26.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:26.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:26.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:26.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:26.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:26.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:26.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:27.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:27.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:27.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:27.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:27.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:27.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:27.366 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:27.468 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:27.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:27.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:27 smithi171 ceph-mon[90031]: 1.3 scrub starts 2024-08-29T15:43:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:27 smithi171 ceph-mon[90031]: 1.3 scrub ok 2024-08-29T15:43:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:27 smithi171 ceph-mon[90031]: 1.12 scrub starts 2024-08-29T15:43:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:27 smithi171 ceph-mon[90031]: 1.12 scrub ok 2024-08-29T15:43:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 13466/127314 objects degraded (10.577%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:27.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:27 smithi171 ceph-mon[90031]: osdmap e459: 8 total, 8 up, 8 in 2024-08-29T15:43:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[98970]: 1.3 scrub starts 2024-08-29T15:43:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[98970]: 1.3 scrub ok 2024-08-29T15:43:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[98970]: 1.12 scrub starts 2024-08-29T15:43:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[98970]: 1.12 scrub ok 2024-08-29T15:43:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 13466/127314 objects degraded (10.577%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[98970]: osdmap e459: 8 total, 8 up, 8 in 2024-08-29T15:43:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[93921]: 1.3 scrub starts 2024-08-29T15:43:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[93921]: 1.3 scrub ok 2024-08-29T15:43:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[93921]: 1.12 scrub starts 2024-08-29T15:43:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[93921]: 1.12 scrub ok 2024-08-29T15:43:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 13466/127314 objects degraded (10.577%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:27 smithi027 ceph-mon[93921]: osdmap e459: 8 total, 8 up, 8 in 2024-08-29T15:43:27.599 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:27.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:27.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:27.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:27.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:27.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:27.846 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:27.868 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:27.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:27.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:27.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:27.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:28.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:28.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:28.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:28.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:28.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:28.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:28.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.4 scrub starts 2024-08-29T15:43:28.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.4 scrub ok 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: pgmap v1630: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 4 peering, 8 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 13155/122004 objects degraded (10.782%); 8743/122004 objects misplaced (7.166%); 10 MiB/s, 162 objects/s recovering 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.13 scrub starts 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.13 scrub ok 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.5 scrub starts 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.5 scrub ok 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.2 scrub starts 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: 1.2 scrub ok 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[93921]: osdmap e460: 8 total, 8 up, 8 in 2024-08-29T15:43:28.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.4 scrub starts 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.4 scrub ok 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: pgmap v1630: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 4 peering, 8 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 13155/122004 objects degraded (10.782%); 8743/122004 objects misplaced (7.166%); 10 MiB/s, 162 objects/s recovering 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.13 scrub starts 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.13 scrub ok 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.5 scrub starts 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.5 scrub ok 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.2 scrub starts 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: 1.2 scrub ok 2024-08-29T15:43:28.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:28 smithi027 ceph-mon[98970]: osdmap e460: 8 total, 8 up, 8 in 2024-08-29T15:43:28.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:28.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:28.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:28.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:28.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:28.520 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:28.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:28.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:28.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:28.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:28.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:28.699 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:43:28.699 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:43:28.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.4 scrub starts 2024-08-29T15:43:28.719 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.4 scrub ok 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: pgmap v1630: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 4 peering, 8 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 13155/122004 objects degraded (10.782%); 8743/122004 objects misplaced (7.166%); 10 MiB/s, 162 objects/s recovering 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.13 scrub starts 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.13 scrub ok 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.5 scrub starts 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.5 scrub ok 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.2 scrub starts 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: 1.2 scrub ok 2024-08-29T15:43:28.720 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:28 smithi171 ceph-mon[90031]: osdmap e460: 8 total, 8 up, 8 in 2024-08-29T15:43:28.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:43:28.738 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:43:28.772 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:43:28.772 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:43:28.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:28.815 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:43:28.815 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:43:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:28.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:28.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:28.848 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:43:28.848 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:43:28.881 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:43:28.881 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:43:28.915 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:43:28.915 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:43:28.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:28.949 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:43:28.949 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:43:28.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:28.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:29.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:29.109 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:29.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:29.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:29.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:29.212 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:43:29] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-29T15:43:29.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:43:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:29.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:29.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:29.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:29.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:29.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:29.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:29.560 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:29.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:29.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:29.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: 1.14 scrub starts 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: 1.14 scrub ok 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: 1.b scrub starts 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: 1.b scrub ok 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/760102344' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[93921]: 1.6 scrub ok 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: 1.14 scrub starts 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: 1.14 scrub ok 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: 1.b scrub starts 2024-08-29T15:43:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: 1.b scrub ok 2024-08-29T15:43:29.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/760102344' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:29.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:29.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:29.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:43:29.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:29 smithi027 ceph-mon[98970]: 1.6 scrub ok 2024-08-29T15:43:29.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:29.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:29.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:29.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: 1.14 scrub starts 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: 1.14 scrub ok 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: 1.b scrub starts 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: 1.b scrub ok 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/760102344' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:43:29.700 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:29 smithi171 ceph-mon[90031]: 1.6 scrub ok 2024-08-29T15:43:29.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:29.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:29.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:29.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:29.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:29.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:29.998 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:30.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:30.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:30.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:30.208 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:30.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:30.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:30.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:30.360 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:30.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:30.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:30.461 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:30.487 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:30.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:30.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[98970]: 1.c scrub starts 2024-08-29T15:43:30.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[98970]: 1.c scrub ok 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[98970]: pgmap v1633: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 4 peering, 6 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 12829/119370 objects degraded (10.747%); 8743/119370 objects misplaced (7.324%); 13 MiB/s, 201 objects/s recovering 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[98970]: 1.d deep-scrub starts 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[98970]: 1.d deep-scrub ok 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[98970]: 1.e scrub ok 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[93921]: 1.c scrub starts 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[93921]: 1.c scrub ok 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[93921]: pgmap v1633: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 4 peering, 6 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 12829/119370 objects degraded (10.747%); 8743/119370 objects misplaced (7.324%); 13 MiB/s, 201 objects/s recovering 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[93921]: 1.d deep-scrub starts 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[93921]: 1.d deep-scrub ok 2024-08-29T15:43:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:43:30.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:30 smithi027 ceph-mon[93921]: 1.e scrub ok 2024-08-29T15:43:30.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:30.622 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:30.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:30.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:30.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:30.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:30.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:30 smithi171 ceph-mon[90031]: 1.c scrub starts 2024-08-29T15:43:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:30 smithi171 ceph-mon[90031]: 1.c scrub ok 2024-08-29T15:43:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:30 smithi171 ceph-mon[90031]: pgmap v1633: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 4 peering, 6 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 12829/119370 objects degraded (10.747%); 8743/119370 objects misplaced (7.324%); 13 MiB/s, 201 objects/s recovering 2024-08-29T15:43:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:30 smithi171 ceph-mon[90031]: 1.d deep-scrub starts 2024-08-29T15:43:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:30 smithi171 ceph-mon[90031]: 1.d deep-scrub ok 2024-08-29T15:43:30.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:30 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:43:30.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:30 smithi171 ceph-mon[90031]: 1.e scrub ok 2024-08-29T15:43:30.855 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:30.893 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:30.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:30.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:30.945 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:30.995 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:31.033 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:31.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:31.134 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:31.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:31.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:31.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:31.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:31.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:31.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:31.331 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[98970]: 1.0 scrub starts 2024-08-29T15:43:31.331 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[98970]: 1.0 scrub ok 2024-08-29T15:43:31.331 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:31.331 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:31.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[93921]: 1.0 scrub starts 2024-08-29T15:43:31.331 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[93921]: 1.0 scrub ok 2024-08-29T15:43:31.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:31.332 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:31.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:31.446 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:31.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:31.547 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:31.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:31.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:31.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:31.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:31.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:31 smithi171 ceph-mon[90031]: 1.0 scrub starts 2024-08-29T15:43:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:31 smithi171 ceph-mon[90031]: 1.0 scrub ok 2024-08-29T15:43:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:31.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:31.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:31.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:31.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:31.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:32.003 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:32.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:32.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:32.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:43:32.105 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:32.188 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:32.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:32.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:32.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:32.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:32.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:32.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:32.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[93921]: pgmap v1634: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 12464/116814 objects degraded (10.670%); 8581/116814 objects misplaced (7.346%); 12 MiB/s, 198 objects/s recovering 2024-08-29T15:43:32.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[93921]: 1.10 scrub starts 2024-08-29T15:43:32.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[93921]: 1.10 scrub ok 2024-08-29T15:43:32.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 12464/116814 objects degraded (10.670%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:32.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[98970]: pgmap v1634: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 12464/116814 objects degraded (10.670%); 8581/116814 objects misplaced (7.346%); 12 MiB/s, 198 objects/s recovering 2024-08-29T15:43:32.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[98970]: 1.10 scrub starts 2024-08-29T15:43:32.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[98970]: 1.10 scrub ok 2024-08-29T15:43:32.419 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 12464/116814 objects degraded (10.670%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:32.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:32.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:32.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:32.590 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:32.618 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:32.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:32.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:32.762 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:32.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:32 smithi171 ceph-mon[90031]: pgmap v1634: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 12464/116814 objects degraded (10.670%); 8581/116814 objects misplaced (7.346%); 12 MiB/s, 198 objects/s recovering 2024-08-29T15:43:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:32 smithi171 ceph-mon[90031]: 1.10 scrub starts 2024-08-29T15:43:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:32 smithi171 ceph-mon[90031]: 1.10 scrub ok 2024-08-29T15:43:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 12464/116814 objects degraded (10.670%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:32.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:32.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:32.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:33.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:33.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:33.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:33.084 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:43:33.084 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:43:33.085 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.5171600066484857 2024-08-29T15:43:33.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 0 0.5171600066484857 2024-08-29T15:43:33.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:33.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:33.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:33.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:33.309 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:43:33.312 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:33.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:33.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2008765518' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:43:33.323 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:33 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:43:33.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:33.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:33.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:33.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:33.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:33.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:33.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:33.708 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:33.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:33.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2008765518' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:43:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:33 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:43:33.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2008765518' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:43:33.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:33 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:43:33.841 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:33.880 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:33.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:33.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:33.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:34.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:34.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:34.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:34.147 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:34.249 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:34.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:34.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[98970]: pgmap v1635: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 12133/111090 objects degraded (10.922%); 8400/111090 objects misplaced (7.561%); 7.7 MiB/s, 124 objects/s recovering 2024-08-29T15:43:34.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:34.370 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:43:34.370 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[98970]: osdmap e461: 8 total, 8 up, 8 in 2024-08-29T15:43:34.377 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:34.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[93921]: pgmap v1635: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 12133/111090 objects degraded (10.922%); 8400/111090 objects misplaced (7.561%); 7.7 MiB/s, 124 objects/s recovering 2024-08-29T15:43:34.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:34.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:43:34.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:34 smithi027 ceph-mon[93921]: osdmap e461: 8 total, 8 up, 8 in 2024-08-29T15:43:34.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:34.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:34.503 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:34.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:34.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:34.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:34.692 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:34.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:34 smithi171 ceph-mon[90031]: pgmap v1635: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 12133/111090 objects degraded (10.922%); 8400/111090 objects misplaced (7.561%); 7.7 MiB/s, 124 objects/s recovering 2024-08-29T15:43:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:34 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:34 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:43:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:34 smithi171 ceph-mon[90031]: osdmap e461: 8 total, 8 up, 8 in 2024-08-29T15:43:34.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:34.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:34.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:34.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:34.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:34.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:34.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:35.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:35.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:35.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:35.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:35.215 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:35.238 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:35.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:35.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:35.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:35.385 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:35.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:35.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:35.512 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:35.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:35.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:35.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:35.643 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:35.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:35.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:35.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:35.789 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:35.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:35.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:35.926 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:35.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:36.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:36.061 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:36.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:36.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:36.189 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:36.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:36.199 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:36.251 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:36.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:36.314 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:36.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:36.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:36.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:36.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:36.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:36.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:36 smithi027 ceph-mon[98970]: pgmap v1637: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 12133/110526 objects degraded (10.978%); 8400/110526 objects misplaced (7.600%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:43:36.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:36 smithi027 ceph-mon[93921]: pgmap v1637: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 12133/110526 objects degraded (10.978%); 8400/110526 objects misplaced (7.600%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:43:36.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:36.606 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:36.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:36.707 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:36.734 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:36.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:36 smithi171 ceph-mon[90031]: pgmap v1637: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 12133/110526 objects degraded (10.978%); 8400/110526 objects misplaced (7.600%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:43:36.836 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:36.877 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:36.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:36.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:37.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:37.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:37.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:37.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:37.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:37.240 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:37.272 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:37.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.423 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:37.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:37.524 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:37.526 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:37 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 12133/110526 objects degraded (10.978%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:37.526 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:37 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 12133/110526 objects degraded (10.978%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:37.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:37.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:37.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:37.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:37.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:43:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:37 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 12133/110526 objects degraded (10.978%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:37.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:37.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:37.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:37.942 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:37.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:37.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:37.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:38.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:38.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:38.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:38.216 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:38.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:38.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:38.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:38.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:38.396 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:38.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:38.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:38.545 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:38.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:38.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:38 smithi027 ceph-mon[93921]: pgmap v1638: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 11792/104718 objects degraded (11.261%); 8216/104718 objects misplaced (7.846%); 7.1 MiB/s, 114 objects/s recovering 2024-08-29T15:43:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:38 smithi027 ceph-mon[98970]: pgmap v1638: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 11792/104718 objects degraded (11.261%); 8216/104718 objects misplaced (7.846%); 7.1 MiB/s, 114 objects/s recovering 2024-08-29T15:43:38.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:38.668 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:38 smithi171 ceph-mon[90031]: pgmap v1638: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 11792/104718 objects degraded (11.261%); 8216/104718 objects misplaced (7.846%); 7.1 MiB/s, 114 objects/s recovering 2024-08-29T15:43:38.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:38.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:38.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:38.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:38.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:38.915 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:172707 9.8 9:1642d911:::benchmark_data_smithi027_341149_object70277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.875102+0000", 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.037455878999999997, 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.037485426000000002, 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:43:38.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 172707 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.875102+0000", 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.875102+0000", 2024-08-29T15:43:38.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.875103+0000", 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0690000000000001e-06 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.875108+0000", 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5959999999999997e-06 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.875121+0000", 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3057999999999999e-05 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.875128+0000", 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3050000000000004e-06 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:43:38.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.878567+0000", 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034387319999999999 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.879542+0000", 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097475800000000005 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910176+0000", 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030634104999999998 2024-08-29T15:43:38.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910276+0000", 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9895999999999994e-05 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910428+0000", 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015265 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:172717 9.8 9.b15fa648 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.893282+0000", 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.019275746999999999, 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019380854999999999, 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:43:38.922 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 172717 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.893282+0000", 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.893282+0000", 2024-08-29T15:43:38.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.893284+0000", 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9889999999999999e-06 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.893291+0000", 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3949999999999998e-06 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.893311+0000", 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0143999999999999e-05 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.893322+0000", 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0958000000000001e-05 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(9.8 461/435 [PushReplyOp(9:1642d911:::benchmark_data_smithi027_341149_object70277:head)])", 2024-08-29T15:43:38.925 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.894146+0000", 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.018412345, 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018562718999999998, 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894146+0000", 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894146+0000", 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894146+0000", 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9999999999999999e-07 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894149+0000", 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1410000000000001e-06 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894157+0000", 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8650000000000001e-06 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894163+0000", 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6679999999999998e-06 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.907139+0000", 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01297593 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.928 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:172718 9.8 9.e4ef18a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.894620+0000", 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.017938619999999999, 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018133461999999999, 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:43:38.929 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 172718 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894620+0000", 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894620+0000", 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894621+0000", 2024-08-29T15:43:38.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3549999999999999e-06 2024-08-29T15:43:38.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894627+0000", 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5840000000000002e-06 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894640+0000", 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3112000000000001e-05 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894649+0000", 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2080000000000006e-06 2024-08-29T15:43:38.932 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46778.0:172691 9.8 e461/435)", 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.899930+0000", 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012628377, 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012865883, 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.933 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.899930+0000", 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.899930+0000", 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.899931+0000", 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0139999999999999e-06 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.899937+0000", 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8919999999999997e-06 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.899961+0000", 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3779999999999999e-05 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.899968+0000", 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8960000000000003e-06 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:172691 9.8 9:1ce27db3:::benchmark_data_smithi027_341149_object70261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.856040+0000", 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.056518668000000001, 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.05681224, 2024-08-29T15:43:38.936 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 172691 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.856040+0000", 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.856040+0000", 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.856041+0000", 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.424e-06 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.856047+0000", 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6540000000000004e-06 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.856063+0000", 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5934e-05 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.856070+0000", 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7980000000000006e-06 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.856298+0000", 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000227863 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.863949+0000", 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076510420000000003 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894490+0000", 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030540567000000001 2024-08-29T15:43:38.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894563+0000", 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2825999999999997e-05 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.894685+0000", 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121994 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.907102+0000", 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012416874 2024-08-29T15:43:38.941 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:172696 9.8 9:110e8bd1:::benchmark_data_smithi027_341149_object70266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.862240+0000", 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.050317951, 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.050683638000000003, 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:43:38.942 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 172696 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.862240+0000", 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.862240+0000", 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.862241+0000", 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.99e-07 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.862244+0000", 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3620000000000002e-06 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.862253+0000", 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.49e-06 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.862260+0000", 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8210000000000004e-06 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:43:38.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.864074+0000", 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001814534 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.865373+0000", 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012985780000000001 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.902698+0000", 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.037325641 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.902788+0000", 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0082999999999998e-05 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.902932+0000", 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014403199999999999 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46778.0:172691 9.8 e461/435)", 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.910529+0000", 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0020290019999999998, 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024594009999999999, 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910529+0000", 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910529+0000", 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910530+0000", 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2759999999999999e-06 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910538+0000", 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4880000000000001e-06 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910550+0000", 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2046e-05 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910560+0000", 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7890000000000006e-06 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46778.0:172696 9.8 e461/435)", 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.910575+0000", 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0019828319999999999, 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024478899999999999, 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:43:38.951 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910575+0000", 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910575+0000", 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910576+0000", 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.96e-07 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910579+0000", 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0120000000000001e-06 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910586+0000", 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0570000000000004e-06 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.910591+0000", 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3109999999999996e-06 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46778.0:172696 9.8 e461/435)", 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.911095+0000", 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0014628740000000001, 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019625699999999999, 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.954 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.911095+0000", 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.911095+0000", 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.911096+0000", 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7700000000000005e-07 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.911102+0000", 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5660000000000003e-06 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.911114+0000", 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1654999999999999e-05 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.911120+0000", 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7180000000000001e-06 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:172731 9.8 9.dc7282d8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e461)", 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:38.912887+0000", 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.999671, 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020864399999999999, 2024-08-29T15:43:38.957 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 172731 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.912887+0000", 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.912887+0000", 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.912889+0000", 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5319999999999999e-06 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.912892+0000", 2024-08-29T15:43:38.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6619999999999998e-06 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.912909+0000", 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6923999999999999e-05 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:38.912916+0000", 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0550000000000002e-06 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:43:38.960 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 11 2024-08-29T15:43:38.961 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:43:38.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:38.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:38.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:38.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:39.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:39.134 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:39.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:39.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:39.236 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:39.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:39.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:43:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:43:39] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-29T15:43:39.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:39.418 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:39.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:39.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:39.562 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:39.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:39.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:39.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:39.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:39.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:39.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:39.832 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:39.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:39.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:39.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:39.968 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:39.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:40.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:40.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:43:40.099 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:40.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:40.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:40.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:40.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:40.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:40.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:40.380 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:40.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:40.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:40.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:40.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:40.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:40 smithi027 ceph-mon[98970]: pgmap v1639: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 11414/101745 objects degraded (11.218%); 8216/101745 objects misplaced (8.075%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:43:40.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:40 smithi027 ceph-mon[93921]: pgmap v1639: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 11414/101745 objects degraded (11.218%); 8216/101745 objects misplaced (8.075%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:43:40.629 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:40.664 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:40.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:40.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:40 smithi171 ceph-mon[90031]: pgmap v1639: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 11414/101745 objects degraded (11.218%); 8216/101745 objects misplaced (8.075%); 7.5 MiB/s, 120 objects/s recovering 2024-08-29T15:43:40.806 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:40.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:40.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:40.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:40.952 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:40.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:40.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:41.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:41.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:41.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:41.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:41.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:41.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:41.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:41.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:41.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:41.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:41.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:41.471 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:41.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:41.573 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:41.604 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:41.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:41.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:41.742 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:41.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:41.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:41.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:41.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:42.011 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:42.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:42.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.150 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:42.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:42.251 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:42.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:42.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:42.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:42.408 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:42.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.552 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:42.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:42.653 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:42.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:42 smithi171 ceph-mon[90031]: pgmap v1640: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 10975/98364 objects degraded (11.158%); 8216/98364 objects misplaced (8.353%); 6.7 MiB/s, 108 objects/s recovering 2024-08-29T15:43:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:42 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 10975/98364 objects degraded (11.158%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:42.779 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:42.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:42 smithi027 ceph-mon[98970]: pgmap v1640: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 10975/98364 objects degraded (11.158%); 8216/98364 objects misplaced (8.353%); 6.7 MiB/s, 108 objects/s recovering 2024-08-29T15:43:42.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:42 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 10975/98364 objects degraded (11.158%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:42.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:42 smithi027 ceph-mon[93921]: pgmap v1640: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 10975/98364 objects degraded (11.158%); 8216/98364 objects misplaced (8.353%); 6.7 MiB/s, 108 objects/s recovering 2024-08-29T15:43:42.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:42 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 10975/98364 objects degraded (11.158%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:42.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:42.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:42.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:43.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:43.074 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:43.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:43.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:43.202 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:43.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:43.330 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:43.370 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.0 to 0.51716 (8464) 2024-08-29T15:43:43.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:43.467 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:43.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:43.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:43.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:43.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:43.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:43.723 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2172888896' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857} : dispatch 2024-08-29T15:43:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857} : dispatch 2024-08-29T15:43:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2172888896' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857} : dispatch 2024-08-29T15:43:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857} : dispatch 2024-08-29T15:43:43.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2172888896' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857} : dispatch 2024-08-29T15:43:43.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857} : dispatch 2024-08-29T15:43:43.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:43.848 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:43.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:43.950 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:43.972 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:43.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:43.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:44.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:44.095 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:44.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:44.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:44.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:44.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:44.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:44.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:44.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:44.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:44.495 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[93921]: pgmap v1641: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 10647/91398 objects degraded (11.649%); 7994/91398 objects misplaced (8.746%); 6.8 MiB/s, 108 objects/s recovering 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857}]': finished 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[93921]: osdmap e462: 8 total, 8 up, 8 in 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2926948294' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[93921]: osdmap e463: 8 total, 8 up, 8 in 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[98970]: pgmap v1641: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 10647/91398 objects degraded (11.649%); 7994/91398 objects misplaced (8.746%); 6.8 MiB/s, 108 objects/s recovering 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857}]': finished 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[98970]: osdmap e462: 8 total, 8 up, 8 in 2024-08-29T15:43:44.496 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:44.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:44.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2926948294' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:44.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:44 smithi027 ceph-mon[98970]: osdmap e463: 8 total, 8 up, 8 in 2024-08-29T15:43:44.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:44.593 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:44.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:44.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:44 smithi171 ceph-mon[90031]: pgmap v1641: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 10647/91398 objects degraded (11.649%); 7994/91398 objects misplaced (8.746%); 6.8 MiB/s, 108 objects/s recovering 2024-08-29T15:43:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:44 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5171600066484857}]': finished 2024-08-29T15:43:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:44 smithi171 ceph-mon[90031]: osdmap e462: 8 total, 8 up, 8 in 2024-08-29T15:43:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2926948294' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:44.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:44 smithi171 ceph-mon[90031]: osdmap e463: 8 total, 8 up, 8 in 2024-08-29T15:43:44.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:44.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:44.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:44.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:44.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:44.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:45.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:45.090 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:45.102 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:43:45.102 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:43:45.102 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:43:45.103 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:43:45.104 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:43:45.104 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:43:45.104 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:43:45.104 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:43:45.104 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:43:45.192 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:45.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:45.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:45.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:45.338 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:45.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:45.440 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:45.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:45.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:45.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:45.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:45.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:45.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:45.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:45.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:45.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:45.991 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:45.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:45.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:46.093 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:46.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:46.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:46.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:46.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:46.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:46.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:46.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:46.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:46.407 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:46.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:46.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:46.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:46.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:46.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:46.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:46.693 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:46.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:46.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:46 smithi171 ceph-mon[90031]: pgmap v1644: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 10655/90903 objects degraded (11.721%); 7994/90903 objects misplaced (8.794%); 5.9 MiB/s, 94 objects/s recovering 2024-08-29T15:43:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:46 smithi171 ceph-mon[90031]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 461'11018 2024-08-29T15:43:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:46 smithi171 ceph-mon[90031]: osdmap e464: 8 total, 8 up, 8 in 2024-08-29T15:43:46.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:46 smithi027 ceph-mon[93921]: pgmap v1644: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 10655/90903 objects degraded (11.721%); 7994/90903 objects misplaced (8.794%); 5.9 MiB/s, 94 objects/s recovering 2024-08-29T15:43:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:46 smithi027 ceph-mon[93921]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 461'11018 2024-08-29T15:43:46.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:46 smithi027 ceph-mon[93921]: osdmap e464: 8 total, 8 up, 8 in 2024-08-29T15:43:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:46 smithi027 ceph-mon[98970]: pgmap v1644: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 1.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 10655/90903 objects degraded (11.721%); 7994/90903 objects misplaced (8.794%); 5.9 MiB/s, 94 objects/s recovering 2024-08-29T15:43:46.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:46 smithi027 ceph-mon[98970]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 461'11018 2024-08-29T15:43:46.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:46 smithi027 ceph-mon[98970]: osdmap e464: 8 total, 8 up, 8 in 2024-08-29T15:43:46.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:46.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:46.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:46.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:46.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:46.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:47.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:47.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:47.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:47.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:47.207 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:47.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:47.252 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:47.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:47.353 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:47.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:47.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:47.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:47.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:47.524 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:47.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:47.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:47 smithi027 ceph-mon[93921]: osdmap e465: 8 total, 8 up, 8 in 2024-08-29T15:43:47.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:47 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 10655/90903 objects degraded (11.721%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:47 smithi027 ceph-mon[98970]: osdmap e465: 8 total, 8 up, 8 in 2024-08-29T15:43:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:47 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 10655/90903 objects degraded (11.721%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:47.627 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:47.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:47.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:47.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:47.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:47 smithi171 ceph-mon[90031]: osdmap e465: 8 total, 8 up, 8 in 2024-08-29T15:43:47.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:47 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 10655/90903 objects degraded (11.721%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:47.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:47.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:47.885 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:47.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:47.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:47.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:48.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:48.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:48.159 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:48.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:48.215 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:48.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:48.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:48.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:48.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:48.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:48.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:48.578 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:48.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.711 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:48.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:48.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:48 smithi027 ceph-mon[98970]: pgmap v1647: 109 pgs: 2 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 8668/87291 objects degraded (9.930%); 9352/87291 objects misplaced (10.714%); 6.1 MiB/s, 98 objects/s recovering 2024-08-29T15:43:48.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:48 smithi027 ceph-mon[98970]: osdmap e466: 8 total, 8 up, 8 in 2024-08-29T15:43:48.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:48 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:43:48.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:48 smithi027 ceph-mon[93921]: pgmap v1647: 109 pgs: 2 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 8668/87291 objects degraded (9.930%); 9352/87291 objects misplaced (10.714%); 6.1 MiB/s, 98 objects/s recovering 2024-08-29T15:43:48.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:48 smithi027 ceph-mon[93921]: osdmap e466: 8 total, 8 up, 8 in 2024-08-29T15:43:48.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:48 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:43:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:48 smithi171 ceph-mon[90031]: pgmap v1647: 109 pgs: 2 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 1.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 8668/87291 objects degraded (9.930%); 9352/87291 objects misplaced (10.714%); 6.1 MiB/s, 98 objects/s recovering 2024-08-29T15:43:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:48 smithi171 ceph-mon[90031]: osdmap e466: 8 total, 8 up, 8 in 2024-08-29T15:43:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:48 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:43:48.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:48.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:48.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:48.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:48.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:48.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:49.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:49.115 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:49.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:49.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:49.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:49.295 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:49.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:43:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:43:49] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.51.0" 2024-08-29T15:43:49.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:49.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:49.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:49.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:49.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:49.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:49.679 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:49.705 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:49 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:43:49.706 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:49 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:43:49.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:49.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:49.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:49.737 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:43:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:49 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:43:49.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:49.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:49.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:49.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:50.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:50.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:50.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:50.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:50.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:50.292 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:50.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:50.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:50.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:50.579 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:50.681 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:50.698 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:50 smithi027 ceph-mon[93921]: pgmap v1649: 109 pgs: 1 active+remapped+backfill_wait, 4 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 3955/84636 objects degraded (4.673%); 14197/84636 objects misplaced (16.774%); 8.1 MiB/s, 130 objects/s recovering 2024-08-29T15:43:50.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:50 smithi027 ceph-mon[98970]: pgmap v1649: 109 pgs: 1 active+remapped+backfill_wait, 4 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 3955/84636 objects degraded (4.673%); 14197/84636 objects misplaced (16.774%); 8.1 MiB/s, 130 objects/s recovering 2024-08-29T15:43:50.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:50 smithi171 ceph-mon[90031]: pgmap v1649: 109 pgs: 1 active+remapped+backfill_wait, 4 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 3955/84636 objects degraded (4.673%); 14197/84636 objects misplaced (16.774%); 8.1 MiB/s, 130 objects/s recovering 2024-08-29T15:43:50.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:50.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:50.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:50.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:50.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:51.080 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:51.127 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:51.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:51.229 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:51.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:51.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:51.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:51.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:51.405 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:51.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:51.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:51.521 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:43:51.521 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:43:51.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:51.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:51.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:51.641 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:51.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:51.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:51.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:51.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:51.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:51.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:51.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:51.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:51.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:52.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:52.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:52.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:52.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:52.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:52.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:52.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:52.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:43:52.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:52.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:52.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:52.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:52.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:52.485 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:52.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:52.586 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:52.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:52.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:52.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:52.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:52.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:52 smithi171 ceph-mon[90031]: pgmap v1650: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 3955/81882 objects degraded (4.830%); 13965/81882 objects misplaced (17.055%); 7.6 MiB/s, 121 objects/s recovering 2024-08-29T15:43:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:52 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3955/81882 objects degraded (4.830%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:52 smithi027 ceph-mon[98970]: pgmap v1650: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 3955/81882 objects degraded (4.830%); 13965/81882 objects misplaced (17.055%); 7.6 MiB/s, 121 objects/s recovering 2024-08-29T15:43:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:52 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3955/81882 objects degraded (4.830%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:52.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:52 smithi027 ceph-mon[93921]: pgmap v1650: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 1.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 3955/81882 objects degraded (4.830%); 13965/81882 objects misplaced (17.055%); 7.6 MiB/s, 121 objects/s recovering 2024-08-29T15:43:52.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:52 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3955/81882 objects degraded (4.830%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:52.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:52.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:52.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:52.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:52.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:53.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:53.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:53.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:53.175 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:43:53.175 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:43:53.176 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-08-29T15:43:53.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:43:53.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:53.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:53.405 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:53.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:53.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:53.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:53.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:53.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:53.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:53.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:53.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:53.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:53.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:53.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:54.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:54.101 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:54.133 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:54.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:54.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:54.236 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:54.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:54.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:54.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:54.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:54.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:54.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:54.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:54.431 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:43:54.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:54.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:54.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:54.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[93921]: pgmap v1651: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 101 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3955/72939 objects degraded (5.422%); 13359/72939 objects misplaced (18.315%); 7.3 MiB/s, 117 objects/s recovering 2024-08-29T15:43:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3435364660' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:43:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[98970]: pgmap v1651: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 101 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3955/72939 objects degraded (5.422%); 13359/72939 objects misplaced (18.315%); 7.3 MiB/s, 117 objects/s recovering 2024-08-29T15:43:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3435364660' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:54 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:43:54.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:54.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:54.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:54.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:43:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:54 smithi171 ceph-mon[90031]: pgmap v1651: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 101 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3955/72939 objects degraded (5.422%); 13359/72939 objects misplaced (18.315%); 7.3 MiB/s, 117 objects/s recovering 2024-08-29T15:43:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:54 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3435364660' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:54 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:43:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:54 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:43:54.805 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:54.830 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:54.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:54.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:54.973 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:54.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:55.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:55.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:55.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:55.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:55.244 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:55.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:55.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:55.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:55.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:55.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:55.511 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:55.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:55.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:55.613 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:55.642 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:55.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:55.744 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:55 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:55 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:43:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:55 smithi171 ceph-mon[90031]: osdmap e467: 8 total, 8 up, 8 in 2024-08-29T15:43:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:55 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:43:55.782 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:55.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:55.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:55.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:55.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:43:55.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[93921]: osdmap e467: 8 total, 8 up, 8 in 2024-08-29T15:43:55.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:43:55.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:43:55.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:43:55.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[98970]: osdmap e467: 8 total, 8 up, 8 in 2024-08-29T15:43:55.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:55 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:43:55.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:55.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:55.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:55.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:56.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:56.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:56.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:56.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:56.205 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:56.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:56.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:56.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:56.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:56.447 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:56.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:56.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:56.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:56.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:56.615 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:56.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:56.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:56.744 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:56.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:56.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:56 smithi171 ceph-mon[90031]: pgmap v1653: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 3955/72210 objects degraded (5.477%); 13359/72210 objects misplaced (18.500%); 4.0 MiB/s, 63 objects/s recovering 2024-08-29T15:43:56.794 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:56 smithi027 ceph-mon[98970]: pgmap v1653: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 3955/72210 objects degraded (5.477%); 13359/72210 objects misplaced (18.500%); 4.0 MiB/s, 63 objects/s recovering 2024-08-29T15:43:56.794 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:56 smithi027 ceph-mon[93921]: pgmap v1653: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 3955/72210 objects degraded (5.477%); 13359/72210 objects misplaced (18.500%); 4.0 MiB/s, 63 objects/s recovering 2024-08-29T15:43:56.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:56.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:56.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:56.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:57.004 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:57.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:57.133 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:57.234 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:57.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:57.296 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:43:57.296 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:43:57.343 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:43:57.343 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:43:57.379 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:43:57.379 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:43:57.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:57.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:57.422 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:43:57.423 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:43:57.453 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:43:57.454 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:43:57.492 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:43:57.492 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:43:57.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:43:57.533 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:57.533 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:43:57.533 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:43:57.536 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:43:57.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:43:57.571 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:43:57.572 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:43:57.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:57 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3955/72210 objects degraded (5.477%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:57.596 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/640199386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:57.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:57 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3955/72210 objects degraded (5.477%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:57.596 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/640199386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:57.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:57.708 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:57 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3955/72210 objects degraded (5.477%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:43:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/640199386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:43:57.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:43:57.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:43:57.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:57.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:58.077 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:58.109 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:58.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:58.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:58.237 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:58.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:58.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:58.376 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:58.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:58.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:58.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:58.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:58 smithi027 ceph-mon[98970]: pgmap v1654: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 102 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3955/66123 objects degraded (5.981%); 12888/66123 objects misplaced (19.491%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:43:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:58 smithi027 ceph-mon[93921]: pgmap v1654: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 102 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3955/66123 objects degraded (5.981%); 12888/66123 objects misplaced (19.491%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:43:58.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:58.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:58.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:58.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:58 smithi171 ceph-mon[90031]: pgmap v1654: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 102 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 3955/66123 objects degraded (5.981%); 12888/66123 objects misplaced (19.491%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:43:58.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:58.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:58.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:58.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:58.946 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:58.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:58.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:59.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:43:59.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:59.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:59.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:43:59.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:59.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:59.255 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:59.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:43:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:43:59] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.51.0" 2024-08-29T15:43:59.357 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:43:59.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:59.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:43:59.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:43:59.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:59.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:43:59.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:43:59.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:43:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:59.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:59.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:43:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:43:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:59.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:43:59.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:43:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:43:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:43:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:43:59.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:43:59.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:59.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:43:59.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:43:59.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:43:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:43:59.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:00.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:00.107 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:00.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:00.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:00.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:00.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:00.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:44:00.350 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:00.375 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:00.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:00.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:00.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:00.507 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:00.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:00.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:00.608 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:00.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:00.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:00.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:00 smithi171 ceph-mon[90031]: pgmap v1655: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 102 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3955/62001 objects degraded (6.379%); 12188/62001 objects misplaced (19.658%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:44:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:00 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:44:00.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:00.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:00 smithi027 ceph-mon[93921]: pgmap v1655: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 102 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3955/62001 objects degraded (6.379%); 12188/62001 objects misplaced (19.658%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:44:00.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:00 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:44:00.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:00 smithi027 ceph-mon[98970]: pgmap v1655: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 102 active+clean; 1.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3955/62001 objects degraded (6.379%); 12188/62001 objects misplaced (19.658%); 4.7 MiB/s, 75 objects/s recovering 2024-08-29T15:44:00.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:00 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:44:00.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:00.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:00.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:00.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:01.023 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:01.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:01.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:01.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:01.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:01.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:01.286 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:01.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:01.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:01.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:01.451 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:01.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:01.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:01.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:01.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:01.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:01.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:01.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:01.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:01 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:44:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:01 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:44:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:01 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:44:01.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:01.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:01.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:01.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:01.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:02.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:02.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:02.129 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:02.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:02.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:02.272 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:02.374 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:02.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:02.506 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:02.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:02.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:02.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:02 smithi171 ceph-mon[90031]: pgmap v1656: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 102 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3955/59127 objects degraded (6.689%); 11717/59127 objects misplaced (19.817%); 5.5 MiB/s, 87 objects/s recovering 2024-08-29T15:44:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:02 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3955/59127 objects degraded (6.689%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:02.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:02 smithi027 ceph-mon[93921]: pgmap v1656: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 102 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3955/59127 objects degraded (6.689%); 11717/59127 objects misplaced (19.817%); 5.5 MiB/s, 87 objects/s recovering 2024-08-29T15:44:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:02 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3955/59127 objects degraded (6.689%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:02 smithi027 ceph-mon[98970]: pgmap v1656: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 102 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3955/59127 objects degraded (6.689%); 11717/59127 objects misplaced (19.817%); 5.5 MiB/s, 87 objects/s recovering 2024-08-29T15:44:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:02 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3955/59127 objects degraded (6.689%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:02.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:02.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:02.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:02.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:02.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:03.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:03.110 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:03.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:03.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:03.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:03.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:03.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:03.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:03.380 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:03.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:03.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:03.519 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:03.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:03.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:03.653 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:03.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:03.755 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:03.784 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:03.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:03.886 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:03.933 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:03.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:04.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:04.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:04.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:04.290 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:04.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:04.431 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:04.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:04.602 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:04.704 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:04.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[93921]: pgmap v1657: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 3955/50817 objects degraded (7.783%); 11058/50817 objects misplaced (21.760%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:44:04.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:44:04.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:44:04.728 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:44:04.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[98970]: pgmap v1657: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 3955/50817 objects degraded (7.783%); 11058/50817 objects misplaced (21.760%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:44:04.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:44:04.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:44:04.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:04 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:44:04.730 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:04 smithi171 ceph-mon[90031]: pgmap v1657: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 3955/50817 objects degraded (7.783%); 11058/50817 objects misplaced (21.760%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:44:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:04 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:44:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:04 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:44:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:04 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:44:04.832 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:04.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:04.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:04.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:04.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:05.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:05.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:05.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:05.133 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:44:05.133 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185371 8.9 8:99e2cb75:::benchmark_data_smithi027_306999_object58592:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.370454+0000", 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.75948952200002, 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8903825300000001, 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.134 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185371 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370454+0000", 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370454+0000", 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370455+0000", 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.418e-06 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370460+0000", 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3050000000000003e-06 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370470+0000", 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0985000000000001e-05 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370479+0000", 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4130000000000006e-06 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.372877+0000", 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023978839999999999 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.372886+0000", 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9719999999999998e-06 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088666+0000", 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.71577981300000004 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088679+0000", 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3108999999999999e-05 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.260749+0000", 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1720699159999999 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.260836+0000", 2024-08-29T15:44:05.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7714999999999996e-05 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185372 8.9 8:940ebccb:::benchmark_data_smithi027_306999_object58593:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.370493+0000", 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.75945055099999, 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8913060079999999, 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.140 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185372 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370493+0000", 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370493+0000", 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370493+0000", 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8700000000000001e-07 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370495+0000", 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2529999999999999e-06 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370502+0000", 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0280000000000003e-06 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370509+0000", 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.302e-06 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.372916+0000", 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024068029999999999 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.372922+0000", 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2199999999999997e-06 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089272+0000", 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.71635068000000002 2024-08-29T15:44:05.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089281+0000", 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8890000000000008e-06 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.261701+0000", 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.172419257 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.261799+0000", 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8189000000000005e-05 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185377 8.9 8:9a1cbc7f:::benchmark_data_smithi027_306999_object58598:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.370518+0000", 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.759425206, 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.920266687, 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:05.146 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185377 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370518+0000", 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370518+0000", 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370519+0000", 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4000000000000002e-07 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370521+0000", 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.261e-06 2024-08-29T15:44:05.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370527+0000", 2024-08-29T15:44:05.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4350000000000002e-06 2024-08-29T15:44:05.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.370533+0000", 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3750000000000002e-06 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.373287+0000", 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027538979999999998 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.373295+0000", 2024-08-29T15:44:05.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2369999999999992e-06 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089348+0000", 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.71605298500000003 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089356+0000", 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5089999999999993e-06 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.261943+0000", 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.172587136 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.262061+0000", 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011786300000000001 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.290622+0000", 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.028560380999999999 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.290785+0000", 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000163167 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185382 8.9 8:963393a8:::benchmark_data_smithi027_306999_object58603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.383001+0000", 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.74694214499999, 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.896804597, 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.153 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185382 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383001+0000", 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383001+0000", 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383003+0000", 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5659999999999999e-06 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383007+0000", 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1620000000000001e-06 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383017+0000", 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6660000000000003e-06 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383025+0000", 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1459999999999996e-06 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383576+0000", 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00055122000000000005 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.383587+0000", 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1122e-05 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089383+0000", 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70579622399999997 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089391+0000", 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8099999999999998e-06 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.262093+0000", 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.172702286 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.262218+0000", 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012447900000000001 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.262363+0000", 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144945 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.278989+0000", 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016625808999999998 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279162+0000", 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017337200000000001 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279387+0000", 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000225279 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279623+0000", 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023578 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279683+0000", 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0353999999999998e-05 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279806+0000", 2024-08-29T15:44:05.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012237700000000001 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185384 8.9 8:9b98cedb:::benchmark_data_smithi027_306999_object58605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.391100+0000", 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.73884380599998, 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8889052400000002, 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.161 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185384 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391100+0000", 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391100+0000", 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391101+0000", 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.88e-06 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391108+0000", 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5780000000000004e-06 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391125+0000", 2024-08-29T15:44:05.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7139999999999999e-05 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391134+0000", 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0180000000000007e-06 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391210+0000", 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5560000000000002e-05 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.391219+0000", 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4259999999999992e-06 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089418+0000", 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69819857699999999 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.089426+0000", 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1810000000000001e-06 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.266150+0000", 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1767243490000001 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.266276+0000", 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012538600000000001 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.266402+0000", 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012649499999999999 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279233+0000", 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012831103 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279287+0000", 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4137e-05 2024-08-29T15:44:05.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279430+0000", 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014285200000000001 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279815+0000", 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038471099999999998 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.279863+0000", 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8359000000000002e-05 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.280005+0000", 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141488 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217412 8.9 8:967865d6:::benchmark_data_smithi027_306999_object90633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e384)", 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.060006+0000", 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 346.06993731400001, 2024-08-29T15:44:05.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.542636999, 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217412 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.060006+0000", 2024-08-29T15:44:05.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.060006+0000", 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.060007+0000", 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.04e-06 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.060012+0000", 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5510000000000001e-06 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.060023+0000", 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1901e-05 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.060031+0000", 2024-08-29T15:44:05.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3320000000000003e-06 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.068855+0000", 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088241489999999999 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069011+0000", 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000155945 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:44:05.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069182+0000", 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000171514 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.087836+0000", 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018653104 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.088026+0000", 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000190433 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.091301+0000", 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032752929999999999 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.091316+0000", 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4289e-05 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.602554+0000", 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.511238887 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.602643+0000", 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8560999999999995e-05 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217415 8.9 8:9c8a4a97:::benchmark_data_smithi027_306999_object90636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e384)", 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.067693+0000", 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 346.06225037199999, 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.535307545, 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.176 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.177 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:05.177 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.177 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.177 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.177 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217415 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.067693+0000", 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.067693+0000", 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.067695+0000", 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.973e-06 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.067702+0000", 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6619999999999999e-06 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.067719+0000", 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7887999999999999e-05 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.067728+0000", 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3960000000000006e-06 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.073785+0000", 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060568699999999998 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.073916+0000", 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131624 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.074059+0000", 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014271100000000001 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.093008+0000", 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018948784999999999 2024-08-29T15:44:05.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.093017+0000", 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5880000000000004e-06 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.602926+0000", 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5099085809999999 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.603000+0000", 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4467000000000004e-05 2024-08-29T15:44:05.182 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217416 8.9 8:9c3f2e7a:::benchmark_data_smithi027_306999_object90637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e384)", 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.069505+0000", 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 346.06043848799999, 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.562697982, 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.183 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217416 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069505+0000", 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069505+0000", 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069506+0000", 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.017e-06 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069510+0000", 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7450000000000001e-06 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069521+0000", 2024-08-29T15:44:05.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0986e-05 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.069527+0000", 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5089999999999999e-06 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.079098+0000", 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095712390000000005 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.079458+0000", 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035972499999999998 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,5,6", 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.079885+0000", 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042690700000000002 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.093054+0000", 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013168637 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.094221+0000", 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011674210000000001 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.603056+0000", 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5088348040000001 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.603169+0000", 2024-08-29T15:44:05.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011325800000000001 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632093+0000", 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.028924425 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632203+0000", 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010930900000000001 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.189 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217420 8.9 8:9bd68d39:::benchmark_data_smithi027_306999_object90641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e384)", 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.070851+0000", 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 346.059092808, 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.551884485, 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217420 2024-08-29T15:44:05.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.070851+0000", 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.070851+0000", 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.070852+0000", 2024-08-29T15:44:05.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.070857+0000", 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7119999999999999e-06 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.070871+0000", 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4100999999999999e-05 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.070878+0000", 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0570000000000004e-06 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.094258+0000", 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023380415000000002 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.094266+0000", 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0639999999999994e-06 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.603215+0000", 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5089491160000001 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.603363+0000", 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000148071 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.603524+0000", 2024-08-29T15:44:05.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161049 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614742+0000", 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011217881000000001 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614752+0000", 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0095000000000001e-05 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622548+0000", 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077960319999999996 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622611+0000", 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2407000000000001e-05 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622735+0000", 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012416900000000001 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.196 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217477 8.9 8:93884e0e:::benchmark_data_smithi027_306999_object90698:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.136390+0000", 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.99355323100002, 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4960133689999999, 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217477 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136390+0000", 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136390+0000", 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136391+0000", 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2419999999999999e-06 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136397+0000", 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.75e-06 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136414+0000", 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6549e-05 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136422+0000", 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9719999999999992e-06 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136486+0000", 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4894999999999994e-05 2024-08-29T15:44:05.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.136498+0000", 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1449e-05 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.607929+0000", 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.471430681 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.608025+0000", 2024-08-29T15:44:05.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6750999999999996e-05 2024-08-29T15:44:05.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.608104+0000", 2024-08-29T15:44:05.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8737000000000002e-05 2024-08-29T15:44:05.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614756+0000", 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066522739999999997 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614817+0000", 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0656999999999999e-05 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632246+0000", 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01742896 2024-08-29T15:44:05.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632306+0000", 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0241999999999997e-05 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632403+0000", 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7209999999999999e-05 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217482 8.9 8:9ded9269:::benchmark_data_smithi027_306999_object90703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.145073+0000", 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.98487065699999, 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4875452300000001, 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217482 2024-08-29T15:44:05.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145073+0000", 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145073+0000", 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145074+0000", 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6500000000000001e-06 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145081+0000", 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2750000000000001e-06 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145098+0000", 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7714999999999999e-05 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145106+0000", 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8590000000000005e-06 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145177+0000", 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0742999999999997e-05 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.145189+0000", 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2156e-05 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.611283+0000", 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.466094335 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.611419+0000", 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013566300000000001 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.611549+0000", 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012951299999999999 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614834+0000", 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032858200000000001 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614857+0000", 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2371999999999999e-05 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632491+0000", 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017634170000000001 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632521+0000", 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0210999999999999e-05 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632618+0000", 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6748000000000003e-05 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.212 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217491 8.9 8:9ae9b65f:::benchmark_data_smithi027_306999_object90712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.157940+0000", 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.97200286200001, 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.474897428, 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217491 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.157940+0000", 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.157940+0000", 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.157942+0000", 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.499e-06 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.157947+0000", 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8600000000000001e-06 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.157966+0000", 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9332000000000001e-05 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.157975+0000", 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4929999999999998e-06 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.158035+0000", 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0304999999999999e-05 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.158044+0000", 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1039999999999996e-06 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614886+0000", 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.456841603 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.614968+0000", 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1865000000000003e-05 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.615045+0000", 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7514000000000004e-05 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622486+0000", 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074411099999999999 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622750+0000", 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000263961 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632668+0000", 2024-08-29T15:44:05.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099177180000000007 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632758+0000", 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0202000000000001e-05 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632838+0000", 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9862000000000002e-05 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217556 8.9 8:9fc74889:::benchmark_data_smithi027_306999_object90777:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.228574+0000", 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.90136958800002, 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4140007960000001, 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217556 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.220 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228574+0000", 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228574+0000", 2024-08-29T15:44:05.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228575+0000", 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2500000000000004e-07 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228579+0000", 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5410000000000002e-06 2024-08-29T15:44:05.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228588+0000", 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3379999999999994e-06 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228596+0000", 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1430000000000001e-06 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228725+0000", 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129278 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.228737+0000", 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1980999999999999e-05 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.618462+0000", 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.389725168 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.618600+0000", 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013779299999999999 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.618751+0000", 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015098499999999999 2024-08-29T15:44:05.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622810+0000", 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040592620000000001 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.631713+0000", 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089031579999999996 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642402+0000", 2024-08-29T15:44:05.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010689147 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642469+0000", 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7063999999999994e-05 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642575+0000", 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010511299999999999 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.228 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217606 8.9 8:90e0ca1d:::benchmark_data_smithi027_306999_object90827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.300597+0000", 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.82934631500001, 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3421819799999999, 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.229 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217606 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300597+0000", 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300597+0000", 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300599+0000", 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5600000000000001e-06 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300605+0000", 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1800000000000001e-06 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300617+0000", 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2169e-05 2024-08-29T15:44:05.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300624+0000", 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4919999999999996e-06 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300688+0000", 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.347e-05 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.300696+0000", 2024-08-29T15:44:05.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3089999999999996e-06 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622858+0000", 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3221615280000001 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.622982+0000", 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012459099999999999 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.623115+0000", 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013309199999999999 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632195+0000", 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090796220000000007 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632445+0000", 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024993999999999998 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642621+0000", 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010175828 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642690+0000", 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8806999999999994e-05 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642779+0000", 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9392000000000001e-05 2024-08-29T15:44:05.235 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217624 8.9 8:9c385285:::benchmark_data_smithi027_306999_object90845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.323462+0000", 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.80648103599998, 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.319759321, 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.236 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217624 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323462+0000", 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323462+0000", 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323464+0000", 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.869e-06 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323470+0000", 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0630000000000002e-06 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323486+0000", 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5347000000000001e-05 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323494+0000", 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7430000000000008e-06 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323588+0000", 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3689000000000004e-05 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.323597+0000", 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0550000000000005e-06 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.627207+0000", 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3036096859999999 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.627345+0000", 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013818399999999999 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.627501+0000", 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015575699999999999 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632458+0000", 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049577579999999996 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.632989+0000", 2024-08-29T15:44:05.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00053028099999999996 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642977+0000", 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099885819999999993 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643124+0000", 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000147174 2024-08-29T15:44:05.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643222+0000", 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7133000000000007e-05 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217630 8.9 8:93e3ac71:::benchmark_data_smithi027_306999_object90851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.328168+0000", 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.801775384, 2024-08-29T15:44:05.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3152574000000001, 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217630 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328168+0000", 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328168+0000", 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328169+0000", 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.455e-06 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328175+0000", 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1989999999999997e-06 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328192+0000", 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7566000000000001e-05 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328200+0000", 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8990000000000001e-06 2024-08-29T15:44:05.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328270+0000", 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9752999999999994e-05 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.328278+0000", 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1620000000000008e-06 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.633029+0000", 2024-08-29T15:44:05.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.304751376 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.633129+0000", 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9734000000000004e-05 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.633234+0000", 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010477 2024-08-29T15:44:05.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.642827+0000", 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095931739999999995 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643233+0000", 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00040558199999999998 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643273+0000", 2024-08-29T15:44:05.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9988999999999999e-05 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643335+0000", 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2052999999999997e-05 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643425+0000", 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0687999999999996e-05 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:217671 8.9 8:98b98ac2:::benchmark_data_smithi027_306999_object90892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e385)", 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:19.386775+0000", 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 345.743168685, 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.284072812, 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 217671 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.251 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386775+0000", 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386775+0000", 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386776+0000", 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.705e-06 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386783+0000", 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0169999999999999e-06 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386801+0000", 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7838000000000001e-05 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386808+0000", 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0779999999999998e-06 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386877+0000", 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8597999999999996e-05 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:19.386883+0000", 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0220000000000001e-06 2024-08-29T15:44:05.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643665+0000", 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2567824759999999 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643772+0000", 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106264 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.643896+0000", 2024-08-29T15:44:05.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012474799999999999 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.660850+0000", 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016953922 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.665724+0000", 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048736550000000002 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.670683+0000", 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049595059999999998 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.670737+0000", 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3168000000000003e-05 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:20.670847+0000", 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110815 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.257 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:103174 9.8 9:1d4ea0cf:::benchmark_data_smithi027_341149_object744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:41:30.176621+0000", 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 154.95332249000001, 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35729601599999999, 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 103174 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.258 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.176621+0000", 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.176621+0000", 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.176623+0000", 2024-08-29T15:44:05.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.277e-06 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.176631+0000", 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7039999999999994e-06 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.176652+0000", 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1430999999999999e-05 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.176662+0000", 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9769999999999995e-06 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.388744+0000", 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.212081521 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.390136+0000", 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013919340000000001 2024-08-29T15:44:05.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.416334+0000", 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.026198069000000001 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.416429+0000", 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4813000000000002e-05 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:44:05.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.416550+0000", 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121897 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.428517+0000", 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011966248000000001 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.529746+0000", 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10122893299999999 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.529802+0000", 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6082000000000003e-05 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.529872+0000", 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0706999999999999e-05 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:30.533917+0000", 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040444230000000001 2024-08-29T15:44:05.264 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:129700 9.8 9:108f6cce:::benchmark_data_smithi027_341149_object27270:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:20.499876+0000", 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 104.63006687399999, 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35268095399999999, 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.265 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:05.266 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:05.266 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 129700 2024-08-29T15:44:05.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.266 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.499876+0000", 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.499876+0000", 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.499878+0000", 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.457e-06 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.499883+0000", 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0180000000000001e-06 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.499897+0000", 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4578e-05 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.499905+0000", 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1980000000000004e-06 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.595996+0000", 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.096091437000000002 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.596817+0000", 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00082067399999999997 2024-08-29T15:44:05.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.756303+0000", 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15948579800000001 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.756920+0000", 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00061718799999999996 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.757080+0000", 2024-08-29T15:44:05.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016000500000000001 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.767055+0000", 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099752899999999995 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.777010+0000", 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099553469999999998 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.848123+0000", 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.071112951999999993 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.848284+0000", 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016064700000000001 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:20.852557+0000", 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042733650000000003 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.272 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:147795 9.8 9:1cf3367c:::benchmark_data_smithi027_341149_object45365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:54.817377+0000", 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 70.312566399000005, 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40733879200000001, 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:05.273 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 147795 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:54.817377+0000", 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:54.817377+0000", 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:05.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:54.817378+0000", 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5290000000000001e-06 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:54.817384+0000", 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0510000000000002e-06 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:54.817397+0000", 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2363e-05 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:54.817405+0000", 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7060000000000004e-06 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.057212+0000", 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.239807886 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.058389+0000", 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011763660000000001 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.161838+0000", 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.103449373 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.161927+0000", 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9165000000000004e-05 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.162051+0000", 2024-08-29T15:44:05.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012375 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.167852+0000", 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058004789999999999 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.177267+0000", 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094154830000000005 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.221025+0000", 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.043757941000000002 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.221077+0000", 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2120000000000002e-05 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:55.224716+0000", 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036385800000000002 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.280 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:05.280 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:05.280 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:44:05.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:05.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:05.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:05.441 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:05.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:05.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:05.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:05.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:05.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:05.681 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:05.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:05.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:05.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:05.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:05.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:05.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:05.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:05.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:05.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:05.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:06.070 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:06.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:06.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:06.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:06.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:06.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:06.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:06.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:06.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:06.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:06.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:06.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:06.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:06.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:06.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:06.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:44:06.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:06.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:06.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:06.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:06 smithi171 ceph-mon[90031]: pgmap v1658: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 3955/50013 objects degraded (7.908%); 11058/50013 objects misplaced (22.110%); 5.5 MiB/s, 87 objects/s recovering 2024-08-29T15:44:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:06 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:44:06.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:06 smithi027 ceph-mon[93921]: pgmap v1658: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 3955/50013 objects degraded (7.908%); 11058/50013 objects misplaced (22.110%); 5.5 MiB/s, 87 objects/s recovering 2024-08-29T15:44:06.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:06 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:44:06.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:06 smithi027 ceph-mon[98970]: pgmap v1658: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 1.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 3955/50013 objects degraded (7.908%); 11058/50013 objects misplaced (22.110%); 5.5 MiB/s, 87 objects/s recovering 2024-08-29T15:44:06.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:06 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:44:06.819 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:06.845 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:06.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:06.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:06.947 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:06.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:06.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:07.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:07.116 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:07.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:07.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:07.217 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:07.241 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:07.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:07.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:07.343 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:07.352 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:07.357 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:07.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:07.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:07.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:07.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:07.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:07.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:07.686 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:07 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:44:07.686 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:07 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3955/50013 objects degraded (7.908%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:07.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:07 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:44:07.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:07 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3955/50013 objects degraded (7.908%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:07.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:07.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:07.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:07.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:07 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:44:07.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:07 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3955/50013 objects degraded (7.908%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:07.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:07.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:07.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:07.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:08.003 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:08.105 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:08.153 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:08.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:08.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:08.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:08.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:08.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:08.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:08.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:08.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:08.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:08.705 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:08 smithi027 ceph-mon[93921]: pgmap v1659: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 910 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3955/43563 objects degraded (9.079%); 10578/43563 objects misplaced (24.282%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:44:08.706 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:44:08.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:08 smithi027 ceph-mon[98970]: pgmap v1659: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 910 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3955/43563 objects degraded (9.079%); 10578/43563 objects misplaced (24.282%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:44:08.711 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:44:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:08 smithi171 ceph-mon[90031]: pgmap v1659: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 910 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 3955/43563 objects degraded (9.079%); 10578/43563 objects misplaced (24.282%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:44:08.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:08.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:08.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:08.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:08.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:09.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:09.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:09.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:09.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:09.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:09.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:09.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:09.269 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:44:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:44:09] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.51.0" 2024-08-29T15:44:09.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:09.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:09.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:09.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:09.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:09.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:09.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:09.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:09.676 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:09.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:09.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:09.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:09.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:09.912 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:09.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:09.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:10.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:10.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:10.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:10.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:10.202 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:10.205 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1659,"stamp":"2024-08-29T15:44:07.067905+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":954537981,"num_objects":14521,"num_object_clones":0,"num_object_copies":43563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3955,"num_objects_misplaced":10578,"num_objects_unfound":0,"num_objects_dirty":14521,"num_whiteouts":0,"num_read":5988,"num_read_kb":13001,"num_write":191271,"num_write_kb":6567954,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25364,"num_bytes_recovered":1658724481,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112846,"ondisk_log_size":112846,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11060520,"kb_used_data":5332200,"kb_used_omap":43,"kb_used_meta":5728020,"kb_avail":738900696,"statfs":{"total":767960285184,"available":756634312704,"internally_reserved":0,"allocated":5460172800,"data_stored":5436700906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":5865492972},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,6,9,8,5,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-625803264,"num_objects":-9549,"num_object_clones":0,"num_object_copies":-28647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2781,"num_objects_unfound":0,"num_objects_dirty":-9549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9549,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1169,"num_bytes_recovered":76611584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005012"},"pg_stats":[{"pgid":"1.11","version":"90'149","reported_seq":848,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.408613+0000","last_change":"2024-08-29T15:43:47.407452+0000","last_active":"2024-08-29T15:43:47.408613+0000","last_peered":"2024-08-29T15:43:47.408613+0000","last_clean":"2024-08-29T15:43:47.408613+0000","last_became_active":"2024-08-29T15:43:47.407022+0000","last_became_peered":"2024-08-29T15:43:47.407022+0000","last_unstale":"2024-08-29T15:43:47.408613+0000","last_undegraded":"2024-08-29T15:43:47.408613+0000","last_fullsized":"2024-08-29T15:43:47.408613+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:23.716435+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:23.716435+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.022267417000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":848,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.434447+0000","last_change":"2024-08-29T15:41:16.221956+0000","last_active":"2024-08-29T15:43:54.434447+0000","last_peered":"2024-08-29T15:43:54.434447+0000","last_clean":"2024-08-29T15:43:54.434447+0000","last_became_active":"2024-08-29T15:41:16.221533+0000","last_became_peered":"2024-08-29T15:41:16.221533+0000","last_unstale":"2024-08-29T15:43:54.434447+0000","last_undegraded":"2024-08-29T15:43:54.434447+0000","last_fullsized":"2024-08-29T15:43:54.434447+0000","mapping_epoch":439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:26:34.947614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":260,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404108+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:43:47.404108+0000","last_peered":"2024-08-29T15:43:47.404108+0000","last_clean":"2024-08-29T15:43:47.404108+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:43:47.404108+0000","last_undegraded":"2024-08-29T15:43:47.404108+0000","last_fullsized":"2024-08-29T15:43:47.404108+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":942,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.440611+0000","last_change":"2024-08-29T15:43:47.427424+0000","last_active":"2024-08-29T15:43:54.440611+0000","last_peered":"2024-08-29T15:43:54.440611+0000","last_clean":"2024-08-29T15:43:54.440611+0000","last_became_active":"2024-08-29T15:43:47.426397+0000","last_became_peered":"2024-08-29T15:43:47.426397+0000","last_unstale":"2024-08-29T15:43:54.440611+0000","last_undegraded":"2024-08-29T15:43:54.440611+0000","last_fullsized":"2024-08-29T15:43:54.440611+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.968809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.968809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:46:15.826260+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":54,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.401552+0000","last_change":"2024-08-29T15:39:53.258205+0000","last_active":"2024-08-29T15:43:47.401552+0000","last_peered":"2024-08-29T15:43:47.401552+0000","last_clean":"2024-08-29T15:43:47.401552+0000","last_became_active":"2024-08-29T15:39:53.257784+0000","last_became_peered":"2024-08-29T15:39:53.257784+0000","last_unstale":"2024-08-29T15:43:47.401552+0000","last_undegraded":"2024-08-29T15:43:47.401552+0000","last_fullsized":"2024-08-29T15:43:47.401552+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:30:42.270982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":704,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.425812+0000","last_change":"2024-08-29T15:43:44.388183+0000","last_active":"2024-08-29T15:43:54.425812+0000","last_peered":"2024-08-29T15:43:54.425812+0000","last_clean":"2024-08-29T15:43:54.425812+0000","last_became_active":"2024-08-29T15:43:44.387812+0000","last_became_peered":"2024-08-29T15:43:44.387812+0000","last_unstale":"2024-08-29T15:43:54.425812+0000","last_undegraded":"2024-08-29T15:43:54.425812+0000","last_fullsized":"2024-08-29T15:43:54.425812+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:06:27.607690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":945,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.446452+0000","last_change":"2024-08-29T15:43:27.615504+0000","last_active":"2024-08-29T15:43:54.446452+0000","last_peered":"2024-08-29T15:43:54.446452+0000","last_clean":"2024-08-29T15:43:54.446452+0000","last_became_active":"2024-08-29T15:43:26.291876+0000","last_became_peered":"2024-08-29T15:43:26.291876+0000","last_unstale":"2024-08-29T15:43:54.446452+0000","last_undegraded":"2024-08-29T15:43:54.446452+0000","last_fullsized":"2024-08-29T15:43:54.446452+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.615446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.615446+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:17:58.749196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":767,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.426497+0000","last_change":"2024-08-29T15:43:44.377372+0000","last_active":"2024-08-29T15:43:54.426497+0000","last_peered":"2024-08-29T15:43:54.426497+0000","last_clean":"2024-08-29T15:43:54.426497+0000","last_became_active":"2024-08-29T15:43:44.377041+0000","last_became_peered":"2024-08-29T15:43:44.377041+0000","last_unstale":"2024-08-29T15:43:54.426497+0000","last_undegraded":"2024-08-29T15:43:54.426497+0000","last_fullsized":"2024-08-29T15:43:54.426497+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:05:58.536193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":253,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404836+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:43:47.404836+0000","last_peered":"2024-08-29T15:43:47.404836+0000","last_clean":"2024-08-29T15:43:47.404836+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:43:47.404836+0000","last_undegraded":"2024-08-29T15:43:47.404836+0000","last_fullsized":"2024-08-29T15:43:47.404836+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":962,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.448380+0000","last_change":"2024-08-29T15:43:26.831164+0000","last_active":"2024-08-29T15:43:54.448380+0000","last_peered":"2024-08-29T15:43:54.448380+0000","last_clean":"2024-08-29T15:43:54.448380+0000","last_became_active":"2024-08-29T15:43:26.280576+0000","last_became_peered":"2024-08-29T15:43:26.280576+0000","last_unstale":"2024-08-29T15:43:54.448380+0000","last_undegraded":"2024-08-29T15:43:54.448380+0000","last_fullsized":"2024-08-29T15:43:54.448380+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.831081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.831081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:08:53.777897+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":547,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.441443+0000","last_change":"2024-08-29T15:41:44.469235+0000","last_active":"2024-08-29T15:43:54.441443+0000","last_peered":"2024-08-29T15:43:54.441443+0000","last_clean":"2024-08-29T15:43:54.441443+0000","last_became_active":"2024-08-29T15:41:44.468857+0000","last_became_peered":"2024-08-29T15:41:44.468857+0000","last_unstale":"2024-08-29T15:43:54.441443+0000","last_undegraded":"2024-08-29T15:43:54.441443+0000","last_fullsized":"2024-08-29T15:43:54.441443+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:22:54.347105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":189,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.423643+0000","last_change":"2024-08-29T15:43:44.389429+0000","last_active":"2024-08-29T15:43:54.423643+0000","last_peered":"2024-08-29T15:43:54.423643+0000","last_clean":"2024-08-29T15:43:54.423643+0000","last_became_active":"2024-08-29T15:43:44.375048+0000","last_became_peered":"2024-08-29T15:43:44.375048+0000","last_unstale":"2024-08-29T15:43:54.423643+0000","last_undegraded":"2024-08-29T15:43:54.423643+0000","last_fullsized":"2024-08-29T15:43:54.423643+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:51:36.427552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":860,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.405138+0000","last_change":"2024-08-29T15:43:29.760340+0000","last_active":"2024-08-29T15:43:47.405138+0000","last_peered":"2024-08-29T15:43:47.405138+0000","last_clean":"2024-08-29T15:43:47.405138+0000","last_became_active":"2024-08-29T15:43:26.257530+0000","last_became_peered":"2024-08-29T15:43:26.257530+0000","last_unstale":"2024-08-29T15:43:47.405138+0000","last_undegraded":"2024-08-29T15:43:47.405138+0000","last_fullsized":"2024-08-29T15:43:47.405138+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:43:29.760194+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:26.122447+0000","scrub_duration":0.001769706,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.5","version":"467'12138","reported_seq":24914,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:05.400114+0000","last_change":"2024-08-29T15:43:28.205037+0000","last_active":"2024-08-29T15:44:05.400114+0000","last_peered":"2024-08-29T15:44:05.400114+0000","last_clean":"2024-08-29T15:44:05.400114+0000","last_became_active":"2024-08-29T15:43:28.181460+0000","last_became_peered":"2024-08-29T15:43:28.181460+0000","last_unstale":"2024-08-29T15:44:05.400114+0000","last_undegraded":"2024-08-29T15:44:05.400114+0000","last_fullsized":"2024-08-29T15:44:05.400114+0000","mapping_epoch":459,"log_start":"432'4913","ondisk_log_start":"432'4913","created":420,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7225,"log_dups_size":0,"ondisk_log_size":7225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:18:06.209113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12138,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6329,"num_bytes_recovered":414777344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":296,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.405516+0000","last_change":"2024-08-29T15:43:44.376246+0000","last_active":"2024-08-29T15:43:47.405516+0000","last_peered":"2024-08-29T15:43:47.405516+0000","last_clean":"2024-08-29T15:43:47.405516+0000","last_became_active":"2024-08-29T15:43:44.371925+0000","last_became_peered":"2024-08-29T15:43:44.371925+0000","last_unstale":"2024-08-29T15:43:47.405516+0000","last_undegraded":"2024-08-29T15:43:47.405516+0000","last_fullsized":"2024-08-29T15:43:47.405516+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:08:57.521364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.399315+0000","last_change":"2024-08-29T15:39:53.258473+0000","last_active":"2024-08-29T15:43:47.399315+0000","last_peered":"2024-08-29T15:43:47.399315+0000","last_clean":"2024-08-29T15:43:47.399315+0000","last_became_active":"2024-08-29T15:39:53.258055+0000","last_became_peered":"2024-08-29T15:39:53.258055+0000","last_unstale":"2024-08-29T15:43:47.399315+0000","last_undegraded":"2024-08-29T15:43:47.399315+0000","last_fullsized":"2024-08-29T15:43:47.399315+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:38:54.441868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'149","reported_seq":978,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.431818+0000","last_change":"2024-08-29T15:43:28.773489+0000","last_active":"2024-08-29T15:43:54.431818+0000","last_peered":"2024-08-29T15:43:54.431818+0000","last_clean":"2024-08-29T15:43:54.431818+0000","last_became_active":"2024-08-29T15:43:26.286702+0000","last_became_peered":"2024-08-29T15:43:26.286702+0000","last_unstale":"2024-08-29T15:43:54.431818+0000","last_undegraded":"2024-08-29T15:43:54.431818+0000","last_fullsized":"2024-08-29T15:43:54.431818+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:28.773404+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:28.773404+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:36:58.854149+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.4","version":"467'11897","reported_seq":12003,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:06.369512+0000","last_change":"2024-08-29T15:39:37.186986+0000","last_active":"2024-08-29T15:44:06.369512+0000","last_peered":"2024-08-29T15:44:06.369512+0000","last_clean":"2024-08-29T15:44:06.369512+0000","last_became_active":"2024-08-29T15:39:37.186669+0000","last_became_peered":"2024-08-29T15:39:37.186669+0000","last_unstale":"2024-08-29T15:44:06.369512+0000","last_undegraded":"2024-08-29T15:44:06.369512+0000","last_fullsized":"2024-08-29T15:44:06.369512+0000","mapping_epoch":420,"log_start":"432'5619","ondisk_log_start":"432'5619","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:33:01.221948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11897,"num_write_kb":408000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":257,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.405521+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:43:47.405521+0000","last_peered":"2024-08-29T15:43:47.405521+0000","last_clean":"2024-08-29T15:43:47.405521+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:43:47.405521+0000","last_undegraded":"2024-08-29T15:43:47.405521+0000","last_fullsized":"2024-08-29T15:43:47.405521+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":290,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404605+0000","last_change":"2024-08-29T15:41:44.436101+0000","last_active":"2024-08-29T15:43:47.404605+0000","last_peered":"2024-08-29T15:43:47.404605+0000","last_clean":"2024-08-29T15:43:47.404605+0000","last_became_active":"2024-08-29T15:41:44.435769+0000","last_became_peered":"2024-08-29T15:41:44.435769+0000","last_unstale":"2024-08-29T15:43:47.404605+0000","last_undegraded":"2024-08-29T15:43:47.404605+0000","last_fullsized":"2024-08-29T15:43:47.404605+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:20:19.269808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":876,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.402153+0000","last_change":"2024-08-29T15:43:26.252268+0000","last_active":"2024-08-29T15:43:47.402153+0000","last_peered":"2024-08-29T15:43:47.402153+0000","last_clean":"2024-08-29T15:43:47.402153+0000","last_became_active":"2024-08-29T15:43:26.251916+0000","last_became_peered":"2024-08-29T15:43:26.251916+0000","last_unstale":"2024-08-29T15:43:47.402153+0000","last_undegraded":"2024-08-29T15:43:47.402153+0000","last_fullsized":"2024-08-29T15:43:47.402153+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:16.688893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:16.688893+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.016432537000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.7","version":"467'11665","reported_seq":26025,"reported_epoch":467,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-29T15:44:02.422148+0000","last_change":"2024-08-29T15:43:44.402856+0000","last_active":"2024-08-29T15:44:02.422148+0000","last_peered":"2024-08-29T15:44:02.422148+0000","last_clean":"2024-08-29T15:41:03.137460+0000","last_became_active":"2024-08-29T15:41:45.687083+0000","last_became_peered":"2024-08-29T15:41:45.687083+0000","last_unstale":"2024-08-29T15:44:02.422148+0000","last_undegraded":"2024-08-29T15:44:02.422148+0000","last_fullsized":"2024-08-29T15:41:45.435192+0000","mapping_epoch":446,"log_start":"432'4598","ondisk_log_start":"432'4598","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:21:14.334460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67698688,"num_objects":1033,"num_object_clones":0,"num_object_copies":3099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1905,"num_objects_unfound":0,"num_objects_dirty":1033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11665,"num_write_kb":406336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4762,"num_bytes_recovered":312016896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"2,3,5","objects":159},{"shards":"2,5","objects":1},{"shards":"3,5,7","objects":873}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":58,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.401251+0000","last_change":"2024-08-29T15:39:53.262167+0000","last_active":"2024-08-29T15:43:47.401251+0000","last_peered":"2024-08-29T15:43:47.401251+0000","last_clean":"2024-08-29T15:43:47.401251+0000","last_became_active":"2024-08-29T15:39:53.258406+0000","last_became_peered":"2024-08-29T15:39:53.258406+0000","last_unstale":"2024-08-29T15:43:47.401251+0000","last_undegraded":"2024-08-29T15:43:47.401251+0000","last_fullsized":"2024-08-29T15:43:47.401251+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:05:19.731559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":428,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.472327+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:43:54.472327+0000","last_peered":"2024-08-29T15:43:54.472327+0000","last_clean":"2024-08-29T15:43:54.472327+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:43:54.472327+0000","last_undegraded":"2024-08-29T15:43:54.472327+0000","last_fullsized":"2024-08-29T15:43:54.472327+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":777,"reported_epoch":467,"state":"active+clean+scrubbing","last_fresh":"2024-08-29T15:44:06.660195+0000","last_change":"2024-08-29T15:43:54.601484+0000","last_active":"2024-08-29T15:44:06.660195+0000","last_peered":"2024-08-29T15:44:06.660195+0000","last_clean":"2024-08-29T15:44:06.660195+0000","last_became_active":"2024-08-29T15:43:47.411562+0000","last_became_peered":"2024-08-29T15:43:47.411562+0000","last_unstale":"2024-08-29T15:44:06.660195+0000","last_undegraded":"2024-08-29T15:44:06.660195+0000","last_fullsized":"2024-08-29T15:44:06.660195+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:29.877467+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:29.877467+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"scrubbing for 0s","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.6","version":"467'11733","reported_seq":11791,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:02.313562+0000","last_change":"2024-08-29T15:39:37.376667+0000","last_active":"2024-08-29T15:44:02.313562+0000","last_peered":"2024-08-29T15:44:02.313562+0000","last_clean":"2024-08-29T15:44:02.313562+0000","last_became_active":"2024-08-29T15:39:37.376372+0000","last_became_peered":"2024-08-29T15:39:37.376372+0000","last_unstale":"2024-08-29T15:44:02.313562+0000","last_undegraded":"2024-08-29T15:44:02.313562+0000","last_fullsized":"2024-08-29T15:44:02.313562+0000","mapping_epoch":420,"log_start":"432'4658","ondisk_log_start":"432'4658","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:52:02.092878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66650112,"num_objects":1017,"num_object_clones":0,"num_object_copies":3051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11733,"num_write_kb":408000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":361,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.400063+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:43:47.400063+0000","last_peered":"2024-08-29T15:43:47.400063+0000","last_clean":"2024-08-29T15:43:47.400063+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:43:47.400063+0000","last_undegraded":"2024-08-29T15:43:47.400063+0000","last_fullsized":"2024-08-29T15:43:47.400063+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":807,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.449675+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:43:54.449675+0000","last_peered":"2024-08-29T15:43:54.449675+0000","last_clean":"2024-08-29T15:43:54.449675+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:43:54.449675+0000","last_undegraded":"2024-08-29T15:43:54.449675+0000","last_fullsized":"2024-08-29T15:43:54.449675+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"467'12025","reported_seq":12132,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:06.633173+0000","last_change":"2024-08-29T15:39:37.311470+0000","last_active":"2024-08-29T15:44:06.633173+0000","last_peered":"2024-08-29T15:44:06.633173+0000","last_clean":"2024-08-29T15:44:06.633173+0000","last_became_active":"2024-08-29T15:39:37.310913+0000","last_became_peered":"2024-08-29T15:39:37.310913+0000","last_unstale":"2024-08-29T15:44:06.633173+0000","last_undegraded":"2024-08-29T15:44:06.633173+0000","last_fullsized":"2024-08-29T15:44:06.633173+0000","mapping_epoch":420,"log_start":"432'5485","ondisk_log_start":"432'5485","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:26:39.550827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54984704,"num_objects":839,"num_object_clones":0,"num_object_copies":2517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12025,"num_write_kb":411648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":932,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.423450+0000","last_change":"2024-08-29T15:37:18.101543+0000","last_active":"2024-08-29T15:43:54.423450+0000","last_peered":"2024-08-29T15:43:54.423450+0000","last_clean":"2024-08-29T15:43:54.423450+0000","last_became_active":"2024-08-29T15:37:01.308140+0000","last_became_peered":"2024-08-29T15:37:01.308140+0000","last_unstale":"2024-08-29T15:43:54.423450+0000","last_undegraded":"2024-08-29T15:43:54.423450+0000","last_fullsized":"2024-08-29T15:43:54.423450+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:18.101472+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:18.101472+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:41.710800+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":834,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.427140+0000","last_change":"2024-08-29T15:43:44.395045+0000","last_active":"2024-08-29T15:43:54.427140+0000","last_peered":"2024-08-29T15:43:54.427140+0000","last_clean":"2024-08-29T15:43:54.427140+0000","last_became_active":"2024-08-29T15:43:44.382969+0000","last_became_peered":"2024-08-29T15:43:44.382969+0000","last_unstale":"2024-08-29T15:43:54.427140+0000","last_undegraded":"2024-08-29T15:43:54.427140+0000","last_fullsized":"2024-08-29T15:43:54.427140+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:41.129154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":253,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.403519+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:43:47.403519+0000","last_peered":"2024-08-29T15:43:47.403519+0000","last_clean":"2024-08-29T15:43:47.403519+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:43:47.403519+0000","last_undegraded":"2024-08-29T15:43:47.403519+0000","last_fullsized":"2024-08-29T15:43:47.403519+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"467'11674","reported_seq":11779,"reported_epoch":467,"state":"active+recovery_wait+remapped","last_fresh":"2024-08-29T15:44:06.428378+0000","last_change":"2024-08-29T15:41:45.679775+0000","last_active":"2024-08-29T15:44:06.428378+0000","last_peered":"2024-08-29T15:44:06.428378+0000","last_clean":"2024-08-29T15:41:43.386876+0000","last_became_active":"2024-08-29T15:41:45.666366+0000","last_became_peered":"2024-08-29T15:41:45.666366+0000","last_unstale":"2024-08-29T15:44:06.428378+0000","last_undegraded":"2024-08-29T15:44:06.428378+0000","last_fullsized":"2024-08-29T15:44:06.428378+0000","mapping_epoch":446,"log_start":"432'5347","ondisk_log_start":"432'5347","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54657024,"num_objects":834,"num_object_clones":0,"num_object_copies":2502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6254,"num_objects_unfound":0,"num_objects_dirty":834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11674,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":-4903}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":906,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.424037+0000","last_change":"2024-08-29T15:41:03.142904+0000","last_active":"2024-08-29T15:43:54.424037+0000","last_peered":"2024-08-29T15:43:54.424037+0000","last_clean":"2024-08-29T15:43:54.424037+0000","last_became_active":"2024-08-29T15:41:03.142474+0000","last_became_peered":"2024-08-29T15:41:03.142474+0000","last_unstale":"2024-08-29T15:43:54.424037+0000","last_undegraded":"2024-08-29T15:43:54.424037+0000","last_fullsized":"2024-08-29T15:43:54.424037+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:40:21.404385+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:40:21.404385+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:00:36.540063+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":249,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.403598+0000","last_change":"2024-08-29T15:41:44.443161+0000","last_active":"2024-08-29T15:43:47.403598+0000","last_peered":"2024-08-29T15:43:47.403598+0000","last_clean":"2024-08-29T15:43:47.403598+0000","last_became_active":"2024-08-29T15:41:44.442573+0000","last_became_peered":"2024-08-29T15:41:44.442573+0000","last_unstale":"2024-08-29T15:43:47.403598+0000","last_undegraded":"2024-08-29T15:43:47.403598+0000","last_fullsized":"2024-08-29T15:43:47.403598+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:31:08.897327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":253,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.405057+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:43:47.405057+0000","last_peered":"2024-08-29T15:43:47.405057+0000","last_clean":"2024-08-29T15:43:47.405057+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:43:47.405057+0000","last_undegraded":"2024-08-29T15:43:47.405057+0000","last_fullsized":"2024-08-29T15:43:47.405057+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":945,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.446286+0000","last_change":"2024-08-29T15:43:28.644229+0000","last_active":"2024-08-29T15:43:54.446286+0000","last_peered":"2024-08-29T15:43:54.446286+0000","last_clean":"2024-08-29T15:43:54.446286+0000","last_became_active":"2024-08-29T15:43:26.297926+0000","last_became_peered":"2024-08-29T15:43:26.297926+0000","last_unstale":"2024-08-29T15:43:54.446286+0000","last_undegraded":"2024-08-29T15:43:54.446286+0000","last_fullsized":"2024-08-29T15:43:54.446286+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:28.644138+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:28.644138+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:14:35.236420+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"467'12010","reported_seq":12117,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:06.720763+0000","last_change":"2024-08-29T15:39:37.258693+0000","last_active":"2024-08-29T15:44:06.720763+0000","last_peered":"2024-08-29T15:44:06.720763+0000","last_clean":"2024-08-29T15:44:06.720763+0000","last_became_active":"2024-08-29T15:39:37.258303+0000","last_became_peered":"2024-08-29T15:39:37.258303+0000","last_unstale":"2024-08-29T15:44:06.720763+0000","last_undegraded":"2024-08-29T15:44:06.720763+0000","last_fullsized":"2024-08-29T15:44:06.720763+0000","mapping_epoch":420,"log_start":"432'5439","ondisk_log_start":"432'5439","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6571,"log_dups_size":0,"ondisk_log_size":6571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:53:38.911068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12010,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":254,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404682+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:43:47.404682+0000","last_peered":"2024-08-29T15:43:47.404682+0000","last_clean":"2024-08-29T15:43:47.404682+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:43:47.404682+0000","last_undegraded":"2024-08-29T15:43:47.404682+0000","last_fullsized":"2024-08-29T15:43:47.404682+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":486,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.405535+0000","last_change":"2024-08-29T15:41:00.747697+0000","last_active":"2024-08-29T15:43:47.405535+0000","last_peered":"2024-08-29T15:43:47.405535+0000","last_clean":"2024-08-29T15:43:47.405535+0000","last_became_active":"2024-08-29T15:41:00.747350+0000","last_became_peered":"2024-08-29T15:41:00.747350+0000","last_unstale":"2024-08-29T15:43:47.405535+0000","last_undegraded":"2024-08-29T15:43:47.405535+0000","last_fullsized":"2024-08-29T15:43:47.405535+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:13:16.221880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"467'11939","reported_seq":16594,"reported_epoch":467,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-29T15:44:06.407360+0000","last_change":"2024-08-29T15:43:44.384853+0000","last_active":"2024-08-29T15:44:06.407360+0000","last_peered":"2024-08-29T15:44:06.407360+0000","last_clean":"2024-08-29T15:41:02.137059+0000","last_became_active":"2024-08-29T15:41:17.378848+0000","last_became_peered":"2024-08-29T15:41:17.378848+0000","last_unstale":"2024-08-29T15:44:06.407360+0000","last_undegraded":"2024-08-29T15:41:17.245306+0000","last_fullsized":"2024-08-29T15:41:17.211749+0000","mapping_epoch":440,"log_start":"432'5680","ondisk_log_start":"432'5680","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51314688,"num_objects":783,"num_object_clones":0,"num_object_copies":2349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3947,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11939,"num_write_kb":407104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2273,"num_bytes_recovered":148963328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":-3164},{"shards":"1,2,3","objects":642}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":823,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.400923+0000","last_change":"2024-08-29T15:37:24.681545+0000","last_active":"2024-08-29T15:43:47.400923+0000","last_peered":"2024-08-29T15:43:47.400923+0000","last_clean":"2024-08-29T15:43:47.400923+0000","last_became_active":"2024-08-29T15:37:01.312787+0000","last_became_peered":"2024-08-29T15:37:01.312787+0000","last_unstale":"2024-08-29T15:43:47.400923+0000","last_undegraded":"2024-08-29T15:43:47.400923+0000","last_fullsized":"2024-08-29T15:43:47.400923+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:24.681392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:24.681392+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:01:45.435539+0000","scrub_duration":0.0064019280000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":333,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.447890+0000","last_change":"2024-08-29T15:41:44.435967+0000","last_active":"2024-08-29T15:43:54.447890+0000","last_peered":"2024-08-29T15:43:54.447890+0000","last_clean":"2024-08-29T15:43:54.447890+0000","last_became_active":"2024-08-29T15:41:44.435570+0000","last_became_peered":"2024-08-29T15:41:44.435570+0000","last_unstale":"2024-08-29T15:43:54.447890+0000","last_undegraded":"2024-08-29T15:43:54.447890+0000","last_fullsized":"2024-08-29T15:43:54.447890+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:57:19.832668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":797,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.441615+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:43:54.441615+0000","last_peered":"2024-08-29T15:43:54.441615+0000","last_clean":"2024-08-29T15:43:54.441615+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:43:54.441615+0000","last_undegraded":"2024-08-29T15:43:54.441615+0000","last_fullsized":"2024-08-29T15:43:54.441615+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"467'11983","reported_seq":28720,"reported_epoch":467,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-29T15:44:05.457084+0000","last_change":"2024-08-29T15:43:44.405415+0000","last_active":"2024-08-29T15:44:05.457084+0000","last_peered":"2024-08-29T15:44:05.457084+0000","last_clean":"2024-08-29T15:40:59.730315+0000","last_became_active":"2024-08-29T15:41:24.597966+0000","last_became_peered":"2024-08-29T15:41:24.597966+0000","last_unstale":"2024-08-29T15:44:05.457084+0000","last_undegraded":"2024-08-29T15:44:05.457084+0000","last_fullsized":"2024-08-29T15:41:19.476039+0000","mapping_epoch":435,"log_start":"432'4784","ondisk_log_start":"432'4784","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7199,"log_dups_size":0,"ondisk_log_size":7199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59179008,"num_objects":903,"num_object_clones":0,"num_object_copies":2709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1444,"num_objects_unfound":0,"num_objects_dirty":903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11983,"num_write_kb":412352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5542,"num_bytes_recovered":363200512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,1","objects":181},{"shards":"0,3,5","objects":722}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":942,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.446945+0000","last_change":"2024-08-29T15:43:47.426690+0000","last_active":"2024-08-29T15:43:54.446945+0000","last_peered":"2024-08-29T15:43:54.446945+0000","last_clean":"2024-08-29T15:43:54.446945+0000","last_became_active":"2024-08-29T15:43:47.426071+0000","last_became_peered":"2024-08-29T15:43:47.426071+0000","last_unstale":"2024-08-29T15:43:54.446945+0000","last_undegraded":"2024-08-29T15:43:54.446945+0000","last_fullsized":"2024-08-29T15:43:54.446945+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.008533+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.008533+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:08:31.158303+0000","scrub_duration":111,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":946,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.399779+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:43:47.399779+0000","last_peered":"2024-08-29T15:43:47.399779+0000","last_clean":"2024-08-29T15:43:47.399779+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:43:47.399779+0000","last_undegraded":"2024-08-29T15:43:47.399779+0000","last_fullsized":"2024-08-29T15:43:47.399779+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":211,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.425707+0000","last_change":"2024-08-29T15:37:26.313495+0000","last_active":"2024-08-29T15:43:54.425707+0000","last_peered":"2024-08-29T15:43:54.425707+0000","last_clean":"2024-08-29T15:43:54.425707+0000","last_became_active":"2024-08-29T15:37:26.313188+0000","last_became_peered":"2024-08-29T15:37:26.313188+0000","last_unstale":"2024-08-29T15:43:54.425707+0000","last_undegraded":"2024-08-29T15:43:54.425707+0000","last_fullsized":"2024-08-29T15:43:54.425707+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:39.580490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":220,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404917+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:43:47.404917+0000","last_peered":"2024-08-29T15:43:47.404917+0000","last_clean":"2024-08-29T15:43:47.404917+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:43:47.404917+0000","last_undegraded":"2024-08-29T15:43:47.404917+0000","last_fullsized":"2024-08-29T15:43:47.404917+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"467'11831","reported_seq":11939,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-29T15:44:02.413049+0000","last_change":"2024-08-29T15:43:45.538693+0000","last_active":"2024-08-29T15:44:02.413049+0000","last_peered":"2024-08-29T15:44:02.413049+0000","last_clean":"2024-08-29T15:43:43.291894+0000","last_became_active":"2024-08-29T15:43:45.478421+0000","last_became_peered":"2024-08-29T15:43:45.478421+0000","last_unstale":"2024-08-29T15:44:02.413049+0000","last_undegraded":"2024-08-29T15:44:02.413049+0000","last_fullsized":"2024-08-29T15:44:02.413049+0000","mapping_epoch":463,"log_start":"432'4842","ondisk_log_start":"432'4842","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6989,"log_dups_size":0,"ondisk_log_size":6989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:29.619953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63897600,"num_objects":975,"num_object_clones":0,"num_object_copies":2925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":975,"num_objects_unfound":0,"num_objects_dirty":975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11831,"num_write_kb":409792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":975}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":945,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.446617+0000","last_change":"2024-08-29T15:43:26.588054+0000","last_active":"2024-08-29T15:43:54.446617+0000","last_peered":"2024-08-29T15:43:54.446617+0000","last_clean":"2024-08-29T15:43:54.446617+0000","last_became_active":"2024-08-29T15:43:26.285267+0000","last_became_peered":"2024-08-29T15:43:26.285267+0000","last_unstale":"2024-08-29T15:43:54.446617+0000","last_undegraded":"2024-08-29T15:43:54.446617+0000","last_fullsized":"2024-08-29T15:43:54.446617+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.587986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.587986+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:32:31.983890+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1270,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.447662+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:43:54.447662+0000","last_peered":"2024-08-29T15:43:54.447662+0000","last_clean":"2024-08-29T15:43:54.447662+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:43:54.447662+0000","last_undegraded":"2024-08-29T15:43:54.447662+0000","last_fullsized":"2024-08-29T15:43:54.447662+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":829,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.448532+0000","last_change":"2024-08-29T15:41:44.432130+0000","last_active":"2024-08-29T15:43:54.448532+0000","last_peered":"2024-08-29T15:43:54.448532+0000","last_clean":"2024-08-29T15:43:54.448532+0000","last_became_active":"2024-08-29T15:41:44.431835+0000","last_became_peered":"2024-08-29T15:41:44.431835+0000","last_unstale":"2024-08-29T15:43:54.448532+0000","last_undegraded":"2024-08-29T15:43:54.448532+0000","last_fullsized":"2024-08-29T15:43:54.448532+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:07.995950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":487,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.445930+0000","last_change":"2024-08-29T15:43:44.385638+0000","last_active":"2024-08-29T15:43:54.445930+0000","last_peered":"2024-08-29T15:43:54.445930+0000","last_clean":"2024-08-29T15:43:54.445930+0000","last_became_active":"2024-08-29T15:43:44.385042+0000","last_became_peered":"2024-08-29T15:43:44.385042+0000","last_unstale":"2024-08-29T15:43:54.445930+0000","last_undegraded":"2024-08-29T15:43:54.445930+0000","last_fullsized":"2024-08-29T15:43:54.445930+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:19:48.932998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"467'11831","reported_seq":11885,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:02.396422+0000","last_change":"2024-08-29T15:39:53.258131+0000","last_active":"2024-08-29T15:44:02.396422+0000","last_peered":"2024-08-29T15:44:02.396422+0000","last_clean":"2024-08-29T15:44:02.396422+0000","last_became_active":"2024-08-29T15:39:53.257746+0000","last_became_peered":"2024-08-29T15:39:53.257746+0000","last_unstale":"2024-08-29T15:44:02.396422+0000","last_undegraded":"2024-08-29T15:44:02.396422+0000","last_fullsized":"2024-08-29T15:44:02.396422+0000","mapping_epoch":424,"log_start":"432'4805","ondisk_log_start":"432'4805","created":420,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:34:48.746732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65339392,"num_objects":997,"num_object_clones":0,"num_object_copies":2991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11831,"num_write_kb":410496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":963,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.448594+0000","last_change":"2024-08-29T15:43:27.841159+0000","last_active":"2024-08-29T15:43:54.448594+0000","last_peered":"2024-08-29T15:43:54.448594+0000","last_clean":"2024-08-29T15:43:54.448594+0000","last_became_active":"2024-08-29T15:43:26.281150+0000","last_became_peered":"2024-08-29T15:43:26.281150+0000","last_unstale":"2024-08-29T15:43:54.448594+0000","last_undegraded":"2024-08-29T15:43:54.448594+0000","last_fullsized":"2024-08-29T15:43:54.448594+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.841106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.841106+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:07:22.911625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":839,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.427281+0000","last_change":"2024-08-29T15:37:29.345555+0000","last_active":"2024-08-29T15:43:54.427281+0000","last_peered":"2024-08-29T15:43:54.427281+0000","last_clean":"2024-08-29T15:43:54.427281+0000","last_became_active":"2024-08-29T15:37:29.344738+0000","last_became_peered":"2024-08-29T15:37:29.344738+0000","last_unstale":"2024-08-29T15:43:54.427281+0000","last_undegraded":"2024-08-29T15:43:54.427281+0000","last_fullsized":"2024-08-29T15:43:54.427281+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:14.103195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":206,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.406876+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:43:47.406876+0000","last_peered":"2024-08-29T15:43:47.406876+0000","last_clean":"2024-08-29T15:43:47.406876+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:43:47.406876+0000","last_undegraded":"2024-08-29T15:43:47.406876+0000","last_fullsized":"2024-08-29T15:43:47.406876+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":380,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.401960+0000","last_change":"2024-08-29T15:43:44.373358+0000","last_active":"2024-08-29T15:43:47.401960+0000","last_peered":"2024-08-29T15:43:47.401960+0000","last_clean":"2024-08-29T15:43:47.401960+0000","last_became_active":"2024-08-29T15:43:44.372934+0000","last_became_peered":"2024-08-29T15:43:44.372934+0000","last_unstale":"2024-08-29T15:43:47.401960+0000","last_undegraded":"2024-08-29T15:43:47.401960+0000","last_fullsized":"2024-08-29T15:43:47.401960+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:42.042366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"467'11915","reported_seq":12022,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:05.452254+0000","last_change":"2024-08-29T15:39:37.258623+0000","last_active":"2024-08-29T15:44:05.452254+0000","last_peered":"2024-08-29T15:44:05.452254+0000","last_clean":"2024-08-29T15:44:05.452254+0000","last_became_active":"2024-08-29T15:39:37.258321+0000","last_became_peered":"2024-08-29T15:39:37.258321+0000","last_unstale":"2024-08-29T15:44:05.452254+0000","last_undegraded":"2024-08-29T15:44:05.452254+0000","last_fullsized":"2024-08-29T15:44:05.452254+0000","mapping_epoch":420,"log_start":"432'4673","ondisk_log_start":"432'4673","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7242,"log_dups_size":0,"ondisk_log_size":7242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:09:19.337857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11915,"num_write_kb":407872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":848,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.408652+0000","last_change":"2024-08-29T15:43:47.408026+0000","last_active":"2024-08-29T15:43:47.408652+0000","last_peered":"2024-08-29T15:43:47.408652+0000","last_clean":"2024-08-29T15:43:47.408652+0000","last_became_active":"2024-08-29T15:43:47.407472+0000","last_became_peered":"2024-08-29T15:43:47.407472+0000","last_unstale":"2024-08-29T15:43:47.408652+0000","last_undegraded":"2024-08-29T15:43:47.408652+0000","last_fullsized":"2024-08-29T15:43:47.408652+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:23.716435+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:23.716435+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.022267417000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":414,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.401007+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:43:47.401007+0000","last_peered":"2024-08-29T15:43:47.401007+0000","last_clean":"2024-08-29T15:43:47.401007+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:43:47.401007+0000","last_undegraded":"2024-08-29T15:43:47.401007+0000","last_fullsized":"2024-08-29T15:43:47.401007+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":254,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404037+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:43:47.404037+0000","last_peered":"2024-08-29T15:43:47.404037+0000","last_clean":"2024-08-29T15:43:47.404037+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:43:47.404037+0000","last_undegraded":"2024-08-29T15:43:47.404037+0000","last_fullsized":"2024-08-29T15:43:47.404037+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":321,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.399801+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:43:47.399801+0000","last_peered":"2024-08-29T15:43:47.399801+0000","last_clean":"2024-08-29T15:43:47.399801+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:43:47.399801+0000","last_undegraded":"2024-08-29T15:43:47.399801+0000","last_fullsized":"2024-08-29T15:43:47.399801+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.f","version":"467'11799","reported_seq":11856,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:02.361558+0000","last_change":"2024-08-29T15:39:37.185853+0000","last_active":"2024-08-29T15:44:02.361558+0000","last_peered":"2024-08-29T15:44:02.361558+0000","last_clean":"2024-08-29T15:44:02.361558+0000","last_became_active":"2024-08-29T15:39:37.185573+0000","last_became_peered":"2024-08-29T15:39:37.185573+0000","last_unstale":"2024-08-29T15:44:02.361558+0000","last_undegraded":"2024-08-29T15:44:02.361558+0000","last_fullsized":"2024-08-29T15:44:02.361558+0000","mapping_epoch":420,"log_start":"432'4753","ondisk_log_start":"432'4753","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":7046,"log_dups_size":0,"ondisk_log_size":7046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:12.924373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68485120,"num_objects":1045,"num_object_clones":0,"num_object_copies":3135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11799,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":877,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.401778+0000","last_change":"2024-08-29T15:43:26.251585+0000","last_active":"2024-08-29T15:43:47.401778+0000","last_peered":"2024-08-29T15:43:47.401778+0000","last_clean":"2024-08-29T15:43:47.401778+0000","last_became_active":"2024-08-29T15:43:26.251218+0000","last_became_peered":"2024-08-29T15:43:26.251218+0000","last_unstale":"2024-08-29T15:43:47.401778+0000","last_undegraded":"2024-08-29T15:43:47.401778+0000","last_fullsized":"2024-08-29T15:43:47.401778+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:16.688893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:16.688893+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.016432537000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":346,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.401481+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:43:47.401481+0000","last_peered":"2024-08-29T15:43:47.401481+0000","last_clean":"2024-08-29T15:43:47.401481+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:43:47.401481+0000","last_undegraded":"2024-08-29T15:43:47.401481+0000","last_fullsized":"2024-08-29T15:43:47.401481+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1235,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.446975+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:43:54.446975+0000","last_peered":"2024-08-29T15:43:54.446975+0000","last_clean":"2024-08-29T15:43:54.446975+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:43:54.446975+0000","last_undegraded":"2024-08-29T15:43:54.446975+0000","last_fullsized":"2024-08-29T15:43:54.446975+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":99,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.399359+0000","last_change":"2024-08-29T15:41:16.222377+0000","last_active":"2024-08-29T15:43:47.399359+0000","last_peered":"2024-08-29T15:43:47.399359+0000","last_clean":"2024-08-29T15:43:47.399359+0000","last_became_active":"2024-08-29T15:41:16.221775+0000","last_became_peered":"2024-08-29T15:41:16.221775+0000","last_unstale":"2024-08-29T15:43:47.399359+0000","last_undegraded":"2024-08-29T15:43:47.399359+0000","last_fullsized":"2024-08-29T15:43:47.399359+0000","mapping_epoch":439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:43:52.584991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"467'11881","reported_seq":11937,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:04.728870+0000","last_change":"2024-08-29T15:39:37.258863+0000","last_active":"2024-08-29T15:44:04.728870+0000","last_peered":"2024-08-29T15:44:04.728870+0000","last_clean":"2024-08-29T15:44:04.728870+0000","last_became_active":"2024-08-29T15:39:37.258296+0000","last_became_peered":"2024-08-29T15:39:37.258296+0000","last_unstale":"2024-08-29T15:44:04.728870+0000","last_undegraded":"2024-08-29T15:44:04.728870+0000","last_fullsized":"2024-08-29T15:44:04.728870+0000","mapping_epoch":420,"log_start":"432'5562","ondisk_log_start":"432'5562","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6319,"log_dups_size":0,"ondisk_log_size":6319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:32:01.990004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295424,"num_objects":859,"num_object_clones":0,"num_object_copies":2577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11881,"num_write_kb":407680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":777,"reported_epoch":467,"state":"active+clean+scrubbing","last_fresh":"2024-08-29T15:44:05.609006+0000","last_change":"2024-08-29T15:43:59.618923+0000","last_active":"2024-08-29T15:44:05.609006+0000","last_peered":"2024-08-29T15:44:05.609006+0000","last_clean":"2024-08-29T15:44:05.609006+0000","last_became_active":"2024-08-29T15:43:47.412069+0000","last_became_peered":"2024-08-29T15:43:47.412069+0000","last_unstale":"2024-08-29T15:44:05.609006+0000","last_undegraded":"2024-08-29T15:44:05.609006+0000","last_fullsized":"2024-08-29T15:44:05.609006+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:28.875520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:28.875520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"scrubbing for 0s","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":817,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.446881+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:43:54.446881+0000","last_peered":"2024-08-29T15:43:54.446881+0000","last_clean":"2024-08-29T15:43:54.446881+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:43:54.446881+0000","last_undegraded":"2024-08-29T15:43:54.446881+0000","last_fullsized":"2024-08-29T15:43:54.446881+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":933,"reported_epoch":466,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-29T15:43:47.407078+0000","last_change":"2024-08-29T15:43:44.383944+0000","last_active":"2024-08-29T15:43:47.407078+0000","last_peered":"2024-08-29T15:43:47.407078+0000","last_clean":"2024-08-29T15:43:33.319091+0000","last_became_active":"2024-08-29T15:43:44.383379+0000","last_became_peered":"2024-08-29T15:43:44.383379+0000","last_unstale":"2024-08-29T15:43:47.407078+0000","last_undegraded":"2024-08-29T15:43:44.371483+0000","last_fullsized":"2024-08-29T15:43:47.407078+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:28:34.359806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":808,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.449527+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:43:54.449527+0000","last_peered":"2024-08-29T15:43:54.449527+0000","last_clean":"2024-08-29T15:43:54.449527+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:43:54.449527+0000","last_undegraded":"2024-08-29T15:43:54.449527+0000","last_fullsized":"2024-08-29T15:43:54.449527+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"467'11850","reported_seq":24485,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:02.407181+0000","last_change":"2024-08-29T15:42:20.219072+0000","last_active":"2024-08-29T15:44:02.407181+0000","last_peered":"2024-08-29T15:44:02.407181+0000","last_clean":"2024-08-29T15:44:02.407181+0000","last_became_active":"2024-08-29T15:42:20.200986+0000","last_became_peered":"2024-08-29T15:42:20.200986+0000","last_unstale":"2024-08-29T15:44:02.407181+0000","last_undegraded":"2024-08-29T15:44:02.407181+0000","last_fullsized":"2024-08-29T15:44:02.407181+0000","mapping_epoch":449,"log_start":"432'4866","ondisk_log_start":"432'4866","created":420,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:03:18.829519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091928,"num_objects":1040,"num_object_clones":0,"num_object_copies":3120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1040,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11850,"num_write_kb":412417,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6273,"num_bytes_recovered":411107328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":860,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404631+0000","last_change":"2024-08-29T15:43:27.819648+0000","last_active":"2024-08-29T15:43:47.404631+0000","last_peered":"2024-08-29T15:43:47.404631+0000","last_clean":"2024-08-29T15:43:47.404631+0000","last_became_active":"2024-08-29T15:43:26.257696+0000","last_became_peered":"2024-08-29T15:43:26.257696+0000","last_unstale":"2024-08-29T15:43:47.404631+0000","last_undegraded":"2024-08-29T15:43:47.404631+0000","last_fullsized":"2024-08-29T15:43:47.404631+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.819558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.819558+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:03:19.953478+0000","scrub_duration":0.0013041389999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":802,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.430904+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:43:54.430904+0000","last_peered":"2024-08-29T15:43:54.430904+0000","last_clean":"2024-08-29T15:43:54.430904+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:43:54.430904+0000","last_undegraded":"2024-08-29T15:43:54.430904+0000","last_fullsized":"2024-08-29T15:43:54.430904+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1453,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.445676+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:43:54.445676+0000","last_peered":"2024-08-29T15:43:54.445676+0000","last_clean":"2024-08-29T15:43:54.445676+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:43:54.445676+0000","last_undegraded":"2024-08-29T15:43:54.445676+0000","last_fullsized":"2024-08-29T15:43:54.445676+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":140,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.423956+0000","last_change":"2024-08-29T15:43:44.383123+0000","last_active":"2024-08-29T15:43:54.423956+0000","last_peered":"2024-08-29T15:43:54.423956+0000","last_clean":"2024-08-29T15:43:54.423956+0000","last_became_active":"2024-08-29T15:43:44.382729+0000","last_became_peered":"2024-08-29T15:43:44.382729+0000","last_unstale":"2024-08-29T15:43:54.423956+0000","last_undegraded":"2024-08-29T15:43:54.423956+0000","last_fullsized":"2024-08-29T15:43:54.423956+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:14:30.890056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"467'12192","reported_seq":12298,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:44:06.671346+0000","last_change":"2024-08-29T15:39:37.259666+0000","last_active":"2024-08-29T15:44:06.671346+0000","last_peered":"2024-08-29T15:44:06.671346+0000","last_clean":"2024-08-29T15:44:06.671346+0000","last_became_active":"2024-08-29T15:39:37.259380+0000","last_became_peered":"2024-08-29T15:39:37.259380+0000","last_unstale":"2024-08-29T15:44:06.671346+0000","last_undegraded":"2024-08-29T15:44:06.671346+0000","last_fullsized":"2024-08-29T15:44:06.671346+0000","mapping_epoch":420,"log_start":"432'5655","ondisk_log_start":"432'5655","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:39:36.173829+0000","last_clean_scrub_stamp":"2024-08-29T15:39:36.173829+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:02:18.132955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53739520,"num_objects":820,"num_object_clones":0,"num_object_copies":2460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12192,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":977,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.431729+0000","last_change":"2024-08-29T15:43:26.775058+0000","last_active":"2024-08-29T15:43:54.431729+0000","last_peered":"2024-08-29T15:43:54.431729+0000","last_clean":"2024-08-29T15:43:54.431729+0000","last_became_active":"2024-08-29T15:43:26.284121+0000","last_became_peered":"2024-08-29T15:43:26.284121+0000","last_unstale":"2024-08-29T15:43:54.431729+0000","last_undegraded":"2024-08-29T15:43:54.431729+0000","last_fullsized":"2024-08-29T15:43:54.431729+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.774861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.774861+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:59:41.253206+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":176,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.445387+0000","last_change":"2024-08-29T15:43:44.393371+0000","last_active":"2024-08-29T15:43:54.445387+0000","last_peered":"2024-08-29T15:43:54.445387+0000","last_clean":"2024-08-29T15:43:54.445387+0000","last_became_active":"2024-08-29T15:43:44.387134+0000","last_became_peered":"2024-08-29T15:43:44.387134+0000","last_unstale":"2024-08-29T15:43:54.445387+0000","last_undegraded":"2024-08-29T15:43:54.445387+0000","last_fullsized":"2024-08-29T15:43:54.445387+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:33.515571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":93,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.426627+0000","last_change":"2024-08-29T15:41:00.761163+0000","last_active":"2024-08-29T15:43:54.426627+0000","last_peered":"2024-08-29T15:43:54.426627+0000","last_clean":"2024-08-29T15:43:54.426627+0000","last_became_active":"2024-08-29T15:41:00.749422+0000","last_became_peered":"2024-08-29T15:41:00.749422+0000","last_unstale":"2024-08-29T15:43:54.426627+0000","last_undegraded":"2024-08-29T15:43:54.426627+0000","last_fullsized":"2024-08-29T15:43:54.426627+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:21.012113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404759+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:43:47.404759+0000","last_peered":"2024-08-29T15:43:47.404759+0000","last_clean":"2024-08-29T15:43:47.404759+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:43:47.404759+0000","last_undegraded":"2024-08-29T15:43:47.404759+0000","last_fullsized":"2024-08-29T15:43:47.404759+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":253,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404232+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:43:47.404232+0000","last_peered":"2024-08-29T15:43:47.404232+0000","last_clean":"2024-08-29T15:43:47.404232+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:43:47.404232+0000","last_undegraded":"2024-08-29T15:43:47.404232+0000","last_fullsized":"2024-08-29T15:43:47.404232+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":263,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.403604+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:43:47.403604+0000","last_peered":"2024-08-29T15:43:47.403604+0000","last_clean":"2024-08-29T15:43:47.403604+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:43:47.403604+0000","last_undegraded":"2024-08-29T15:43:47.403604+0000","last_fullsized":"2024-08-29T15:43:47.403604+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.399285+0000","last_change":"2024-08-29T15:43:44.375331+0000","last_active":"2024-08-29T15:43:47.399285+0000","last_peered":"2024-08-29T15:43:47.399285+0000","last_clean":"2024-08-29T15:43:47.399285+0000","last_became_active":"2024-08-29T15:43:44.375027+0000","last_became_peered":"2024-08-29T15:43:44.375027+0000","last_unstale":"2024-08-29T15:43:47.399285+0000","last_undegraded":"2024-08-29T15:43:47.399285+0000","last_fullsized":"2024-08-29T15:43:47.399285+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:04:37.416657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":564,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.426115+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:43:54.426115+0000","last_peered":"2024-08-29T15:43:54.426115+0000","last_clean":"2024-08-29T15:43:54.426115+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:43:54.426115+0000","last_undegraded":"2024-08-29T15:43:54.426115+0000","last_fullsized":"2024-08-29T15:43:54.426115+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'149","reported_seq":977,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.434548+0000","last_change":"2024-08-29T15:43:27.778054+0000","last_active":"2024-08-29T15:43:54.434548+0000","last_peered":"2024-08-29T15:43:54.434548+0000","last_clean":"2024-08-29T15:43:54.434548+0000","last_became_active":"2024-08-29T15:43:26.284501+0000","last_became_peered":"2024-08-29T15:43:26.284501+0000","last_unstale":"2024-08-29T15:43:54.434548+0000","last_undegraded":"2024-08-29T15:43:54.434548+0000","last_fullsized":"2024-08-29T15:43:54.434548+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.778004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.778004+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:21:07.158496+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":342,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.400565+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:43:47.400565+0000","last_peered":"2024-08-29T15:43:47.400565+0000","last_clean":"2024-08-29T15:43:47.400565+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:43:47.400565+0000","last_undegraded":"2024-08-29T15:43:47.400565+0000","last_fullsized":"2024-08-29T15:43:47.400565+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":765,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.431748+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:43:54.431748+0000","last_peered":"2024-08-29T15:43:54.431748+0000","last_clean":"2024-08-29T15:43:54.431748+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:43:54.431748+0000","last_undegraded":"2024-08-29T15:43:54.431748+0000","last_fullsized":"2024-08-29T15:43:54.431748+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":284,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.402670+0000","last_change":"2024-08-29T15:43:44.372361+0000","last_active":"2024-08-29T15:43:47.402670+0000","last_peered":"2024-08-29T15:43:47.402670+0000","last_clean":"2024-08-29T15:43:47.402670+0000","last_became_active":"2024-08-29T15:43:44.371982+0000","last_became_peered":"2024-08-29T15:43:44.371982+0000","last_unstale":"2024-08-29T15:43:47.402670+0000","last_undegraded":"2024-08-29T15:43:47.402670+0000","last_fullsized":"2024-08-29T15:43:47.402670+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:38:20.070361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":817,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.424118+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:43:54.424118+0000","last_peered":"2024-08-29T15:43:54.424118+0000","last_clean":"2024-08-29T15:43:54.424118+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:43:54.424118+0000","last_undegraded":"2024-08-29T15:43:54.424118+0000","last_fullsized":"2024-08-29T15:43:54.424118+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":324,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.402588+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:43:47.402588+0000","last_peered":"2024-08-29T15:43:47.402588+0000","last_clean":"2024-08-29T15:43:47.402588+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:43:47.402588+0000","last_undegraded":"2024-08-29T15:43:47.402588+0000","last_fullsized":"2024-08-29T15:43:47.402588+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":61,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.424180+0000","last_change":"2024-08-29T15:43:44.374180+0000","last_active":"2024-08-29T15:43:54.424180+0000","last_peered":"2024-08-29T15:43:54.424180+0000","last_clean":"2024-08-29T15:43:54.424180+0000","last_became_active":"2024-08-29T15:43:44.373767+0000","last_became_peered":"2024-08-29T15:43:44.373767+0000","last_unstale":"2024-08-29T15:43:54.424180+0000","last_undegraded":"2024-08-29T15:43:54.424180+0000","last_fullsized":"2024-08-29T15:43:54.424180+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:07.501547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":778,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.445526+0000","last_change":"2024-08-29T15:43:44.384584+0000","last_active":"2024-08-29T15:43:54.445526+0000","last_peered":"2024-08-29T15:43:54.445526+0000","last_clean":"2024-08-29T15:43:54.445526+0000","last_became_active":"2024-08-29T15:43:44.383996+0000","last_became_peered":"2024-08-29T15:43:44.383996+0000","last_unstale":"2024-08-29T15:43:54.445526+0000","last_undegraded":"2024-08-29T15:43:54.445526+0000","last_fullsized":"2024-08-29T15:43:54.445526+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:01:14.990354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":796,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.440520+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:43:54.440520+0000","last_peered":"2024-08-29T15:43:54.440520+0000","last_clean":"2024-08-29T15:43:54.440520+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:43:54.440520+0000","last_undegraded":"2024-08-29T15:43:54.440520+0000","last_fullsized":"2024-08-29T15:43:54.440520+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":801,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.426015+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:43:54.426015+0000","last_peered":"2024-08-29T15:43:54.426015+0000","last_clean":"2024-08-29T15:43:54.426015+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:43:54.426015+0000","last_undegraded":"2024-08-29T15:43:54.426015+0000","last_fullsized":"2024-08-29T15:43:54.426015+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":252,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.403673+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:43:47.403673+0000","last_peered":"2024-08-29T15:43:47.403673+0000","last_clean":"2024-08-29T15:43:47.403673+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:43:47.403673+0000","last_undegraded":"2024-08-29T15:43:47.403673+0000","last_fullsized":"2024-08-29T15:43:47.403673+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":842,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.447105+0000","last_change":"2024-08-29T15:39:11.832387+0000","last_active":"2024-08-29T15:43:54.447105+0000","last_peered":"2024-08-29T15:43:54.447105+0000","last_clean":"2024-08-29T15:43:54.447105+0000","last_became_active":"2024-08-29T15:39:11.832056+0000","last_became_peered":"2024-08-29T15:39:11.832056+0000","last_unstale":"2024-08-29T15:43:54.447105+0000","last_undegraded":"2024-08-29T15:43:54.447105+0000","last_fullsized":"2024-08-29T15:43:54.447105+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:47:42.049015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":130,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.404307+0000","last_change":"2024-08-29T15:43:44.378845+0000","last_active":"2024-08-29T15:43:47.404307+0000","last_peered":"2024-08-29T15:43:47.404307+0000","last_clean":"2024-08-29T15:43:47.404307+0000","last_became_active":"2024-08-29T15:43:44.378491+0000","last_became_peered":"2024-08-29T15:43:44.378491+0000","last_unstale":"2024-08-29T15:43:47.404307+0000","last_undegraded":"2024-08-29T15:43:47.404307+0000","last_fullsized":"2024-08-29T15:43:47.404307+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:13:25.137263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":279,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.403970+0000","last_change":"2024-08-29T15:41:44.446288+0000","last_active":"2024-08-29T15:43:47.403970+0000","last_peered":"2024-08-29T15:43:47.403970+0000","last_clean":"2024-08-29T15:43:47.403970+0000","last_became_active":"2024-08-29T15:41:44.445958+0000","last_became_peered":"2024-08-29T15:41:44.445958+0000","last_unstale":"2024-08-29T15:43:47.403970+0000","last_undegraded":"2024-08-29T15:43:47.403970+0000","last_fullsized":"2024-08-29T15:43:47.403970+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:47:53.474043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":385,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.399987+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:43:47.399987+0000","last_peered":"2024-08-29T15:43:47.399987+0000","last_clean":"2024-08-29T15:43:47.399987+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:43:47.399987+0000","last_undegraded":"2024-08-29T15:43:47.399987+0000","last_fullsized":"2024-08-29T15:43:47.399987+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":252,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.403745+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:43:47.403745+0000","last_peered":"2024-08-29T15:43:47.403745+0000","last_clean":"2024-08-29T15:43:47.403745+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:43:47.403745+0000","last_undegraded":"2024-08-29T15:43:47.403745+0000","last_fullsized":"2024-08-29T15:43:47.403745+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":385,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.402412+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:43:47.402412+0000","last_peered":"2024-08-29T15:43:47.402412+0000","last_clean":"2024-08-29T15:43:47.402412+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:43:47.402412+0000","last_undegraded":"2024-08-29T15:43:47.402412+0000","last_fullsized":"2024-08-29T15:43:47.402412+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":832,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.427456+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:43:54.427456+0000","last_peered":"2024-08-29T15:43:54.427456+0000","last_clean":"2024-08-29T15:43:54.427456+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:43:54.427456+0000","last_undegraded":"2024-08-29T15:43:54.427456+0000","last_fullsized":"2024-08-29T15:43:54.427456+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":835,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.426575+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:43:54.426575+0000","last_peered":"2024-08-29T15:43:54.426575+0000","last_clean":"2024-08-29T15:43:54.426575+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:43:54.426575+0000","last_undegraded":"2024-08-29T15:43:54.426575+0000","last_fullsized":"2024-08-29T15:43:54.426575+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":253,"reported_epoch":467,"state":"active+clean","last_fresh":"2024-08-29T15:43:54.430994+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:43:54.430994+0000","last_peered":"2024-08-29T15:43:54.430994+0000","last_clean":"2024-08-29T15:43:54.430994+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:43:54.430994+0000","last_undegraded":"2024-08-29T15:43:54.430994+0000","last_fullsized":"2024-08-29T15:43:54.430994+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":291,"reported_epoch":466,"state":"active+clean","last_fresh":"2024-08-29T15:43:47.402077+0000","last_change":"2024-08-29T15:43:44.391961+0000","last_active":"2024-08-29T15:43:47.402077+0000","last_peered":"2024-08-29T15:43:47.402077+0000","last_clean":"2024-08-29T15:43:47.402077+0000","last_became_active":"2024-08-29T15:43:44.375278+0000","last_became_peered":"2024-08-29T15:43:44.375278+0000","last_unstale":"2024-08-29T15:43:47.402077+0000","last_undegraded":"2024-08-29T15:43:47.402077+0000","last_fullsized":"2024-08-29T15:43:47.402077+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:51:42.693350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":949747736,"num_objects":14493,"num_object_clones":0,"num_object_copies":43479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3947,"num_objects_misplaced":10578,"num_objects_unfound":0,"num_objects_dirty":14493,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190363,"num_write_kb":6555329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25179,"num_bytes_recovered":1650065408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5327630336,"data_stored":5327618120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108684,"ondisk_log_size":108684,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":7758,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":8388627,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234593,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2015156,"kb_used_data":1181740,"kb_used_omap":1,"kb_used_meta":833406,"kb_avail":91729996,"statfs":{"total":95995035648,"available":93931515904,"internally_reserved":0,"allocated":1210101760,"data_stored":1207155544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":853408203},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496224,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":706864,"kb_used_data":180264,"kb_used_omap":1,"kb_used_meta":526590,"kb_avail":93038288,"statfs":{"total":95995035648,"available":95271206912,"internally_reserved":0,"allocated":184590336,"data_stored":181648988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":539228615},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426810,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1502824,"kb_used_data":938272,"kb_used_omap":8,"kb_used_meta":564535,"kb_avail":92242328,"statfs":{"total":95995035648,"available":94456143872,"internally_reserved":0,"allocated":960790528,"data_stored":957852060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":578084746},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019503,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":960176,"kb_used_data":243240,"kb_used_omap":1,"kb_used_meta":716926,"kb_avail":92784976,"statfs":{"total":95995035648,"available":95011815424,"internally_reserved":0,"allocated":249077760,"data_stored":246135367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":734132678},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435688,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2300896,"kb_used_data":1443224,"kb_used_omap":4,"kb_used_meta":857659,"kb_avail":91444256,"statfs":{"total":95995035648,"available":93638918144,"internally_reserved":0,"allocated":1477861376,"data_stored":1474931360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":878243081},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903539,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":821788,"kb_used_data":198548,"kb_used_omap":10,"kb_used_meta":623157,"kb_avail":92923364,"statfs":{"total":95995035648,"available":95153524736,"internally_reserved":0,"allocated":203313152,"data_stored":200386578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":638113296},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762741,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1807832,"kb_used_data":909712,"kb_used_omap":7,"kb_used_meta":897976,"kb_avail":91937320,"statfs":{"total":95995035648,"available":94143815680,"internally_reserved":0,"allocated":931545088,"data_stored":928621504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":919527778},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,2,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926264,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":944984,"kb_used_data":237200,"kb_used_omap":7,"kb_used_meta":707768,"kb_avail":92800168,"statfs":{"total":95995035648,"available":95027372032,"internally_reserved":0,"allocated":242892800,"data_stored":239969505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":724754575},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,5,2,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":224657408,"data_stored":224657408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":906100736,"data_stored":906100736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":193986560,"data_stored":193986560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1470038016,"data_stored":1470038016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":237899776,"data_stored":237895704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":941297664,"data_stored":941293592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":171704320,"data_stored":171704320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1181945856,"data_stored":1181941784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:44:10.211 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-29T15:44:10.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool set .mgr pg_num 31 2024-08-29T15:44:10.222 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:10.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:10.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:10.365 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:10.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:10.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:10.493 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:10.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:10.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:10.595 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:10.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:10.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:10.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:10.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:10.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:10 smithi027 ceph-mon[93921]: from='client.56650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:44:10.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:10 smithi027 ceph-mon[93921]: pgmap v1660: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 824 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 3955/39411 objects degraded (10.035%); 9869/39411 objects misplaced (25.041%); 6.1 MiB/s, 98 objects/s recovering 2024-08-29T15:44:10.765 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:10 smithi027 ceph-mon[98970]: from='client.56650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:44:10.765 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:10 smithi027 ceph-mon[98970]: pgmap v1660: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 824 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 3955/39411 objects degraded (10.035%); 9869/39411 objects misplaced (25.041%); 6.1 MiB/s, 98 objects/s recovering 2024-08-29T15:44:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:10 smithi171 ceph-mon[90031]: from='client.56650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:44:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:10 smithi171 ceph-mon[90031]: pgmap v1660: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 824 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 3955/39411 objects degraded (10.035%); 9869/39411 objects misplaced (25.041%); 6.1 MiB/s, 98 objects/s recovering 2024-08-29T15:44:10.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:10.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:10.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:10.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:10.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:10.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:10.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:11.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:11.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:11.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:11.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:11.295 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:11.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:11.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:11.433 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:11.461 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:11.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:11.597 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:11.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:11.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:11 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:44:11.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:11 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:44:11.698 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:11.725 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:11.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:11.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:11.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:11.958 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:11.987 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:11.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:11.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:12.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:11 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:44:12.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:12.128 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:12.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:12.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:12.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:12.259 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:12.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:12.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:12.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:12.410 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:12.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:12.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:12.566 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:12.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:12.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:12.697 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:12.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:12.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:12.703 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:44:12.703 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:44:12.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:44:12.738 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:44:12.773 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:12 smithi027 ceph-mon[98970]: pgmap v1661: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 767 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3955/36666 objects degraded (10.787%); 9384/36666 objects misplaced (25.593%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:44:12.773 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:12 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3955/36666 objects degraded (10.787%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:12.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:12 smithi027 ceph-mon[93921]: pgmap v1661: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 767 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3955/36666 objects degraded (10.787%); 9384/36666 objects misplaced (25.593%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:44:12.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:12 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3955/36666 objects degraded (10.787%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:12.778 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:44:12.778 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:44:12.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:12.812 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:44:12.812 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:44:12.847 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:44:12.848 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:44:12.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:12.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:12.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:12.884 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:44:12.884 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:44:12.922 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:44:12.923 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:44:12.953 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:12.957 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:44:12.957 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:44:12.999 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:12 smithi171 ceph-mon[90031]: pgmap v1661: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 767 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 3955/36666 objects degraded (10.787%); 9384/36666 objects misplaced (25.593%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:44:13.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:12 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3955/36666 objects degraded (10.787%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:13.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:13.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:13.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:13.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:13.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:13.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:13.519 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:13.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:13.547 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:44:13.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:13.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:13.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:13.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:13.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:13.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:13.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1328902463' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1254588857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1328902463' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1254588857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:44:13.817 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:13 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:44:13.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:13.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:13.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:13.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1328902463' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:44:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:13 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:44:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1254588857' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:44:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:13 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:44:14.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:14.074 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:14.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:14.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:14.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:14.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:14.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:14.340 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:14.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:14.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:14.487 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:14.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:14.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:14.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:14.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:14.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:14.683 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[93921]: pgmap v1662: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 591 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 3955/28236 objects degraded (14.007%); 8678/28236 objects misplaced (30.734%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[93921]: osdmap e468: 8 total, 8 up, 8 in 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[98970]: pgmap v1662: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 591 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 3955/28236 objects degraded (14.007%); 8678/28236 objects misplaced (30.734%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[98970]: osdmap e468: 8 total, 8 up, 8 in 2024-08-29T15:44:14.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:14.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:14.777 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:14.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:14.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:14.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:14.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:15.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:14 smithi171 ceph-mon[90031]: pgmap v1662: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 591 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 3955/28236 objects degraded (14.007%); 8678/28236 objects misplaced (30.734%); 6.2 MiB/s, 99 objects/s recovering 2024-08-29T15:44:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:14 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:44:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:14 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:44:15.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:14 smithi171 ceph-mon[90031]: osdmap e468: 8 total, 8 up, 8 in 2024-08-29T15:44:15.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:15.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:15.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:15.136 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:15.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:15.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:15.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:15.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:15.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:15.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:15.351 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:15.410 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:15.440 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:15.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:15.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:15.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:15.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:15.695 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:15.723 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:15.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:15.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:15.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:15.855 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:44:15.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:15.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:15.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:15.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:16.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:16.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:16.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:16.142 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:16.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:16.221 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:44:16.221 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:44:16.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:16.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:16.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:16.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:16.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:16.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:16.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:16.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:16.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:16 smithi027 ceph-mon[93921]: pgmap v1664: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 576 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3955/27537 objects degraded (14.362%); 8678/27537 objects misplaced (31.514%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:44:16.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:16.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:16.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:16.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:16.734 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:16.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:16.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:16 smithi027 ceph-mon[98970]: pgmap v1664: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 576 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3955/27537 objects degraded (14.362%); 8678/27537 objects misplaced (31.514%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:44:16.836 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:16.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:16.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:16.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:16.972 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:17.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:17.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:17.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:16 smithi171 ceph-mon[90031]: pgmap v1664: 109 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 101 active+clean; 576 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3955/27537 objects degraded (14.362%); 8678/27537 objects misplaced (31.514%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:44:17.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:17.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:17.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:17.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:17.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:17.268 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:17.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:17.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:44:17.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:17.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:44:17.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:17.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:17.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:17.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:17.596 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:17.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:17.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:17.698 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:17.727 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:17.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:17.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:17 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:44:17.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:17 smithi027 ceph-mon[93921]: 1.6 scrub ok 2024-08-29T15:44:17.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:17 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3955/27537 objects degraded (14.362%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:17.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:17 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:44:17.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:17 smithi027 ceph-mon[98970]: 1.6 scrub ok 2024-08-29T15:44:17.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:17 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3955/27537 objects degraded (14.362%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:17.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:17.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:17.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:17.892 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:17.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:17 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:44:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:17 smithi171 ceph-mon[90031]: 1.6 scrub ok 2024-08-29T15:44:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:17 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3955/27537 objects degraded (14.362%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:18.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.036 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:18.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:18.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:18.176 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:18.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:18.278 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:18.306 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:18.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:18.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:18.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:18.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:18.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.600 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:18.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:18.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.739 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:18.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:18.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:18 smithi027 ceph-mon[98970]: pgmap v1665: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 103 active+clean; 438 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 3955/20877 objects degraded (18.944%); 8197/20877 objects misplaced (39.263%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:44:18.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:18 smithi027 ceph-mon[93921]: pgmap v1665: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 103 active+clean; 438 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 3955/20877 objects degraded (18.944%); 8197/20877 objects misplaced (39.263%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:44:18.840 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:18.872 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:18.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:18.988 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:18.992 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:18.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:18.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:19.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:18 smithi171 ceph-mon[90031]: pgmap v1665: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 103 active+clean; 438 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 3955/20877 objects degraded (18.944%); 8197/20877 objects misplaced (39.263%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:44:19.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:19.134 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:44:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:44:19] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-08-29T15:44:19.139 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:19.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:19.241 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:19.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:19.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:19.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:19.412 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:19.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:19.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:19.548 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:19.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:19.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:19.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:19.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:19.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:19.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:19 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:44:19.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:19 smithi027 ceph-mon[93921]: 1.e scrub ok 2024-08-29T15:44:19.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:19 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:44:19.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:19 smithi027 ceph-mon[98970]: 1.e scrub ok 2024-08-29T15:44:19.845 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:19.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:19.947 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:19.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:19.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:19 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:44:20.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:19 smithi171 ceph-mon[90031]: 1.e scrub ok 2024-08-29T15:44:20.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:20.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:20.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:20.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:20.248 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:20.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:20.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:20.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:20.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:20.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:20.498 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:20.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:20.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:20.591 INFO:teuthology.orchestra.run.smithi027.stderr:set pool 1 pg_num to 31 2024-08-29T15:44:20.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:20.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:20.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:20.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:20 smithi027 ceph-mon[93921]: pgmap v1666: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 103 active+clean; 344 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 3955/16380 objects degraded (24.145%); 7467/16380 objects misplaced (45.586%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:44:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:20 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1292065651' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-29T15:44:20.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:20 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-29T15:44:20.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:20 smithi027 ceph-mon[98970]: pgmap v1666: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 103 active+clean; 344 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 3955/16380 objects degraded (24.145%); 7467/16380 objects misplaced (45.586%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:44:20.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:20 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1292065651' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-29T15:44:20.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:20 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-29T15:44:20.793 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:20.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:20.803 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:44:20.803 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:44:20.832 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:44:20.832 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:44:20.856 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:44:20.856 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:44:20.876 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:44:20.876 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:44:20.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:20.895 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:44:20.895 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:44:20.915 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:44:20.915 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:44:20.916 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:20.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:20.932 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:44:20.932 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:44:20.949 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:44:20.950 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:44:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:20 smithi171 ceph-mon[90031]: pgmap v1666: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 103 active+clean; 344 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 3955/16380 objects degraded (24.145%); 7467/16380 objects misplaced (45.586%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:44:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:20 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1292065651' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-29T15:44:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:20 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-29T15:44:21.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:21.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:21.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:21.165 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:21.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:21.289 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:21.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:21.412 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:21.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:21.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:21.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:21.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:21.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:21.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:21.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:21.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-08-29T15:44:21.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[98970]: osdmap e469: 8 total, 8 up, 8 in 2024-08-29T15:44:21.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3132094792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:44:21.801 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-29T15:44:21.802 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-08-29T15:44:21.802 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[93921]: osdmap e469: 8 total, 8 up, 8 in 2024-08-29T15:44:21.802 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3132094792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:44:21.802 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-29T15:44:21.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:21.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:21.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:21.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:21.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:21 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-08-29T15:44:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:21 smithi171 ceph-mon[90031]: osdmap e469: 8 total, 8 up, 8 in 2024-08-29T15:44:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3132094792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:44:22.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-29T15:44:22.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:22.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:22.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:22.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:22.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:22.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:22.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:22.363 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:22.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:22.465 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:22.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:22.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:22.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:22.609 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:22.652 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:22.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:22.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:22.753 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:22.783 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:22.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:22.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[93921]: pgmap v1668: 109 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 103 active+clean; 286 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 3955/13617 objects degraded (29.045%); 6989/13617 objects misplaced (51.326%); 5.7 MiB/s, 91 objects/s recovering 2024-08-29T15:44:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-08-29T15:44:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[93921]: osdmap e470: 8 total, 8 up, 8 in 2024-08-29T15:44:22.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3955/13617 objects degraded (29.045%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[98970]: pgmap v1668: 109 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 103 active+clean; 286 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 3955/13617 objects degraded (29.045%); 6989/13617 objects misplaced (51.326%); 5.7 MiB/s, 91 objects/s recovering 2024-08-29T15:44:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-08-29T15:44:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[98970]: osdmap e470: 8 total, 8 up, 8 in 2024-08-29T15:44:22.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:22 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3955/13617 objects degraded (29.045%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:22.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:22.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:22.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:22.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:22 smithi171 ceph-mon[90031]: pgmap v1668: 109 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 103 active+clean; 286 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 3955/13617 objects degraded (29.045%); 6989/13617 objects misplaced (51.326%); 5.7 MiB/s, 91 objects/s recovering 2024-08-29T15:44:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-08-29T15:44:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:22 smithi171 ceph-mon[90031]: osdmap e470: 8 total, 8 up, 8 in 2024-08-29T15:44:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:22 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3955/13617 objects degraded (29.045%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:23.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:23.063 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:23.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:23.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:23.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:23.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:23.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:23.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:23.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:23.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:23.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:23.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:23.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:23.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:23.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:23.628 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:23.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:23.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:23.729 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:23.768 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:23.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:23.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:23.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:23.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:23.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:23.998 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:24.010 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Removed 102427 objects 2024-08-29T15:44:24.010 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Clean up completed and total clean up time :175.105 2024-08-29T15:44:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:23 smithi171 ceph-mon[90031]: 1.f scrub starts 2024-08-29T15:44:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:23 smithi171 ceph-mon[90031]: 1.f scrub ok 2024-08-29T15:44:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:23 smithi171 ceph-mon[90031]: osdmap e471: 8 total, 8 up, 8 in 2024-08-29T15:44:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:23 smithi171 ceph-mon[90031]: 1.5 scrub starts 2024-08-29T15:44:24.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:23 smithi171 ceph-mon[90031]: 1.5 scrub ok 2024-08-29T15:44:24.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:23 smithi171 ceph-mon[90031]: 1.b scrub starts 2024-08-29T15:44:24.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:23 smithi171 ceph-mon[90031]: 1.b scrub ok 2024-08-29T15:44:24.022 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[98970]: 1.f scrub starts 2024-08-29T15:44:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[98970]: 1.f scrub ok 2024-08-29T15:44:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[98970]: osdmap e471: 8 total, 8 up, 8 in 2024-08-29T15:44:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[98970]: 1.5 scrub starts 2024-08-29T15:44:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[98970]: 1.5 scrub ok 2024-08-29T15:44:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[98970]: 1.b scrub starts 2024-08-29T15:44:24.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[98970]: 1.b scrub ok 2024-08-29T15:44:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[93921]: 1.f scrub starts 2024-08-29T15:44:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[93921]: 1.f scrub ok 2024-08-29T15:44:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[93921]: osdmap e471: 8 total, 8 up, 8 in 2024-08-29T15:44:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[93921]: 1.5 scrub starts 2024-08-29T15:44:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[93921]: 1.5 scrub ok 2024-08-29T15:44:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[93921]: 1.b scrub starts 2024-08-29T15:44:24.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:23 smithi027 ceph-mon[93921]: 1.b scrub ok 2024-08-29T15:44:24.124 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:24.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:24.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:24.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:24.269 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:24.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:24.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:24.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:24.497 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:24.518 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:24.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:24.620 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:24.641 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:24.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:24.764 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:24.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:24.886 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:24.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:24.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:24.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:25.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:24 smithi171 ceph-mon[90031]: 1.8 scrub starts 2024-08-29T15:44:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:24 smithi171 ceph-mon[90031]: 1.8 scrub ok 2024-08-29T15:44:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:24 smithi171 ceph-mon[90031]: pgmap v1671: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 unknown, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 128 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 4013/6030 objects degraded (66.551%); 6445/6030 objects misplaced (106.882%); 6.6 MiB/s, 154 objects/s recovering 2024-08-29T15:44:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:24 smithi171 ceph-mon[90031]: 1.1b scrub starts 2024-08-29T15:44:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:24 smithi171 ceph-mon[90031]: 1.1b scrub ok 2024-08-29T15:44:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[98970]: 1.8 scrub starts 2024-08-29T15:44:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[98970]: 1.8 scrub ok 2024-08-29T15:44:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[98970]: pgmap v1671: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 unknown, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 128 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 4013/6030 objects degraded (66.551%); 6445/6030 objects misplaced (106.882%); 6.6 MiB/s, 154 objects/s recovering 2024-08-29T15:44:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[98970]: 1.1b scrub starts 2024-08-29T15:44:25.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[98970]: 1.1b scrub ok 2024-08-29T15:44:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[93921]: 1.8 scrub starts 2024-08-29T15:44:25.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[93921]: 1.8 scrub ok 2024-08-29T15:44:25.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[93921]: pgmap v1671: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 unknown, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 128 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 4013/6030 objects degraded (66.551%); 6445/6030 objects misplaced (106.882%); 6.6 MiB/s, 154 objects/s recovering 2024-08-29T15:44:25.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[93921]: 1.1b scrub starts 2024-08-29T15:44:25.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:24 smithi027 ceph-mon[93921]: 1.1b scrub ok 2024-08-29T15:44:25.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:25.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:25.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:25.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:25.263 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:25.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:25.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:25.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:25.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:25.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:25.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:25.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:25.643 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:25.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:25.754 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:44:25.754 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[98970]: 1.6 scrub ok 2024-08-29T15:44:25.755 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[98970]: 1.d scrub starts 2024-08-29T15:44:25.755 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[98970]: 1.d scrub ok 2024-08-29T15:44:25.755 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:44:25.755 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[93921]: 1.6 scrub ok 2024-08-29T15:44:25.755 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[93921]: 1.d scrub starts 2024-08-29T15:44:25.755 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:25 smithi027 ceph-mon[93921]: 1.d scrub ok 2024-08-29T15:44:25.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:25.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:25.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:25.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:25.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:25.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:26.003 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:25 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:44:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:25 smithi171 ceph-mon[90031]: 1.6 scrub ok 2024-08-29T15:44:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:25 smithi171 ceph-mon[90031]: 1.d scrub starts 2024-08-29T15:44:26.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:25 smithi171 ceph-mon[90031]: 1.d scrub ok 2024-08-29T15:44:26.028 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:26.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:26.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:26.154 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:26.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:26.255 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-08-29T15:44:26.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-08-29T15:44:26.255 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:26.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:26.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:26.381 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:44:26.381 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:44:26.382 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:44:26.382 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:44:26.382 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:44:26.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:26.408 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:26.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:26.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:26.536 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:26.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:26.643 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:44:26.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:26.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:26.792 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:26.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:26.917 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:26.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:26 smithi171 ceph-mon[90031]: 1.c scrub starts 2024-08-29T15:44:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:26 smithi171 ceph-mon[90031]: 1.c scrub ok 2024-08-29T15:44:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:26 smithi171 ceph-mon[90031]: pgmap v1672: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 8 unknown, 1 active+undersized+remapped, 99 active+clean; 118 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 4005/5523 objects degraded (72.515%); 6445/5523 objects misplaced (116.694%); 4.6 MiB/s, 113 objects/s recovering 2024-08-29T15:44:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:26 smithi171 ceph-mon[90031]: osdmap e472: 8 total, 8 up, 8 in 2024-08-29T15:44:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:26 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/194029057' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:44:27.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:26 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:44:27.019 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:27.046 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[98970]: 1.c scrub starts 2024-08-29T15:44:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[98970]: 1.c scrub ok 2024-08-29T15:44:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[98970]: pgmap v1672: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 8 unknown, 1 active+undersized+remapped, 99 active+clean; 118 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 4005/5523 objects degraded (72.515%); 6445/5523 objects misplaced (116.694%); 4.6 MiB/s, 113 objects/s recovering 2024-08-29T15:44:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[98970]: osdmap e472: 8 total, 8 up, 8 in 2024-08-29T15:44:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/194029057' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:44:27.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:44:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[93921]: 1.c scrub starts 2024-08-29T15:44:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[93921]: 1.c scrub ok 2024-08-29T15:44:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[93921]: pgmap v1672: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 8 unknown, 1 active+undersized+remapped, 99 active+clean; 118 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 4005/5523 objects degraded (72.515%); 6445/5523 objects misplaced (116.694%); 4.6 MiB/s, 113 objects/s recovering 2024-08-29T15:44:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[93921]: osdmap e472: 8 total, 8 up, 8 in 2024-08-29T15:44:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/194029057' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:44:27.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:26 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:44:27.148 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:27.173 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:27.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:27.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:27.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:27.400 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:27.426 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:27.527 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:27.553 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:27.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:27.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:27.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:27.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:27.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:27.926 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:27.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:27.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: 1.1c scrub starts 2024-08-29T15:44:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: 1.1c scrub ok 2024-08-29T15:44:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:44:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:44:28.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: osdmap e473: 8 total, 8 up, 8 in 2024-08-29T15:44:28.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: 1.1 deep-scrub starts 2024-08-29T15:44:28.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: 1.1 deep-scrub ok 2024-08-29T15:44:28.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: 1.15 scrub starts 2024-08-29T15:44:28.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:27 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 4005/5523 objects degraded (72.515%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:28.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:28.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: 1.1c scrub starts 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: 1.1c scrub ok 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: osdmap e473: 8 total, 8 up, 8 in 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: 1.1 deep-scrub starts 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: 1.1 deep-scrub ok 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: 1.15 scrub starts 2024-08-29T15:44:28.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 4005/5523 objects degraded (72.515%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: 1.1c scrub starts 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: 1.1c scrub ok 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: osdmap e473: 8 total, 8 up, 8 in 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: 1.1 deep-scrub starts 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: 1.1 deep-scrub ok 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: 1.15 scrub starts 2024-08-29T15:44:28.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:27 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 4005/5523 objects degraded (72.515%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-29T15:44:28.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:28.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:28.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:28.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:28.303 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:28.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:28.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:28.428 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:28.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:28.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:28.552 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:28.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:28.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:28.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:28.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:28.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:28.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:28.928 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:28.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:29.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:28 smithi027 ceph-mon[98970]: 1.18 scrub starts 2024-08-29T15:44:29.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:28 smithi027 ceph-mon[98970]: 1.18 scrub ok 2024-08-29T15:44:29.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:28 smithi027 ceph-mon[98970]: pgmap v1675: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 peering, 2 unknown, 111 active+clean; 39 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 4005/1749 objects degraded (228.988%); 5928/1749 objects misplaced (338.937%); 2.3 MiB/s, 104 objects/s recovering 2024-08-29T15:44:29.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:28 smithi027 ceph-mon[93921]: 1.18 scrub starts 2024-08-29T15:44:29.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:28 smithi027 ceph-mon[93921]: 1.18 scrub ok 2024-08-29T15:44:29.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:28 smithi027 ceph-mon[93921]: pgmap v1675: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 peering, 2 unknown, 111 active+clean; 39 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 4005/1749 objects degraded (228.988%); 5928/1749 objects misplaced (338.937%); 2.3 MiB/s, 104 objects/s recovering 2024-08-29T15:44:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:28 smithi171 ceph-mon[90031]: 1.18 scrub starts 2024-08-29T15:44:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:28 smithi171 ceph-mon[90031]: 1.18 scrub ok 2024-08-29T15:44:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:28 smithi171 ceph-mon[90031]: pgmap v1675: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 peering, 2 unknown, 111 active+clean; 39 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 4005/1749 objects degraded (228.988%); 5928/1749 objects misplaced (338.937%); 2.3 MiB/s, 104 objects/s recovering 2024-08-29T15:44:29.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:29.053 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:29.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:29.155 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:29.181 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:29.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:29.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:29.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:29.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:44:29] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-08-29T15:44:29.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:29.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:29.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:29.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:29.557 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:29.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:29.681 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:29.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:29.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:29.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:29.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:29.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:29.931 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:29.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:29 smithi171 ceph-mon[90031]: 1.1e scrub starts 2024-08-29T15:44:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:29 smithi171 ceph-mon[90031]: 1.1e scrub ok 2024-08-29T15:44:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:44:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:30.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:29 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:44:30.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:30.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:30.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[93921]: 1.1e scrub starts 2024-08-29T15:44:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[93921]: 1.1e scrub ok 2024-08-29T15:44:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:44:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:30.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:44:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[98970]: 1.1e scrub starts 2024-08-29T15:44:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[98970]: 1.1e scrub ok 2024-08-29T15:44:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:44:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:29 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:44:30.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:30.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:30.250 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:44:30.250 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:44:30.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:30.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:30.412 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:30.438 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:30.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:30.540 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:30.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:30.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:30.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:30.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.817 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:30.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:30.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:30.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:30.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:30 smithi171 ceph-mon[90031]: 1.16 scrub starts 2024-08-29T15:44:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:30 smithi171 ceph-mon[90031]: 1.16 scrub ok 2024-08-29T15:44:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:30 smithi171 ceph-mon[90031]: pgmap v1676: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 116 active+clean; 4.4 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 3947/84 objects degraded (4698.810%); 5737/84 objects misplaced (6829.762%); 612 KiB/s, 85 objects/s recovering 2024-08-29T15:44:31.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:30 smithi027 ceph-mon[93921]: 1.16 scrub starts 2024-08-29T15:44:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:30 smithi027 ceph-mon[93921]: 1.16 scrub ok 2024-08-29T15:44:31.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:30 smithi027 ceph-mon[93921]: pgmap v1676: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 116 active+clean; 4.4 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 3947/84 objects degraded (4698.810%); 5737/84 objects misplaced (6829.762%); 612 KiB/s, 85 objects/s recovering 2024-08-29T15:44:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:30 smithi027 ceph-mon[98970]: 1.16 scrub starts 2024-08-29T15:44:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:30 smithi027 ceph-mon[98970]: 1.16 scrub ok 2024-08-29T15:44:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:30 smithi027 ceph-mon[98970]: pgmap v1676: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 116 active+clean; 4.4 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 3947/84 objects degraded (4698.810%); 5737/84 objects misplaced (6829.762%); 612 KiB/s, 85 objects/s recovering 2024-08-29T15:44:31.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:31.172 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:31.197 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:31.299 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:31.324 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:31.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:44:31.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:31.448 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:31.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:31.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:31.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:31.700 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:31.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:31.828 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:31.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:31.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:31.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:31.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:31 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:44:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:31 smithi171 ceph-mon[90031]: 1.e scrub ok 2024-08-29T15:44:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:31 smithi171 ceph-mon[90031]: 1.15 scrub starts 2024-08-29T15:44:32.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:31 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:44:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:31 smithi027 ceph-mon[98970]: 1.e scrub ok 2024-08-29T15:44:32.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:31 smithi027 ceph-mon[98970]: 1.15 scrub starts 2024-08-29T15:44:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:31 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:44:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:31 smithi027 ceph-mon[93921]: 1.e scrub ok 2024-08-29T15:44:32.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:31 smithi027 ceph-mon[93921]: 1.15 scrub starts 2024-08-29T15:44:32.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:32.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:32.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:32.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:32.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:32.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:32.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:32.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:32.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:32.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:32.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:32.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:32.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:32.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:32.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:32.964 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:32.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:32 smithi171 ceph-mon[90031]: 1.9 scrub starts 2024-08-29T15:44:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:32 smithi171 ceph-mon[90031]: 1.9 scrub ok 2024-08-29T15:44:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:32 smithi171 ceph-mon[90031]: pgmap v1677: 119 pgs: 2 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 114 active+clean; 4.4 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 247 op/s; 3947/84 objects degraded (4698.810%); 5737/84 objects misplaced (6829.762%); 496 KiB/s, 69 objects/s recovering 2024-08-29T15:44:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:32 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:44:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:32 smithi171 ceph-mon[90031]: Health check update: Degraded data redundancy: 3947/84 objects degraded (4698.810%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:44:33.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[98970]: 1.9 scrub starts 2024-08-29T15:44:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[98970]: 1.9 scrub ok 2024-08-29T15:44:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[98970]: pgmap v1677: 119 pgs: 2 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 114 active+clean; 4.4 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 247 op/s; 3947/84 objects degraded (4698.810%); 5737/84 objects misplaced (6829.762%); 496 KiB/s, 69 objects/s recovering 2024-08-29T15:44:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:44:33.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[98970]: Health check update: Degraded data redundancy: 3947/84 objects degraded (4698.810%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:44:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[93921]: 1.9 scrub starts 2024-08-29T15:44:33.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[93921]: 1.9 scrub ok 2024-08-29T15:44:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[93921]: pgmap v1677: 119 pgs: 2 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 114 active+clean; 4.4 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 247 op/s; 3947/84 objects degraded (4698.810%); 5737/84 objects misplaced (6829.762%); 496 KiB/s, 69 objects/s recovering 2024-08-29T15:44:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:44:33.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:32 smithi027 ceph-mon[93921]: Health check update: Degraded data redundancy: 3947/84 objects degraded (4698.810%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-08-29T15:44:33.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:33.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:33.216 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:33.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:33.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:33.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:33.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:33.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:33.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:33.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:33.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:33.719 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:33.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:33.821 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:33.846 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:33.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:33.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:33.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:33.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:33 smithi171 ceph-mon[90031]: 1.19 scrub starts 2024-08-29T15:44:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:33 smithi171 ceph-mon[90031]: 1.19 scrub ok 2024-08-29T15:44:34.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:33 smithi171 ceph-mon[90031]: 1.15 scrub starts 2024-08-29T15:44:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:33 smithi027 ceph-mon[98970]: 1.19 scrub starts 2024-08-29T15:44:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:33 smithi027 ceph-mon[98970]: 1.19 scrub ok 2024-08-29T15:44:34.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:33 smithi027 ceph-mon[98970]: 1.15 scrub starts 2024-08-29T15:44:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:33 smithi027 ceph-mon[93921]: 1.19 scrub starts 2024-08-29T15:44:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:33 smithi027 ceph-mon[93921]: 1.19 scrub ok 2024-08-29T15:44:34.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:33 smithi027 ceph-mon[93921]: 1.15 scrub starts 2024-08-29T15:44:34.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:34.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:34.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:34.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:34.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:34.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:34.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:34.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:34.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:34.479 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:34.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:34.603 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:34.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:34.730 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:34.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:34 smithi027 ceph-mon[93921]: pgmap v1678: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 3947/84 objects degraded (4698.810%); 5688/84 objects misplaced (6771.429%); 0 B/s, 122 objects/s recovering 2024-08-29T15:44:34.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:34 smithi027 ceph-mon[98970]: pgmap v1678: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 3947/84 objects degraded (4698.810%); 5688/84 objects misplaced (6771.429%); 0 B/s, 122 objects/s recovering 2024-08-29T15:44:34.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:34.854 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:34.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:34.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:34.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:34.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:34 smithi171 ceph-mon[90031]: pgmap v1678: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 3947/84 objects degraded (4698.810%); 5688/84 objects misplaced (6771.429%); 0 B/s, 122 objects/s recovering 2024-08-29T15:44:35.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:35.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:35.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:35.208 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.235 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:35.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:35.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:35.358 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:35.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:35.483 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:35.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:35.607 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:35.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:35.732 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:35.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:35.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:35.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:35.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:35.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:35.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:35.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:35 smithi171 ceph-mon[90031]: pgmap v1679: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 3947/84 objects degraded (4698.810%); 5688/84 objects misplaced (6771.429%); 0 B/s, 104 objects/s recovering 2024-08-29T15:44:36.047 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:35 smithi027 ceph-mon[93921]: pgmap v1679: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 3947/84 objects degraded (4698.810%); 5688/84 objects misplaced (6771.429%); 0 B/s, 104 objects/s recovering 2024-08-29T15:44:36.048 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:35 smithi027 ceph-mon[98970]: pgmap v1679: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 3947/84 objects degraded (4698.810%); 5688/84 objects misplaced (6771.429%); 0 B/s, 104 objects/s recovering 2024-08-29T15:44:36.087 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:36.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:36.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:36.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:36.239 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:36.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:36.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:36.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:36.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:36.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:36.489 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:36.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:36.614 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:36.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:36.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:36.739 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:36.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:36.863 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:36.965 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:36.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:36.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:36.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:37.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:36 smithi027 ceph-mon[98970]: 1.11 scrub starts 2024-08-29T15:44:37.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:36 smithi027 ceph-mon[98970]: 1.11 scrub ok 2024-08-29T15:44:37.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:36 smithi027 ceph-mon[93921]: 1.11 scrub starts 2024-08-29T15:44:37.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:36 smithi027 ceph-mon[93921]: 1.11 scrub ok 2024-08-29T15:44:37.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:37.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:37.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:37.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:37.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:37.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:37.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:36 smithi171 ceph-mon[90031]: 1.11 scrub starts 2024-08-29T15:44:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:36 smithi171 ceph-mon[90031]: 1.11 scrub ok 2024-08-29T15:44:37.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:37.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:37.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:37.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:37.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:37.496 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:37.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:37.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:37.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:37.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:37.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:37.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:37.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:37.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:37.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:37.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:37.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:37.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:38.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:37 smithi027 ceph-mon[98970]: pgmap v1680: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 126 objects/s recovering 2024-08-29T15:44:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:37 smithi027 ceph-mon[93921]: pgmap v1680: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 126 objects/s recovering 2024-08-29T15:44:38.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:38.126 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:38.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:37 smithi171 ceph-mon[90031]: pgmap v1680: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 126 objects/s recovering 2024-08-29T15:44:38.228 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:38.254 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:38.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:38.381 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:38.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:38.482 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:38.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:38.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:38.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:38.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:38.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:38.760 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:38.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:38.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:38.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:38.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:39.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:39.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:38 smithi027 ceph-mon[98970]: 1.7 scrub starts 2024-08-29T15:44:39.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:38 smithi027 ceph-mon[93921]: 1.7 scrub starts 2024-08-29T15:44:39.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:44:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:44:39] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.51.0" 2024-08-29T15:44:39.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:39.135 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:39.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:39.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:39.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:38 smithi171 ceph-mon[90031]: 1.7 scrub starts 2024-08-29T15:44:39.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:39.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:39.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:39.487 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:39.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:39.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:39.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:39.634 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:39.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:39.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:39.759 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:39.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:39.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:39.884 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:39.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:39.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:39.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:40.011 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:39 smithi027 ceph-mon[93921]: 1.1d deep-scrub starts 2024-08-29T15:44:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:39 smithi027 ceph-mon[93921]: pgmap v1681: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 46 op/s; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 109 objects/s recovering 2024-08-29T15:44:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:39 smithi027 ceph-mon[93921]: 1.a scrub starts 2024-08-29T15:44:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:39 smithi027 ceph-mon[98970]: 1.1d deep-scrub starts 2024-08-29T15:44:40.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:39 smithi027 ceph-mon[98970]: pgmap v1681: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 46 op/s; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 109 objects/s recovering 2024-08-29T15:44:40.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:39 smithi027 ceph-mon[98970]: 1.a scrub starts 2024-08-29T15:44:40.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:40.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:40.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:40.263 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:39 smithi171 ceph-mon[90031]: 1.1d deep-scrub starts 2024-08-29T15:44:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:39 smithi171 ceph-mon[90031]: pgmap v1681: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 46 op/s; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 109 objects/s recovering 2024-08-29T15:44:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:39 smithi171 ceph-mon[90031]: 1.a scrub starts 2024-08-29T15:44:40.365 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:40.389 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:40.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:40.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:44:40.516 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:40.618 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:40.643 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:40.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:40.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:40.769 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:40.870 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:40.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:40.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:40.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:40.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:41.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:41.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:40 smithi027 ceph-mon[98970]: 1.17 scrub starts 2024-08-29T15:44:41.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:40 smithi027 ceph-mon[93921]: 1.17 scrub starts 2024-08-29T15:44:41.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:41.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:41.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:41.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:40 smithi171 ceph-mon[90031]: 1.17 scrub starts 2024-08-29T15:44:41.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:41.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:41.375 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:41.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:41.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:41.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:41.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:41.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:41.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:41.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:41.780 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:41.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:41.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:41.905 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:41.908 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:41.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:41.942 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:42.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:42.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:42.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:41 smithi027 ceph-mon[93921]: pgmap v1682: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 74 objects/s recovering 2024-08-29T15:44:42.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:41 smithi027 ceph-mon[98970]: pgmap v1682: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 74 objects/s recovering 2024-08-29T15:44:42.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:42.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:42.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:42.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:41 smithi171 ceph-mon[90031]: pgmap v1682: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing, 114 active+clean; 4.6 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 5228/84 objects misplaced (6223.810%); 0 B/s, 74 objects/s recovering 2024-08-29T15:44:42.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:42.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:42.411 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:42.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:42.536 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:42.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:42.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:42.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:42.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:42.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:42.912 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:42.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:43.014 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:43.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:43.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:43.058 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:42 smithi027 ceph-mon[98970]: 1.1a scrub starts 2024-08-29T15:44:43.058 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:42 smithi027 ceph-mon[93921]: 1.1a scrub starts 2024-08-29T15:44:43.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:43.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.169 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:43.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:43.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:42 smithi171 ceph-mon[90031]: 1.1a scrub starts 2024-08-29T15:44:43.270 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:43.295 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:43.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:43.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:43.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.421 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:43.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:43.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:43.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:43.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:43.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:43.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:43.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:43.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:43.924 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:43.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:43.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:44:44.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:44.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[98970]: 1.15 deep-scrub starts 2024-08-29T15:44:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[98970]: pgmap v1683: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing+deep, 4 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4731/84 objects misplaced (5632.143%); 0 B/s, 116 objects/s recovering 2024-08-29T15:44:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[98970]: 1.7 scrub starts 2024-08-29T15:44:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[93921]: 1.15 deep-scrub starts 2024-08-29T15:44:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[93921]: pgmap v1683: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing+deep, 4 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4731/84 objects misplaced (5632.143%); 0 B/s, 116 objects/s recovering 2024-08-29T15:44:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[93921]: 1.7 scrub starts 2024-08-29T15:44:44.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:44.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:44.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:43 smithi171 ceph-mon[90031]: 1.15 deep-scrub starts 2024-08-29T15:44:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:43 smithi171 ceph-mon[90031]: pgmap v1683: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing+deep, 4 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4731/84 objects misplaced (5632.143%); 0 B/s, 116 objects/s recovering 2024-08-29T15:44:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:43 smithi171 ceph-mon[90031]: 1.7 scrub starts 2024-08-29T15:44:44.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:44:44.275 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:44.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:44.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:44.402 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:44.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:44.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:44.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:44.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:44.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:44.681 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:44.783 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:44.808 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:44.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:44.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:44.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:45.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:45.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:45.163 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:45.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:45.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:45.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:45.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:45.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:45.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:45.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:45.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:45.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:45.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:45.663 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:45.689 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:45.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:45.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:45.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:45.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:45.944 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:45.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:46.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:46.069 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:46.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:46.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:46.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:46.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:46 smithi027 ceph-mon[98970]: pgmap v1684: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4731/84 objects misplaced (5632.143%); 0 B/s, 79 objects/s recovering 2024-08-29T15:44:46.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:46 smithi027 ceph-mon[98970]: 1.a deep-scrub starts 2024-08-29T15:44:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:46 smithi027 ceph-mon[93921]: pgmap v1684: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4731/84 objects misplaced (5632.143%); 0 B/s, 79 objects/s recovering 2024-08-29T15:44:46.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:46 smithi027 ceph-mon[93921]: 1.a deep-scrub starts 2024-08-29T15:44:46.322 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:46.423 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:46.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:46 smithi171 ceph-mon[90031]: pgmap v1684: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4731/84 objects misplaced (5632.143%); 0 B/s, 79 objects/s recovering 2024-08-29T15:44:46.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:46 smithi171 ceph-mon[90031]: 1.a deep-scrub starts 2024-08-29T15:44:46.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:46.577 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:46.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:46.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:46.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:46.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:46.809 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:46.833 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:46.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:46.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:46.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:47.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:47.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:47.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:47.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:47.316 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:47.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:47 smithi027 ceph-mon[93921]: 1.1d deep-scrub starts 2024-08-29T15:44:47.320 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:47 smithi027 ceph-mon[98970]: 1.1d deep-scrub starts 2024-08-29T15:44:47.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:47.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:47.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:47 smithi171 ceph-mon[90031]: 1.1d deep-scrub starts 2024-08-29T15:44:47.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:47.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:47.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:47.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:47.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:47.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:47.818 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:47.843 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:47.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:47.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:47.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:47.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:48.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:48.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:48.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:48.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:48.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:48.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:48.346 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:48.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:48.470 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:48 smithi027 ceph-mon[98970]: pgmap v1685: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 124 objects/s recovering 2024-08-29T15:44:48.470 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:48 smithi027 ceph-mon[98970]: 1.17 scrub starts 2024-08-29T15:44:48.470 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:48 smithi027 ceph-mon[98970]: 1.15 scrub starts 2024-08-29T15:44:48.470 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:48 smithi027 ceph-mon[93921]: pgmap v1685: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 124 objects/s recovering 2024-08-29T15:44:48.470 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:48 smithi027 ceph-mon[93921]: 1.17 scrub starts 2024-08-29T15:44:48.470 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:48 smithi027 ceph-mon[93921]: 1.15 scrub starts 2024-08-29T15:44:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.476 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:48.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:48 smithi171 ceph-mon[90031]: pgmap v1685: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 2 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 111 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 124 objects/s recovering 2024-08-29T15:44:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:48 smithi171 ceph-mon[90031]: 1.17 scrub starts 2024-08-29T15:44:48.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:48 smithi171 ceph-mon[90031]: 1.15 scrub starts 2024-08-29T15:44:48.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:48.599 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:48.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:48.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:48.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:48.855 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:48.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:48.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:48.990 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:48.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:49.092 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:49.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:49.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:49.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:49.220 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:49.245 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:49.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:49.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:49 smithi027 ceph-mon[98970]: 1.1d deep-scrub starts 2024-08-29T15:44:49.271 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:49 smithi027 ceph-mon[93921]: 1.1d deep-scrub starts 2024-08-29T15:44:49.272 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:44:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:44:49] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-29T15:44:49.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:49.372 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:49.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:49.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:49.474 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:49.500 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:49.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:49 smithi171 ceph-mon[90031]: 1.1d deep-scrub starts 2024-08-29T15:44:49.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:49.627 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:49.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:49.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:49.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:49.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:49.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:49.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:49.856 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:49.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:49.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:49.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:50.009 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:50.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:50.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:50.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:50.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:50.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:50.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:50.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:50.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:50.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:50.384 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:50.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:50.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:50.485 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185381 8.a 8:593dd5d4:::benchmark_data_smithi027_306999_object58602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 432.58805496299999, 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8689627519999998, 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.486 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185381 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377780+0000", 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377781+0000", 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377785+0000", 2024-08-29T15:44:50.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5480000000000002e-06 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377794+0000", 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6290000000000005e-06 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.377804+0000", 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3079999999999997e-06 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.387009+0000", 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092053380000000004 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.387019+0000", 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8940000000000005e-06 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088587+0000", 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70156798499999995 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088600+0000", 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2778e-05 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238539+0000", 2024-08-29T15:44:50.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1499392150000001 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238659+0000", 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012044699999999999 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.238741+0000", 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1568000000000002e-05 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246057+0000", 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007315963 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246083+0000", 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5687999999999999e-05 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246298+0000", 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021570700000000001 2024-08-29T15:44:50.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246644+0000", 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034599799999999998 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246681+0000", 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6872999999999997e-05 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246743+0000", 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1496999999999998e-05 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185404 8.a 8:510e3083:::benchmark_data_smithi027_306999_object58625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 432.54237325000003, 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.823589948, 2024-08-29T15:44:50.495 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185404 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:44:50.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423462+0000", 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423463+0000", 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4500000000000001e-06 2024-08-29T15:44:50.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423469+0000", 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1870000000000002e-06 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423483+0000", 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3193000000000001e-05 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423489+0000", 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0220000000000001e-06 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423549+0000", 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0466e-05 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.423556+0000", 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9029999999999998e-06 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088626+0000", 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.66507021300000002 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.088634+0000", 2024-08-29T15:44:50.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9379999999999992e-06 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242305+0000", 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1536706910000001 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242434+0000", 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129484 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.242523+0000", 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8343000000000004e-05 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246560+0000", 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040375419999999999 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246569+0000", 2024-08-29T15:44:50.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6570000000000003e-06 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246605+0000", 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6331000000000002e-05 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246965+0000", 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00036038900000000002 2024-08-29T15:44:50.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.246996+0000", 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0054000000000001e-05 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.247052+0000", 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6085000000000003e-05 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.504 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218179 8.a 8:58241dc9:::benchmark_data_smithi027_306999_object91400:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.086769+0000", 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 387.879065443, 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0426059379999999, 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218179 2024-08-29T15:44:50.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086769+0000", 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086769+0000", 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086772+0000", 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3649999999999998e-06 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086779+0000", 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3629999999999996e-06 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086803+0000", 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3312999999999999e-05 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.086812+0000", 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6879999999999994e-06 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.088104+0000", 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012921460000000001 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.088245+0000", 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000140309 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.088389+0000", 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014480600000000001 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.097809+0000", 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094199079999999994 2024-08-29T15:44:50.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.097859+0000", 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9901999999999998e-05 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.108191+0000", 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010332134 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110586+0000", 2024-08-29T15:44:50.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002394611 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110597+0000", 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1059e-05 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129298+0000", 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.018701063 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129375+0000", 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7270999999999999e-05 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218195 8.a 8:55cc424e:::benchmark_data_smithi027_306999_object91416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.098464+0000", 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 387.86737136300002, 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0314149020000001, 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.512 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218195 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098464+0000", 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098464+0000", 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098466+0000", 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.951e-06 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098471+0000", 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4029999999999998e-06 2024-08-29T15:44:50.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098488+0000", 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6866000000000001e-05 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098496+0000", 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1280000000000008e-06 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.098814+0000", 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031784800000000002 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.099180+0000", 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00036671700000000002 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,5", 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.099316+0000", 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013535700000000001 2024-08-29T15:44:50.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110692+0000", 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011376569 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110701+0000", 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1249999999999993e-06 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129806+0000", 2024-08-29T15:44:50.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0191050269999999 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129878+0000", 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2910999999999994e-05 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218198 8.a 8:5081af65:::benchmark_data_smithi027_306999_object91419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.104328+0000", 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 387.86150651600002, 2024-08-29T15:44:50.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414401179999999, 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218198 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104328+0000", 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104328+0000", 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104330+0000", 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104335+0000", 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8790000000000002e-06 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104347+0000", 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2407e-05 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.104354+0000", 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1810000000000004e-06 2024-08-29T15:44:50.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110741+0000", 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006386404 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.110750+0000", 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6569999999999992e-06 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.129925+0000", 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.019174354 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.130034+0000", 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010951999999999999 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.130160+0000", 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012536900000000001 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137494+0000", 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073345889999999999 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142034+0000", 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045402749999999999 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145649+0000", 2024-08-29T15:44:50.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036144290000000002 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145713+0000", 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4381999999999994e-05 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145769+0000", 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5356000000000003e-05 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.525 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218219 8.a 8:5cb7da76:::benchmark_data_smithi027_306999_object91440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.131766+0000", 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 387.83406921199997, 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0142084140000001, 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.526 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218219 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131766+0000", 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131766+0000", 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131768+0000", 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.782e-06 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131773+0000", 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9880000000000001e-06 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131792+0000", 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8474999999999998e-05 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.131800+0000", 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3070000000000003e-06 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.132949+0000", 2024-08-29T15:44:50.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001149207 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.132957+0000", 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0069999999999997e-06 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.133800+0000", 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.000842011 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.133898+0000", 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8140000000000006e-05 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.133972+0000", 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4816999999999999e-05 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137603+0000", 2024-08-29T15:44:50.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036306039999999999 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145782+0000", 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0081790950000000008 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145894+0000", 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011136700000000001 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145933+0000", 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9104999999999999e-05 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145974+0000", 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1508999999999998e-05 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218223 8.a 8:5d762aaa:::benchmark_data_smithi027_306999_object91444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.134391+0000", 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 387.83144397699999, 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0117640050000001, 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218223 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134391+0000", 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134391+0000", 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134393+0000", 2024-08-29T15:44:50.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6700000000000001e-06 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134398+0000", 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9529999999999998e-06 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134410+0000", 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2024e-05 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134417+0000", 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1350000000000003e-06 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134470+0000", 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3601999999999999e-05 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.134478+0000", 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9019999999999999e-06 2024-08-29T15:44:50.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137660+0000", 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.003182156 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137751+0000", 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0586000000000003e-05 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.137875+0000", 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012370299999999999 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145834+0000", 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079591920000000003 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145984+0000", 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015044099999999999 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146083+0000", 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8190999999999996e-05 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146112+0000", 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9539999999999998e-05 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146155+0000", 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2910000000000001e-05 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218252 8.a 8:50ec3c52:::benchmark_data_smithi027_306999_object91473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:44:50.541 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.166915+0000", 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 387.79891956900002, 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99340993799999999, 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218252 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166915+0000", 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166915+0000", 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166917+0000", 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.469e-06 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166922+0000", 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8760000000000004e-06 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166934+0000", 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1935e-05 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166941+0000", 2024-08-29T15:44:50.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7209999999999994e-06 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.166998+0000", 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6833999999999998e-05 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.167006+0000", 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8630000000000008e-06 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142076+0000", 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97506998899999997 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142153+0000", 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7260000000000002e-05 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.142233+0000", 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9566000000000004e-05 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.145988+0000", 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003755525 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146029+0000", 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0460000000000002e-05 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160213+0000", 2024-08-29T15:44:50.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014184311999999999 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160276+0000", 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2817999999999994e-05 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160325+0000", 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9310000000000001e-05 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.548 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:218281 8.a 8:52ee9bfa:::benchmark_data_smithi027_306999_object91502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:21.197901+0000", 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 387.76793380100003, 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96259614299999996, 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:44:50.549 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 218281 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197901+0000", 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197901+0000", 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197902+0000", 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4099999999999997e-07 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197906+0000", 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1949999999999996e-06 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197916+0000", 2024-08-29T15:44:50.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7680000000000003e-06 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197923+0000", 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1330000000000001e-06 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197987+0000", 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4399999999999993e-05 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:44:50.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:21.197995+0000", 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2320000000000004e-06 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146210+0000", 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94821484599999994 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146314+0000", 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010488999999999999 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.146425+0000", 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110705 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160082+0000", 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013656975 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160099+0000", 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6920999999999999e-05 2024-08-29T15:44:50.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160393+0000", 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029413600000000002 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160434+0000", 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1162999999999999e-05 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:22.160497+0000", 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2937999999999999e-05 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:130028 9.0 9:03a7de37:::benchmark_data_smithi027_341149_object27598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e450)", 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:21.220675+0000", 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 147.745160247, 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16093682500000001, 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 130028 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.220675+0000", 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.220675+0000", 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.220676+0000", 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.796e-06 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.220683+0000", 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1970000000000001e-06 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.220700+0000", 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7756000000000001e-05 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.220709+0000", 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2830000000000001e-06 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.221956+0000", 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001247705 2024-08-29T15:44:50.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.222112+0000", 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000155165 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,2,6", 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.222268+0000", 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015628899999999999 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.226339+0000", 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040706420000000002 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.226522+0000", 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018372199999999999 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.229907+0000", 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033850899999999999 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.377956+0000", 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.148048718 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.378029+0000", 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3156000000000002e-05 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:21.381612+0000", 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035823059999999999 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145181 9.2 9:474313dd:::benchmark_data_smithi027_341149_object42751:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.159702+0000", 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 118.806132694, 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.149229734, 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.563 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145181 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.159702+0000", 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.159702+0000", 2024-08-29T15:44:50.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.159704+0000", 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.606e-06 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.159709+0000", 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1760000000000001e-06 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.159721+0000", 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1738e-05 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.159728+0000", 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8809999999999998e-06 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.191313+0000", 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.031585296999999998 2024-08-29T15:44:50.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for rw locks", 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.191530+0000", 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021707900000000001 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.286232+0000", 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.094701952000000006 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.286338+0000", 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105964 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.286496+0000", 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015829799999999999 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.300920+0000", 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014423661000000001 2024-08-29T15:44:50.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.300936+0000", 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6436e-05 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.305106+0000", 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041698530000000003 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.305167+0000", 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0865000000000001e-05 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.308932+0000", 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037649279999999999 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145346 9.4 9:2da78b2a:::benchmark_data_smithi027_341149_object42916:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:50.570 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.463379+0000", 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 118.502455696, 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.221362903, 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145346 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463379+0000", 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463379+0000", 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463380+0000", 2024-08-29T15:44:50.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1000000000000001e-06 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463385+0000", 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.374e-06 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463401+0000", 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6308e-05 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463408+0000", 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2810000000000003e-06 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.465915+0000", 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025071009999999999 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.466023+0000", 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107811 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.466133+0000", 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010950000000000001 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470663+0000", 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045307819999999997 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.474864+0000", 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042004750000000004 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.684648+0000", 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.209783885 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.684700+0000", 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2476999999999999e-05 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.684742+0000", 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1808999999999999e-05 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145347 9.4 9:2e71130c:::benchmark_data_smithi027_341149_object42917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 118.50154542999999, 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.227039728, 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.577 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145347 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0399999999999995e-07 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464294+0000", 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8010000000000001e-06 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464308+0000", 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3567000000000001e-05 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464314+0000", 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9040000000000003e-06 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470690+0000", 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063753079999999997 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470816+0000", 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126632 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470946+0000", 2024-08-29T15:44:50.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012938000000000001 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.478372+0000", 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074261450000000003 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.478408+0000", 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6189999999999997e-05 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691246+0000", 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21283770399999999 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691290+0000", 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3763000000000001e-05 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691329+0000", 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9629999999999998e-05 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.583 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145349 9.4 9:2a2db507:::benchmark_data_smithi027_341149_object42919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.471338+0000", 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 118.49449664700001, 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.220100609, 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.584 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145349 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471338+0000", 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471338+0000", 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471340+0000", 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3039999999999999e-06 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471344+0000", 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1570000000000002e-06 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471355+0000", 2024-08-29T15:44:50.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1581e-05 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471367+0000", 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1378e-05 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.474937+0000", 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035707260000000002 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.475093+0000", 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000155501 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.475213+0000", 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120015 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.478582+0000", 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033687000000000001 2024-08-29T15:44:50.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.479052+0000", 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00047079199999999999 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691366+0000", 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21231396999999999 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691406+0000", 2024-08-29T15:44:50.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9988999999999999e-05 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691439+0000", 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2496000000000002e-05 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145357 9.4 9:2f78fc4c:::benchmark_data_smithi027_341149_object42927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.483919+0000", 2024-08-29T15:44:50.590 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 118.481915489, 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20763130699999999, 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145357 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483919+0000", 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483919+0000", 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483921+0000", 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4899999999999999e-06 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483926+0000", 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2859999999999999e-06 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483942+0000", 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5673999999999999e-05 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483949+0000", 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4139999999999997e-06 2024-08-29T15:44:50.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.484005+0000", 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6084e-05 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.484127+0000", 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121888 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:44:50.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.484246+0000", 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118497 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.487752+0000", 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035064229999999998 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.487856+0000", 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103711 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691479+0000", 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.203623532 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691519+0000", 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9487999999999999e-05 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691551+0000", 2024-08-29T15:44:50.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1819999999999997e-05 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145358 9.4 9:22faca06:::benchmark_data_smithi027_341149_object42928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.505077+0000", 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 118.460758111, 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.186618967, 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.597 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145358 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505077+0000", 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505077+0000", 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505079+0000", 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7710000000000001e-06 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505085+0000", 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.426e-06 2024-08-29T15:44:50.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505103+0000", 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7544999999999999e-05 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505112+0000", 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4350000000000003e-06 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505178+0000", 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6161000000000006e-05 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505268+0000", 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0314000000000003e-05 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.505349+0000", 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0739000000000001e-05 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.509036+0000", 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003686422 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.509240+0000", 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000204439 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691609+0000", 2024-08-29T15:44:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18236902599999999 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691656+0000", 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7138999999999997e-05 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691696+0000", 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9549999999999999e-05 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.603 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145370 9.4 9:2a60613a:::benchmark_data_smithi027_341149_object42940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.551392+0000", 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 118.41444245, 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14062950599999999, 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.604 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145370 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551392+0000", 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551392+0000", 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551394+0000", 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.502e-06 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551400+0000", 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7540000000000003e-06 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551415+0000", 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5418e-05 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551423+0000", 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5000000000000002e-06 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551477+0000", 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3976e-05 2024-08-29T15:44:50.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551615+0000", 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013850000000000001 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.551748+0000", 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013308499999999999 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.555568+0000", 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003820099 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.555633+0000", 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4882999999999995e-05 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691741+0000", 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.136107531 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691985+0000", 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024450900000000001 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.692022+0000", 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6748999999999997e-05 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:152586 9.2 9:49e71a4b:::benchmark_data_smithi027_341149_object50156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:03.830082+0000", 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 105.13575274900001, 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14931251600000001, 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 152586 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.830082+0000", 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.830082+0000", 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.830084+0000", 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.829e-06 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.830091+0000", 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7569999999999998e-06 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.830103+0000", 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2493e-05 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.830111+0000", 2024-08-29T15:44:50.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7880000000000007e-06 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.889294+0000", 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.059182650000000003 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for rw locks", 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.889424+0000", 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013022899999999999 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.946175+0000", 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.056751123000000001 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.946285+0000", 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109992 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.946451+0000", 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016551599999999999 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.966607+0000", 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020156252999999999 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.966647+0000", 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0404999999999997e-05 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.976523+0000", 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.009876002 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.976571+0000", 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7997999999999997e-05 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.979395+0000", 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028234810000000001 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.617 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:152646 9.2 9:49106b33:::benchmark_data_smithi027_341149_object50216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:03.979876+0000", 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 104.985959114, 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14232431400000001, 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.618 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.619 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 152646 2024-08-29T15:44:50.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.619 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.979876+0000", 2024-08-29T15:44:50.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.979876+0000", 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.979876+0000", 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4099999999999999e-07 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.979879+0000", 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9919999999999999e-06 2024-08-29T15:44:50.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.979886+0000", 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9140000000000002e-06 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.979891+0000", 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1429999999999997e-06 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.980019+0000", 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012800100000000001 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for rw locks", 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:03.980126+0000", 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106215 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.027031+0000", 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046905593000000002 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.090067+0000", 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.063035807999999999 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.090232+0000", 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016491499999999999 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.118523+0000", 2024-08-29T15:44:50.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.02829077 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.118572+0000", 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9568999999999999e-05 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.118648+0000", 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5241999999999996e-05 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.118713+0000", 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5037999999999996e-05 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:04.122200+0000", 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034875729999999999 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:188223 9.0 9:063271cf:::benchmark_data_smithi027_341149_object85793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:44:01.836050+0000", 2024-08-29T15:44:50.626 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 47.129784553999997, 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13379740300000001, 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 188223 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.836050+0000", 2024-08-29T15:44:50.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.836050+0000", 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.836052+0000", 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.229e-06 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.836055+0000", 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6009999999999999e-06 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.836064+0000", 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1549999999999996e-06 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.836070+0000", 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7509999999999997e-06 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.845430+0000", 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093593879999999997 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.845585+0000", 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015525900000000001 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,2,6", 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.845739+0000", 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015428600000000001 2024-08-29T15:44:50.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.852708+0000", 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069690020000000002 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.860339+0000", 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076310429999999997 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.860377+0000", 2024-08-29T15:44:50.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7707999999999999e-05 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.888000+0000", 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027623223999999998 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.888074+0000", 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3813000000000006e-05 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:44:50.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:44:01.969848+0000", 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.081773944000000001 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:44:50.633 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:44:50.633 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:50 smithi027 ceph-mon[93921]: pgmap v1686: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 86 objects/s recovering 2024-08-29T15:44:50.634 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:50 smithi027 ceph-mon[93921]: 1.15 scrub starts 2024-08-29T15:44:50.634 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:50 smithi027 ceph-mon[98970]: pgmap v1686: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 86 objects/s recovering 2024-08-29T15:44:50.634 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:50 smithi027 ceph-mon[98970]: 1.15 scrub starts 2024-08-29T15:44:50.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:50.635 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:50 smithi171 ceph-mon[90031]: pgmap v1686: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 86 objects/s recovering 2024-08-29T15:44:50.635 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:50 smithi171 ceph-mon[90031]: 1.15 scrub starts 2024-08-29T15:44:50.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:50.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:50.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:50.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:50.781 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:50.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:50.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:50.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:50.910 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:50.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:50.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:51.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:51.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:51.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:51.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:51.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:44:51.255 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:51.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:51.289 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:51.391 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:51.418 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:51.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:51 smithi171 ceph-mon[90031]: 1.1a scrub starts 2024-08-29T15:44:51.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:51 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:44:51.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:51 smithi027 ceph-mon[98970]: 1.1a scrub starts 2024-08-29T15:44:51.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:51 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:44:51.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:51 smithi027 ceph-mon[93921]: 1.1a scrub starts 2024-08-29T15:44:51.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:51 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:44:51.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:51.546 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:51.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:51.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:51.673 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:51.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:51.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:51.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:51.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:51.925 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:51.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:51.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:52.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:52.052 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:52.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:52.177 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:52.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:52.302 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:52.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:52.430 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:52 smithi171 ceph-mon[90031]: pgmap v1687: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing+deep, 4 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 86 objects/s recovering 2024-08-29T15:44:52.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:52 smithi171 ceph-mon[90031]: 1.7 deep-scrub starts 2024-08-29T15:44:52.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:52.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:52 smithi027 ceph-mon[98970]: pgmap v1687: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing+deep, 4 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 86 objects/s recovering 2024-08-29T15:44:52.545 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:52 smithi027 ceph-mon[98970]: 1.7 deep-scrub starts 2024-08-29T15:44:52.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:52 smithi027 ceph-mon[93921]: pgmap v1687: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing+deep, 4 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 4195/84 objects misplaced (4994.048%); 0 B/s, 86 objects/s recovering 2024-08-29T15:44:52.545 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:52 smithi027 ceph-mon[93921]: 1.7 deep-scrub starts 2024-08-29T15:44:52.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:52.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:52.660 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:52.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:52.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:52.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:52.913 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:52.939 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:52.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:52.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:53.041 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:53.067 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:53.169 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:53.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:53.296 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:53.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:53.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:53.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:53.432 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_4' removed 2024-08-29T15:44:53.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:53 smithi171 ceph-mon[90031]: 1.15 scrub starts 2024-08-29T15:44:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3920030479' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:44:53.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:53 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:44:53.544 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:53 smithi027 ceph-mon[98970]: 1.15 scrub starts 2024-08-29T15:44:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3920030479' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:44:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:53 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:44:53.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:53 smithi027 ceph-mon[93921]: 1.15 scrub starts 2024-08-29T15:44:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3920030479' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:44:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:53 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:44:53.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:53.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:53.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:53.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:53.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:53.836 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:53.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:53.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:53.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:53.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:53.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:54.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:54.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:54.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:54.208 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:54.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:54.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:54.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:54.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:54.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:54 smithi171 ceph-mon[90031]: pgmap v1688: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 3633/84 objects misplaced (4325.000%); 0 B/s, 132 objects/s recovering 2024-08-29T15:44:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:54 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:44:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:54 smithi171 ceph-mon[90031]: osdmap e474: 8 total, 8 up, 8 in 2024-08-29T15:44:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:54 smithi171 ceph-mon[90031]: 1.a scrub starts 2024-08-29T15:44:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:54 smithi171 ceph-mon[90031]: 1.a scrub ok 2024-08-29T15:44:54.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[93921]: pgmap v1688: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 3633/84 objects misplaced (4325.000%); 0 B/s, 132 objects/s recovering 2024-08-29T15:44:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:44:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[93921]: osdmap e474: 8 total, 8 up, 8 in 2024-08-29T15:44:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[93921]: 1.a scrub starts 2024-08-29T15:44:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[93921]: 1.a scrub ok 2024-08-29T15:44:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[98970]: pgmap v1688: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 3 active+clean+scrubbing+deep, 3 active+clean+scrubbing, 110 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3947/84 objects degraded (4698.810%); 3633/84 objects misplaced (4325.000%); 0 B/s, 132 objects/s recovering 2024-08-29T15:44:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:44:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[98970]: osdmap e474: 8 total, 8 up, 8 in 2024-08-29T15:44:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[98970]: 1.a scrub starts 2024-08-29T15:44:54.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:54 smithi027 ceph-mon[98970]: 1.a scrub ok 2024-08-29T15:44:54.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:54.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:54.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:54.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:54.812 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:54.833 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:54.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:54.957 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:54.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:54.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:55.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:55.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:55.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:55.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:55.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:55.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:55.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:55.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:55.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:55.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:55.431 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:55 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-29T15:44:55.431 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:55 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-29T15:44:55.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:55.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:55.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:55.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:55 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-29T15:44:55.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:55.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:55.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:55.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:55.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:55.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:55.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:55.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:55.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:55.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:55.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:55.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.033 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:56.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:56.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:44:56.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:44:56.223 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:44:56.240 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[93921]: pgmap v1690: 103 pgs: 2 active+clean+scrubbing+deep, 2 active+clean+scrubbing, 99 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:44:56.240 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[93921]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3947/84 objects degraded (4698.810%), 1 pg degraded, 1 pg undersized) 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[93921]: osdmap e475: 8 total, 8 up, 8 in 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1936748398' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[98970]: pgmap v1690: 103 pgs: 2 active+clean+scrubbing+deep, 2 active+clean+scrubbing, 99 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[98970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3947/84 objects degraded (4698.810%), 1 pg degraded, 1 pg undersized) 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[98970]: osdmap e475: 8 total, 8 up, 8 in 2024-08-29T15:44:56.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1936748398' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:44:56.242 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:56 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:44:56.261 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:56.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:56.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:56.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:56.410 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:56.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:56.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:56 smithi171 ceph-mon[90031]: pgmap v1690: 103 pgs: 2 active+clean+scrubbing+deep, 2 active+clean+scrubbing, 99 active+clean; 4.6 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:44:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:56 smithi171 ceph-mon[90031]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3947/84 objects degraded (4698.810%), 1 pg degraded, 1 pg undersized) 2024-08-29T15:44:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:56 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:44:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:56 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-29T15:44:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:56 smithi171 ceph-mon[90031]: osdmap e475: 8 total, 8 up, 8 in 2024-08-29T15:44:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:56 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1936748398' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:44:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:56 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:44:56.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:56.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:56.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.661 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:56.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:56.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:56.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:56.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:56.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:56.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:56.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:57.011 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:57.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:57.139 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:57.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.166 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:57.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:57.267 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:57.291 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:57.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:57.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:57 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:44:57.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:57 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:44:57.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:57 smithi171 ceph-mon[90031]: osdmap e476: 8 total, 8 up, 8 in 2024-08-29T15:44:57.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:57 smithi171 ceph-mon[90031]: osdmap e477: 8 total, 8 up, 8 in 2024-08-29T15:44:57.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:57.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.544 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:57.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:44:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:44:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[98970]: osdmap e476: 8 total, 8 up, 8 in 2024-08-29T15:44:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[98970]: osdmap e477: 8 total, 8 up, 8 in 2024-08-29T15:44:57.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:44:57.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:44:57.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[93921]: osdmap e476: 8 total, 8 up, 8 in 2024-08-29T15:44:57.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:57 smithi027 ceph-mon[93921]: osdmap e477: 8 total, 8 up, 8 in 2024-08-29T15:44:57.621 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-29T15:44:57.622 INFO:tasks.radosbench:Beginning radosbench... 2024-08-29T15:44:57.622 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-08-29T15:44:57.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-08-29T15:44:57.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:57.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:57.772 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:57.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:57.865 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:57.899 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:57.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:57.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:58.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:58.047 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:58.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:58.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:58.172 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:58.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:58.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:44:58.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:58.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:44:58.400 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:58.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:58 smithi171 ceph-mon[90031]: pgmap v1693: 103 pgs: 10 remapped+peering, 2 active+clean+scrubbing+deep, 2 active+clean+scrubbing, 89 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:44:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:58 smithi171 ceph-mon[90031]: osdmap e478: 8 total, 8 up, 8 in 2024-08-29T15:44:58.526 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:58.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:58 smithi027 ceph-mon[98970]: pgmap v1693: 103 pgs: 10 remapped+peering, 2 active+clean+scrubbing+deep, 2 active+clean+scrubbing, 89 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:44:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:58 smithi027 ceph-mon[98970]: osdmap e478: 8 total, 8 up, 8 in 2024-08-29T15:44:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:58 smithi027 ceph-mon[93921]: pgmap v1693: 103 pgs: 10 remapped+peering, 2 active+clean+scrubbing+deep, 2 active+clean+scrubbing, 89 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:44:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:58 smithi027 ceph-mon[93921]: osdmap e478: 8 total, 8 up, 8 in 2024-08-29T15:44:58.651 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:58.674 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:58.776 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:44:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:58.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:44:58.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:58.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:58.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:58.973 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:44:58.973 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:44:58.988 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:44:58.988 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:44:59.006 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:44:59.007 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:44:59.012 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:44:59.027 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:44:59.027 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:44:59.027 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:44:59.047 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:44:59.047 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:44:59.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:59.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:44:59.064 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:44:59.065 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:44:59.081 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:44:59.081 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:44:59.097 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:44:59.098 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:44:59.135 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:44:59] "GET /metrics HTTP/1.1" 200 34209 "" "Prometheus/2.51.0" 2024-08-29T15:44:59.154 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:44:59.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:44:59.279 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:59.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:44:59.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:44:59.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:44:59.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:44:59.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:59.549 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:59.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:59.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.675 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:59.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:59.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:44:59.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:44:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:44:59.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:44:59.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:44:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:44:59.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:59 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/32650901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:59 smithi171 ceph-mon[90031]: pgmap v1696: 103 pgs: 1 peering, 3 active+undersized+remapped, 9 remapped+peering, 1 active+clean+scrubbing, 89 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:00.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:44:59 smithi171 ceph-mon[90031]: osdmap e479: 8 total, 8 up, 8 in 2024-08-29T15:45:00.024 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.048 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:00.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/32650901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:00.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[98970]: pgmap v1696: 103 pgs: 1 peering, 3 active+undersized+remapped, 9 remapped+peering, 1 active+clean+scrubbing, 89 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[98970]: osdmap e479: 8 total, 8 up, 8 in 2024-08-29T15:45:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/32650901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[93921]: pgmap v1696: 103 pgs: 1 peering, 3 active+undersized+remapped, 9 remapped+peering, 1 active+clean+scrubbing, 89 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:00.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:44:59 smithi027 ceph-mon[93921]: osdmap e479: 8 total, 8 up, 8 in 2024-08-29T15:45:00.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:00.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:00.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:00.298 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:00.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:00.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.424 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:00.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:00.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:00.551 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:00.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:00.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:00.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:00.778 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:00.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:00.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:00.912 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:00 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:45:00.912 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:00 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:45:00.931 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:00.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:00.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:00 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:45:01.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:01.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:01.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:01.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:01.184 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:01.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:01.311 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:01.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:01.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:01.436 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:01.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:01.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:01.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:01.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:01.594 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:45:01.594 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:45:01.665 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:01.690 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:01.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:01.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:01.815 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:01.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:01.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:01 smithi027 ceph-mon[98970]: pgmap v1698: 103 pgs: 1 peering, 3 active+undersized+remapped, 9 remapped+peering, 90 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 5 op/s; 16 KiB/s, 0 objects/s recovering 2024-08-29T15:45:01.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:01 smithi027 ceph-mon[93921]: pgmap v1698: 103 pgs: 1 peering, 3 active+undersized+remapped, 9 remapped+peering, 90 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 5 op/s; 16 KiB/s, 0 objects/s recovering 2024-08-29T15:45:01.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:01.940 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:01.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:01.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:02.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:01 smithi171 ceph-mon[90031]: pgmap v1698: 103 pgs: 1 peering, 3 active+undersized+remapped, 9 remapped+peering, 90 active+clean; 4.6 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 5 op/s; 16 KiB/s, 0 objects/s recovering 2024-08-29T15:45:02.042 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:02.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:02.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:02.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:02.196 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:02.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:45:02.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:02.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:02.429 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:02.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:02.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:02.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:02.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:45:02.679 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:02.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:02.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:02.773 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:02.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:02.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:02.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:02.956 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:02.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:02.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:03.058 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:03.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:03.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:03.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:03.311 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:03.337 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:03.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.462 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:03.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:03.509 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:03.563 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:03.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:03.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:03.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:03.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:03.811 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:03.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:03.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:03.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:03.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:03.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:04.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:04.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:04.188 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:04.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:04.222 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:04 smithi027 ceph-mon[98970]: pgmap v1699: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 8 op/s; 101 KiB/s, 0 objects/s recovering 2024-08-29T15:45:04.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:04 smithi027 ceph-mon[93921]: pgmap v1699: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 8 op/s; 101 KiB/s, 0 objects/s recovering 2024-08-29T15:45:04.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:04.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:04.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:04.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:04.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:04 smithi171 ceph-mon[90031]: pgmap v1699: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 8 op/s; 101 KiB/s, 0 objects/s recovering 2024-08-29T15:45:04.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:04.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:04.693 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:04.716 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:04.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:04.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:04.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:04.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:04.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:04.964 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:04.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:05.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:05.089 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:05.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:05.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:05.212 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:05.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:05.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/585436665' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:05.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/585436665' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-29T15:45:05.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:05.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:05.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:05.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:05.469 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:05 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:45:05.469 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:05 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:05.469 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:05 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:05.469 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/585436665' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-29T15:45:05.545 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:05.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:05.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:05.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:05.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:05.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:05.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:05.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:05.865 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_5' created 2024-08-29T15:45:05.868 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:45:05.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:05.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:05.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:06.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:06.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:06.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:06.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:06.306 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: pgmap v1700: 103 pgs: 103 active+clean; 4.4 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 9.5 KiB/s rd, 28 KiB/s wr, 6 op/s; 77 KiB/s, 0 objects/s recovering 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4149538776' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/585436665' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:45:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[98970]: osdmap e480: 8 total, 8 up, 8 in 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: pgmap v1700: 103 pgs: 103 active+clean; 4.4 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 9.5 KiB/s rd, 28 KiB/s wr, 6 op/s; 77 KiB/s, 0 objects/s recovering 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4149538776' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/585436665' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:45:06.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:06 smithi027 ceph-mon[93921]: osdmap e480: 8 total, 8 up, 8 in 2024-08-29T15:45:06.328 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: pgmap v1700: 103 pgs: 103 active+clean; 4.4 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 9.5 KiB/s rd, 28 KiB/s wr, 6 op/s; 77 KiB/s, 0 objects/s recovering 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4149538776' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/585436665' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:45:06.416 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:06 smithi171 ceph-mon[90031]: osdmap e480: 8 total, 8 up, 8 in 2024-08-29T15:45:06.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:06.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:06.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:06.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:06.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:06.703 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:06.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:06.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:06.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:06.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:06.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:06.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:06.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:06.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:07.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:07.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:07.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:07.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:07.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:07.287 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:45:07.287 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:45:07.303 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:07.304 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:45:07.305 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:45:07.322 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:45:07.322 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:45:07.323 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:07.340 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:45:07.340 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:45:07.357 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:45:07.358 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:45:07.374 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:45:07.374 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:45:07.391 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:45:07.391 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:45:07.407 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:45:07.408 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:45:07.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:07.446 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:07.548 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:07.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:07.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:07.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:07.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:07.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:07.799 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:07.822 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:07.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:07.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:07.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:08.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-08-29T15:45:08.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:08.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:08.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:08.195 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:07 smithi171 ceph-mon[90031]: osdmap e481: 8 total, 8 up, 8 in 2024-08-29T15:45:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:07 smithi171 ceph-mon[90031]: pgmap v1703: 119 pgs: 5 creating+peering, 11 unknown, 103 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 9.5 KiB/s rd, 28 KiB/s wr, 6 op/s; 77 KiB/s, 0 objects/s recovering 2024-08-29T15:45:08.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:07 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1643266285' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:08.298 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:07 smithi027 ceph-mon[93921]: osdmap e481: 8 total, 8 up, 8 in 2024-08-29T15:45:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:07 smithi027 ceph-mon[93921]: pgmap v1703: 119 pgs: 5 creating+peering, 11 unknown, 103 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 9.5 KiB/s rd, 28 KiB/s wr, 6 op/s; 77 KiB/s, 0 objects/s recovering 2024-08-29T15:45:08.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:07 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1643266285' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:07 smithi027 ceph-mon[98970]: osdmap e481: 8 total, 8 up, 8 in 2024-08-29T15:45:08.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:07 smithi027 ceph-mon[98970]: pgmap v1703: 119 pgs: 5 creating+peering, 11 unknown, 103 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 9.5 KiB/s rd, 28 KiB/s wr, 6 op/s; 77 KiB/s, 0 objects/s recovering 2024-08-29T15:45:08.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:07 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1643266285' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:08.321 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:08.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:08.443 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:08.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:08.569 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:08.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:08.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:08.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.693 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:08.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:08.795 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:08.816 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:08.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:08.939 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:08.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:08.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:09.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:09.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:09.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:09.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:09.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:09.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:09.220 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:08 smithi027 ceph-mon[98970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:45:09.220 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:08 smithi027 ceph-mon[98970]: osdmap e482: 8 total, 8 up, 8 in 2024-08-29T15:45:09.220 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:08 smithi027 ceph-mon[93921]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:45:09.220 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:08 smithi027 ceph-mon[93921]: osdmap e482: 8 total, 8 up, 8 in 2024-08-29T15:45:09.221 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:45:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:45:09] "GET /metrics HTTP/1.1" 200 34209 "" "Prometheus/2.51.0" 2024-08-29T15:45:09.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:08 smithi171 ceph-mon[90031]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:45:09.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:08 smithi171 ceph-mon[90031]: osdmap e482: 8 total, 8 up, 8 in 2024-08-29T15:45:09.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:09.310 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:09.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:09.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:09.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:09.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.558 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:09.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:09.659 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:09.680 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:09.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:09.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:09.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:09.932 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:09.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:09.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:10.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:10.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:10.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:10.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:10.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:10 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:10.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:10 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:10.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:10 smithi027 ceph-mon[98970]: pgmap v1705: 119 pgs: 5 creating+peering, 8 unknown, 106 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:10.272 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:10 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:10.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:10 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:10.273 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:10 smithi027 ceph-mon[93921]: pgmap v1705: 119 pgs: 5 creating+peering, 8 unknown, 106 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:10.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:10.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:10.396 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:45:10.397 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:45:10.397 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:45:10.397 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:45:10.397 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:45:10.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:10.429 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:10.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:10 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:10 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:10 smithi171 ceph-mon[90031]: pgmap v1705: 119 pgs: 5 creating+peering, 8 unknown, 106 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:10.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:10.554 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:10.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:10.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:10.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:10.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:10.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:10.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:10.927 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:10.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:10.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:11.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:11.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.053 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:11.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:11.105 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:45:11.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:11.178 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:11.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:11.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:11.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:11.427 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:11 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:45:11.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:11.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:11 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:45:11.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:11 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:45:11.654 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:11.679 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:11.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:11.680 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:11.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:11.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:11.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:11.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:11.934 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:11.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:11.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:11.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:45:12.036 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:12.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:12.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:12.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:12.189 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:12.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:12.314 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:12.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:12 smithi027 ceph-mon[93921]: pgmap v1706: 119 pgs: 5 creating+peering, 114 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:12 smithi027 ceph-mon[98970]: pgmap v1706: 119 pgs: 5 creating+peering, 114 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:12.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:12.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:12.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:12 smithi171 ceph-mon[90031]: pgmap v1706: 119 pgs: 5 creating+peering, 114 active+clean; 4.4 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2024-08-29T15:45:12.541 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:12.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.566 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:12.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:12.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:12.692 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:12.794 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:12.819 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:12.822 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:12.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:12.942 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:12.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:13.044 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:13.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:13.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:13.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:13.194 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:13.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:13.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:13.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:13.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:13.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:13.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:13.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:13.572 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:13.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:45:13.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:13.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:13.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:13.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:13.827 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:13.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:13.952 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:13.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:14.054 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:14.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:14.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:14.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:14.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:14.329 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:14.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:14.431 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:14.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:14.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:14 smithi027 ceph-mon[93921]: pgmap v1707: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:14.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:14.478 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:14.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:14 smithi027 ceph-mon[98970]: pgmap v1707: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:14.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:14.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:14 smithi171 ceph-mon[90031]: pgmap v1707: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:14.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:14.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:14.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:14.709 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:14.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:14.811 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:14.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:14.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:14.963 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:14.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:15.049 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:45:15.049 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:45:15.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185356 8.c 8:3e65ed30:::benchmark_data_smithi027_306999_object58577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 458.68703123400002, 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.954196238, 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.050 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185356 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347620+0000", 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347621+0000", 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4959999999999999e-06 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347626+0000", 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7110000000000003e-06 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347636+0000", 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7690000000000008e-06 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347643+0000", 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9439999999999999e-06 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347708+0000", 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5822999999999996e-05 2024-08-29T15:45:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.347924+0000", 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021531600000000001 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.348044+0000", 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120091 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359802+0000", 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011758006 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359853+0000", 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1329999999999998e-05 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390576+0000", 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030722635000000002 2024-08-29T15:45:15.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390589+0000", 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2792000000000001e-05 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085219+0000", 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69463000399999997 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085233+0000", 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3964e-05 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.301754+0000", 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2165213420000001 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.301816+0000", 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2014999999999995e-05 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185365 8.c 8:34b703f9:::benchmark_data_smithi027_306999_object58586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 458.68078831999998, 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.948337904, 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.058 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185365 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:45:15.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353863+0000", 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353864+0000", 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.136e-06 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353867+0000", 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4340000000000001e-06 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353876+0000", 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8980000000000002e-06 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.353882+0000", 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9800000000000003e-06 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359882+0000", 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0059996040000000004 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359992+0000", 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110373 2024-08-29T15:45:15.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.360098+0000", 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010622499999999999 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390623+0000", 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.030524273000000001 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390631+0000", 2024-08-29T15:45:15.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9530000000000005e-06 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085266+0000", 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69463474700000005 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085276+0000", 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3610000000000007e-06 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302132+0000", 2024-08-29T15:45:15.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2168561900000001 2024-08-29T15:45:15.065 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.065 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302200+0000", 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8730000000000001e-05 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185370 8.c 8:321ac8e5:::benchmark_data_smithi027_306999_object58591:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 458.67515415899999, 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.942814085, 2024-08-29T15:45:15.067 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185370 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:45:15.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359497+0000", 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359498+0000", 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7150000000000001e-06 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359504+0000", 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9279999999999998e-06 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359518+0000", 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3708e-05 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.359526+0000", 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5100000000000001e-06 2024-08-29T15:45:15.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.367931+0000", 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008405615 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.368066+0000", 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013519199999999999 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 5,7", 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.368197+0000", 2024-08-29T15:45:15.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013018200000000001 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390661+0000", 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022464378 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390669+0000", 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1960000000000008e-06 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.072 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085313+0000", 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69464367800000004 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085321+0000", 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0919999999999998e-06 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302245+0000", 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2169240729999999 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.073 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302311+0000", 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5817999999999999e-05 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185373 8.c 8:3539b1da:::benchmark_data_smithi027_306999_object58594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 458.671438733, 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9786454089999999, 2024-08-29T15:45:15.074 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185373 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.075 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363212+0000", 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363214+0000", 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4559999999999999e-06 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363221+0000", 2024-08-29T15:45:15.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0700000000000001e-06 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363234+0000", 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3397999999999999e-05 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.363244+0000", 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7629999999999995e-06 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390698+0000", 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027454427 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.390706+0000", 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2250000000000003e-06 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085347+0000", 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69464127399999998 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.078 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085355+0000", 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3180000000000007e-06 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302347+0000", 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.216992074 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.302464+0000", 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116505 2024-08-29T15:45:15.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.303107+0000", 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064362000000000002 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329229+0000", 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.026121961999999999 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329277+0000", 2024-08-29T15:45:15.080 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7749e-05 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338695+0000", 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094183740000000002 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338857+0000", 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016147700000000001 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.339098+0000", 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024068099999999999 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341858+0000", 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002760036 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.082 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185396 8.c 8:37b7f43f:::benchmark_data_smithi027_306999_object58617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 458.62604357999999, 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9334865830000001, 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185396 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.083 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408607+0000", 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408609+0000", 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.114e-06 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408615+0000", 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9900000000000002e-06 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408631+0000", 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5426000000000001e-05 2024-08-29T15:45:15.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408639+0000", 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3499999999999997e-06 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408716+0000", 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6490999999999997e-05 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.408726+0000", 2024-08-29T15:45:15.086 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0570999999999999e-05 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085379+0000", 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.67665228200000005 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085386+0000", 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8730000000000007e-06 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.087 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319619+0000", 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2342329700000001 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319745+0000", 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125611 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.319880+0000", 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134673 2024-08-29T15:45:15.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329291+0000", 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094111300000000002 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329493+0000", 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020221700000000001 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.089 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338735+0000", 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092423220000000007 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341916+0000", 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031804369999999999 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.341975+0000", 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9617000000000002e-05 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.090 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342094+0000", 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118509 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:185406 8.c 8:388927f1:::benchmark_data_smithi027_306999_object58627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 458.60638928399999, 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9141273249999999, 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.091 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185406 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.092 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428262+0000", 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1699999999999997e-07 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428267+0000", 2024-08-29T15:45:15.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1339999999999997e-06 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428278+0000", 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0927999999999999e-05 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428283+0000", 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1510000000000003e-06 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.094 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428329+0000", 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6029000000000003e-05 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:36.428334+0000", 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3979999999999998e-06 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085411+0000", 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.657076836 2024-08-29T15:45:15.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:37.085419+0000", 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.644e-06 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.323864+0000", 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2384456589999999 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.096 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.323988+0000", 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012354099999999999 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.324118+0000", 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129987 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.329297+0000", 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051789770000000004 2024-08-29T15:45:15.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338774+0000", 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094772769999999992 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.338812+0000", 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7799000000000002e-05 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.098 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342228+0000", 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034166130000000002 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342282+0000", 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3149999999999998e-05 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:37:38.342389+0000", 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107385 2024-08-29T15:45:15.099 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210331 8.c 8:3aae235d:::benchmark_data_smithi027_306999_object83552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e381)", 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 425.01125264799998, 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.042995135, 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:45:15.100 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210331 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023398+0000", 2024-08-29T15:45:15.101 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023400+0000", 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.875e-06 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023406+0000", 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5149999999999997e-06 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.102 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023422+0000", 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6733999999999999e-05 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.023430+0000", 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7279999999999995e-06 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024594+0000", 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011642289999999999 2024-08-29T15:45:15.103 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024726+0000", 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013213600000000001 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.024866+0000", 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013915999999999999 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.033244+0000", 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008378155 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.034487+0000", 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012431300000000001 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.035958+0000", 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001471539 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.040150+0000", 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041912520000000003 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.040161+0000", 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1744e-05 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066310+0000", 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.02614841 2024-08-29T15:45:15.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066393+0000", 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3528000000000004e-05 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210341 8.c 8:3b7d0476:::benchmark_data_smithi027_306999_object83562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 424.99587610999998, 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0358083629999999, 2024-08-29T15:45:15.107 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210341 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.108 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038775+0000", 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038776+0000", 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.409e-06 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038781+0000", 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5680000000000001e-06 2024-08-29T15:45:15.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038791+0000", 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0149000000000001e-05 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.038799+0000", 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8630000000000008e-06 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.041098+0000", 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022990480000000002 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.041106+0000", 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0080000000000002e-06 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066885+0000", 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0257787739999999 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.066990+0000", 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010541500000000001 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.067139+0000", 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014869899999999999 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074258+0000", 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007119383 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074420+0000", 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016190599999999999 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074489+0000", 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8998000000000006e-05 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074542+0000", 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2697e-05 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074583+0000", 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1446000000000003e-05 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210361 8.c 8:36e2a08e:::benchmark_data_smithi027_306999_object83582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:45:15.114 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 424.97404298599997, 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.026884127, 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210361 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.115 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060608+0000", 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060609+0000", 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5700000000000002e-07 2024-08-29T15:45:15.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060613+0000", 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0419999999999996e-06 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060624+0000", 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0821e-05 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060630+0000", 2024-08-29T15:45:15.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2890000000000003e-06 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060687+0000", 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7071000000000003e-05 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.060695+0000", 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.605e-06 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070807+0000", 2024-08-29T15:45:15.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.010112353 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070906+0000", 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8680999999999999e-05 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.070983+0000", 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7495000000000003e-05 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074476+0000", 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003492673 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074595+0000", 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119187 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087278+0000", 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012683589 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087441+0000", 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016204500000000001 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087492+0000", 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1518999999999999e-05 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210366 8.c 8:3bfc388b:::benchmark_data_smithi027_306999_object83587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 424.96569453699999, 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0187623729999999, 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210366 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.122 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068956+0000", 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068958+0000", 2024-08-29T15:45:15.123 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3519999999999999e-06 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068962+0000", 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7690000000000004e-06 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068975+0000", 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.217e-05 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.124 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.068982+0000", 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3080000000000002e-06 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.069045+0000", 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2791000000000002e-05 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.069054+0000", 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5640000000000002e-06 2024-08-29T15:45:15.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074624+0000", 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.005569739 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074808+0000", 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184026 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:45:15.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.074951+0000", 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014334600000000001 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087555+0000", 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012603130000000001 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087570+0000", 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5834000000000001e-05 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087625+0000", 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4398999999999999e-05 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087681+0000", 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6036999999999999e-05 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087719+0000", 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7907999999999997e-05 2024-08-29T15:45:15.128 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210405 8.c 8:3354f347:::benchmark_data_smithi027_306999_object83626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 424.9219612, 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98371990300000001, 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.129 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210405 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112690+0000", 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112691+0000", 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5740000000000001e-06 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112697+0000", 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5999999999999997e-06 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112714+0000", 2024-08-29T15:45:15.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6898e-05 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112721+0000", 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6669999999999996e-06 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112782+0000", 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0148000000000001e-05 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.112790+0000", 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1089999999999998e-06 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083315+0000", 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97052495400000005 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083455+0000", 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139925 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.083591+0000", 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013614500000000001 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.091643+0000", 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080527510000000004 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.095910+0000", 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042662530000000002 2024-08-29T15:45:15.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.095950+0000", 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0290000000000002e-05 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096361+0000", 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041099000000000002 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096410+0000", 2024-08-29T15:45:15.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8599e-05 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:210417 8.c 8:351de282:::benchmark_data_smithi027_306999_object83638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 424.907386207, 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97409499899999996, 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.136 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 210417 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127265+0000", 2024-08-29T15:45:15.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127266+0000", 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.291e-06 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127271+0000", 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3689999999999998e-06 2024-08-29T15:45:15.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127281+0000", 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9320000000000008e-06 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127289+0000", 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5730000000000002e-06 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127353+0000", 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4462999999999996e-05 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:10.127362+0000", 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1330000000000005e-06 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087778+0000", 2024-08-29T15:45:15.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96041514900000002 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.087926+0000", 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014825999999999999 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.088065+0000", 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013917199999999999 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.096459+0000", 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.008393546 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101179+0000", 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047199649999999996 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101267+0000", 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8187000000000001e-05 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101319+0000", 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2432999999999998e-05 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:11.101360+0000", 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0525999999999998e-05 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236777 8.f 8:f2ec1397:::benchmark_data_smithi027_306999_object109998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.236625+0000", 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.79802551900002, 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0719445379999999, 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236777 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236625+0000", 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236625+0000", 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236627+0000", 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5519999999999999e-06 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236632+0000", 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3489999999999999e-06 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236643+0000", 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1117e-05 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.236649+0000", 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8340000000000004e-06 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.249513+0000", 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012863556999999999 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.249647+0000", 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013372500000000001 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2,4", 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.249786+0000", 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013996700000000001 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260509+0000", 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010722109000000001 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260550+0000", 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1396999999999997e-05 2024-08-29T15:45:15.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260587+0000", 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6711000000000001e-05 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278680+0000", 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018093046000000002 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278688+0000", 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2740000000000007e-06 2024-08-29T15:45:15.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.308488+0000", 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.029800271 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.308570+0000", 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1629000000000001e-05 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236783 8.f 8:f3550e7b:::benchmark_data_smithi027_306999_object110004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e393)", 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.237275+0000", 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.79737621200002, 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.071859028, 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236783 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237275+0000", 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237275+0000", 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237276+0000", 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2900000000000002e-07 2024-08-29T15:45:15.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237280+0000", 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6160000000000004e-06 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237291+0000", 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0828e-05 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.237298+0000", 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8360000000000003e-06 2024-08-29T15:45:15.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260617+0000", 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.023319096000000001 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260753+0000", 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013595 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2,4", 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.260899+0000", 2024-08-29T15:45:15.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014577500000000001 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.264873+0000", 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039747280000000003 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278798+0000", 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013924153 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.278808+0000", 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9899999999999992e-06 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309071+0000", 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0302634500000001 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309134+0000", 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2676999999999996e-05 2024-08-29T15:45:15.157 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236829 8.f 8:f7cd12c4:::benchmark_data_smithi027_306999_object110050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.316115+0000", 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.71853550100002, 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0129513210000001, 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.158 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236829 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316115+0000", 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316115+0000", 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316117+0000", 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.19e-06 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316121+0000", 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6380000000000003e-06 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316134+0000", 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2989e-05 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316141+0000", 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7569999999999998e-06 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316213+0000", 2024-08-29T15:45:15.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1662999999999995e-05 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.316222+0000", 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6209999999999999e-06 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309160+0000", 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99293816300000004 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309252+0000", 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1131000000000006e-05 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.309357+0000", 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105801 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.316947+0000", 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075896039999999998 2024-08-29T15:45:15.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.317335+0000", 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038852800000000002 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.328895+0000", 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011559908000000001 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329020+0000", 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124146 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329067+0000", 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7182000000000003e-05 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236830 8.f 8:f2d34e4d:::benchmark_data_smithi027_306999_object110051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:45:15.165 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.319225+0000", 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.71542574400002, 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0100289629999999, 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236830 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319225+0000", 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319225+0000", 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319226+0000", 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316e-06 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319232+0000", 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1630000000000004e-06 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319245+0000", 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3664e-05 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319253+0000", 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2409999999999998e-06 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319324+0000", 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1743999999999996e-05 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.319333+0000", 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3550000000000005e-06 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.312729+0000", 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99339648999999997 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.312869+0000", 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013976799999999999 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.312996+0000", 2024-08-29T15:45:15.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127391 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.317317+0000", 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043208580000000003 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.317341+0000", 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3623999999999999e-05 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329111+0000", 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011769998 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329211+0000", 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100616 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329254+0000", 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2735000000000003e-05 2024-08-29T15:45:15.172 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236837 8.f 8:ff15e796:::benchmark_data_smithi027_306999_object110058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.325967+0000", 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.708683909, 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0078366379999999, 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236837 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325967+0000", 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325967+0000", 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325969+0000", 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.592e-06 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325973+0000", 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8600000000000001e-06 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325989+0000", 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5569e-05 2024-08-29T15:45:15.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.325996+0000", 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9399999999999996e-06 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.326261+0000", 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026476199999999998 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.326269+0000", 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8040000000000002e-06 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.320687+0000", 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99441820299999995 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.320787+0000", 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010012499999999999 2024-08-29T15:45:15.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.320868+0000", 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1366000000000007e-05 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329434+0000", 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085656870000000006 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329509+0000", 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5117000000000006e-05 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333707+0000", 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041977300000000002 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333757+0000", 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0417999999999997e-05 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333804+0000", 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6465000000000003e-05 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.179 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236843 8.f 8:f7533c45:::benchmark_data_smithi027_306999_object110064:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.330976+0000", 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.70367462799999, 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0029207099999999, 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.180 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236843 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330976+0000", 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330976+0000", 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330978+0000", 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0109999999999999e-06 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330984+0000", 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4609999999999999e-06 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.330996+0000", 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2041e-05 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.331003+0000", 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4889999999999998e-06 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.334063+0000", 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030598380000000001 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.334071+0000", 2024-08-29T15:45:15.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8679999999999999e-06 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.324739+0000", 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99066757900000002 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.324856+0000", 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117119 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.324977+0000", 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012087900000000001 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329465+0000", 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044884139999999996 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329550+0000", 2024-08-29T15:45:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4844000000000003e-05 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333812+0000", 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004262493 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333857+0000", 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4994000000000003e-05 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333897+0000", 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.968e-05 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236872 8.f 8:f57e9354:::benchmark_data_smithi027_306999_object110093:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.368172+0000", 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.66647844900001, 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97553822099999998, 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236872 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368172+0000", 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368172+0000", 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368175+0000", 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2390000000000001e-06 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368180+0000", 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7950000000000004e-06 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368195+0000", 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4482e-05 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368203+0000", 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5210000000000002e-06 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368322+0000", 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011834599999999999 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.368330+0000", 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0620000000000001e-06 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329581+0000", 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96125158899999996 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329686+0000", 2024-08-29T15:45:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010479 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.329803+0000", 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116474 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333943+0000", 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041398049999999999 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.333956+0000", 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3508e-05 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343595+0000", 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096386920000000008 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343671+0000", 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6726999999999999e-05 2024-08-29T15:45:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343711+0000", 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9190999999999998e-05 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46475.0:236891 8.f 8:fb7fc2c9:::benchmark_data_smithi027_306999_object110112:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e394)", 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:38:46.392116+0000", 2024-08-29T15:45:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.64253465899998, 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97105291000000005, 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46475", 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 236891 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392116+0000", 2024-08-29T15:45:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392116+0000", 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392118+0000", 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.325e-06 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392123+0000", 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1580000000000004e-06 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392135+0000", 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2327000000000001e-05 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392143+0000", 2024-08-29T15:45:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9000000000000006e-06 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392219+0000", 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.593e-05 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:46.392229+0000", 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9410000000000002e-06 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:45:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.343974+0000", 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95174541999999995 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.344082+0000", 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107692 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.344155+0000", 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2698000000000001e-05 2024-08-29T15:45:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.352435+0000", 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082806140000000004 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.352452+0000", 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6252999999999998e-05 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.362986+0000", 2024-08-29T15:45:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010534109999999999 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.363124+0000", 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138052 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:38:47.363169+0000", 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.549e-05 2024-08-29T15:45:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:45:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:45:15.202 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:45:15.203 INFO:teuthology.orchestra.run.smithi027.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-08-29T15:45:15.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:15.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:15.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:15.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:15.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:15 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3088010827' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:45:15.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:15 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:45:15.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:15 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3088010827' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:45:15.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:15 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:45:15.330 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:15.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:15.431 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:15.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:15.456 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:15.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:15 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3088010827' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:45:15.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:15 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-29T15:45:15.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:15.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:15.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:15.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:15.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:15.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:15.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:15.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:15.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:15.831 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:15.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:15.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:15.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:15.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:15.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:16.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:16.083 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:16.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:16.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:16.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:16.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:16.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:16.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:16.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:16.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:16.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:16.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:16 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:45:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:16 smithi171 ceph-mon[90031]: pgmap v1708: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:16 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:45:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:16 smithi171 ceph-mon[90031]: osdmap e483: 8 total, 8 up, 8 in 2024-08-29T15:45:16.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:45:16.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[98970]: pgmap v1708: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:16.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:45:16.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[98970]: osdmap e483: 8 total, 8 up, 8 in 2024-08-29T15:45:16.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:45:16.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[93921]: pgmap v1708: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:16.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-29T15:45:16.531 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:16 smithi027 ceph-mon[93921]: osdmap e483: 8 total, 8 up, 8 in 2024-08-29T15:45:16.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:16.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:16.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:16.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:16.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:16.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:16.806 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:16.828 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:16.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:16.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:16.958 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:16.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:16.974 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:45:17.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:17.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:17.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:17.173 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:17.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:17.202 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:17 smithi027 ceph-mon[93921]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:45:17.214 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:17.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:17 smithi171 ceph-mon[90031]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:45:17.316 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:17.339 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:17.441 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:17.465 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:17.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:17.560 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:17 smithi027 ceph-mon[98970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:45:17.568 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:17.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:17.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:17.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:17.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:17.838 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:17.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:17.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:17.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:18.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:18.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:18.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:18.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:18.212 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:18.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:18.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:18 smithi171 ceph-mon[90031]: pgmap v1710: 119 pgs: 119 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:18.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:18.335 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:18.337 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:18.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:18.366 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:18.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:18.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:18 smithi027 ceph-mon[93921]: pgmap v1710: 119 pgs: 119 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:18.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:18 smithi027 ceph-mon[98970]: pgmap v1710: 119 pgs: 119 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:18.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:18.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:18.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:18.563 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:18.583 DEBUG:teuthology.orchestra.run.smithi027:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-08-29T15:45:18.584 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-29T15:45:18.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:45:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:18.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:18.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:18.586 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-29T15:45:18.586 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-29T15:45:18.650 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:hints = 1 2024-08-29T15:45:18.651 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-29T15:45:18.651 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object prefix: benchmark_data_smithi027_374610 2024-08-29T15:45:18.651 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:45:18.651 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 0 0 0 0 0 0 - 0 2024-08-29T15:45:18.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:18.724 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:18.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:18.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:18.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:18.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:18.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:19.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:19.064 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:1332 10.d 10:bf6ee46a:::benchmark_data_smithi027_374610_object1331:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:19.061331+0000", 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00076616399999999995, 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00080145699999999999, 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:45:19.065 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 1332 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061331+0000", 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061331+0000", 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.066 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061330+0000", 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061546+0000", 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021589299999999999 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061556+0000", 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4979999999999995e-06 2024-08-29T15:45:19.067 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061563+0000", 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8589999999999999e-06 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061605+0000", 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2896999999999999e-05 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:19.068 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061780+0000", 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017429500000000001 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 3,6", 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061922+0000", 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014197799999999999 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:1331 10.0 e483/480 10:06ca9fed:::benchmark_data_smithi027_374610_object1330:head v 483'108, mlcod=483'104)", 2024-08-29T15:45:19.069 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:19.061602+0000", 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00049569800000000004, 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00061173200000000003, 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061602+0000", 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061602+0000", 2024-08-29T15:45:19.070 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061601+0000", 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061633+0000", 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1704e-05 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.071 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061642+0000", 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3950000000000007e-06 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061648+0000", 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9059999999999999e-06 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061694+0000", 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6078999999999997e-05 2024-08-29T15:45:19.072 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061721+0000", 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6659e-05 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:1334 10.4 10:292ff3d0:::benchmark_data_smithi027_374610_object1333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e483)", 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:19.061577+0000", 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.000520533, 2024-08-29T15:45:19.073 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00070305599999999997, 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 1334 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061577+0000", 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.074 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061577+0000", 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061576+0000", 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061767+0000", 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019023099999999999 2024-08-29T15:45:19.075 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061776+0000", 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3959999999999995e-06 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061782+0000", 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5030000000000002e-06 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:19.076 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061865+0000", 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2194999999999995e-05 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.061973+0000", 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010877 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062100+0000", 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012692899999999999 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:19.077 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop_reply(client.47036.0:1325 10.9 e483/480 ondisk, result = 0)", 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:19.062079+0000", 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 1.8343e-05, 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000263441, 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062079+0000", 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.078 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062079+0000", 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062079+0000", 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:19.079 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062082+0000", 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2899999999999998e-06 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062088+0000", 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5130000000000002e-06 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062097+0000", 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3270000000000001e-06 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.080 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062273+0000", 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017642299999999999 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:19.062279+0000", 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6370000000000004e-06 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 4 2024-08-29T15:45:19.081 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:45:19.107 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:19.139 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:19.241 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:19.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:19.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:19.304 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:45:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:45:19] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-29T15:45:19.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:19.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:19.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:19.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:19.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:19.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:19.641 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:19.651 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 1 16 3404 3388 211.737 211.75 0.00543739 0.00470799 2024-08-29T15:45:19.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.680 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:19.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:19.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:19.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:19.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:19.920 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:19.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:45:19.997 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:19.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:20.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:20.139 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:20.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:20.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:20.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:20.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:20.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:20.319 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:20.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:20.417 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:20.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:20.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:20.509 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:20 smithi171 ceph-mon[90031]: pgmap v1711: 119 pgs: 119 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:20.519 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:20.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:20 smithi027 ceph-mon[93921]: pgmap v1711: 119 pgs: 119 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:20.543 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:20 smithi027 ceph-mon[98970]: pgmap v1711: 119 pgs: 119 active+clean; 4.4 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:45:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:20.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:20.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:20.651 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 2 16 6851 6835 213.578 215.438 0.00134192 0.0046285 2024-08-29T15:45:20.664 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:20.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:20.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:20.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:20.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:20.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:20.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:20.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:20.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:20.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:20.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:21.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:21.129 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:21.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:21.230 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:21.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:21.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:21.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:21.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:21.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:21.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:21.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:21 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:45:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:45:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:45:21.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:21 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.545 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:21.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:45:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:45:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:21 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:21.647 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:21.651 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 3 16 9874 9858 205.359 188.938 0.00569692 0.00486391 2024-08-29T15:45:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:21.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:21.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:21.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:21.827 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:21.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:21.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:21.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:21.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:22.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:22.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:22.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:22.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:22.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:22.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:22.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:22.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:22.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:22.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:22.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:22.552 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:22.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:22 smithi027 ceph-mon[93921]: pgmap v1712: 119 pgs: 119 active+clean; 143 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s wr, 221 op/s 2024-08-29T15:45:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:22 smithi027 ceph-mon[98970]: pgmap v1712: 119 pgs: 119 active+clean; 143 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s wr, 221 op/s 2024-08-29T15:45:22.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:22.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 4 16 12874 12858 200.057 187.5 0.00607274 0.00499168 2024-08-29T15:45:22.694 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:22.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:22.741 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:45:22.742 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:45:22.742 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:45:22.742 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:45:22.742 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:45:22.756 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:22 smithi171 ceph-mon[90031]: pgmap v1712: 119 pgs: 119 active+clean; 143 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s wr, 221 op/s 2024-08-29T15:45:22.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:22.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:22.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:22.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:22.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:22.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:22.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:23.029 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:45:23.029 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:45:23.030 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:45:23.031 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:45:23.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:23.071 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:45:23.072 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:45:23.108 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:45:23.108 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:45:23.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:23.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:23.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:23.146 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:45:23.146 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:45:23.184 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:45:23.185 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:45:23.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:23.224 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:45:23.224 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:45:23.251 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:23.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:23.263 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:45:23.263 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:45:23.352 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:23.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:23.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:23.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:23.489 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:23.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:45:23.518 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:23.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:23.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:23.621 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:23.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:23.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:23.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:23.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 5 16 15297 15281 190.363 151.438 0.00530562 0.00524519 2024-08-29T15:45:23.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1517537361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:23.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1517537361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:23.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1517537361' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:23.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:23.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:23.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:23.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:23.899 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:23.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:23.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:23.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:24.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:24.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:24.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:24.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:24.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:24.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:24.316 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:24.357 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:24.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:24.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:24.482 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:24.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:24.504 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:45:24.584 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:24.620 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:24.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:24.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:24.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 6 16 17723 17707 183.922 151.625 0.00541299 0.00543065 2024-08-29T15:45:24.724 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:24.756 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:24.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:24 smithi171 ceph-mon[90031]: pgmap v1713: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 477 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2024-08-29T15:45:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2021451979' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:45:24.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:24 smithi027 ceph-mon[98970]: pgmap v1713: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 477 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2024-08-29T15:45:24.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2021451979' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:45:24.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:24 smithi027 ceph-mon[93921]: pgmap v1713: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 477 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s 2024-08-29T15:45:24.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2021451979' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:45:24.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:24.889 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:24.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:24.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:24.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:25.025 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:25.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:25.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:25.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:25.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:25.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:25.303 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:25.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:25.429 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:25.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:25.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:25.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 7 16 20140 20124 179.237 151.062 0.00819126 0.00557303 2024-08-29T15:45:25.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:25.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:25.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:25.732 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:25 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:25.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2021451979' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:45:25.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:25 smithi171 ceph-mon[90031]: osdmap e484: 8 total, 8 up, 8 in 2024-08-29T15:45:25.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:25 smithi171 ceph-mon[90031]: 1.1d scrub starts 2024-08-29T15:45:25.733 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:25 smithi171 ceph-mon[90031]: 1.1d scrub ok 2024-08-29T15:45:25.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2021451979' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:45:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[98970]: osdmap e484: 8 total, 8 up, 8 in 2024-08-29T15:45:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[98970]: 1.1d scrub starts 2024-08-29T15:45:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[98970]: 1.1d scrub ok 2024-08-29T15:45:25.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:25.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2021451979' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:45:25.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[93921]: osdmap e484: 8 total, 8 up, 8 in 2024-08-29T15:45:25.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[93921]: 1.1d scrub starts 2024-08-29T15:45:25.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:25 smithi027 ceph-mon[93921]: 1.1d scrub ok 2024-08-29T15:45:25.843 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:25.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:25.945 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:25.975 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:25.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:25.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:26.077 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:26.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:26.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:26.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:26.219 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:45:26.220 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:45:26.249 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:26.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:26.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:26.387 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:26.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:26.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:26.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:26.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:26.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:26.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 8 16 22424 22408 174.684 142.75 0.00836229 0.00571778 2024-08-29T15:45:26.700 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:26.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:26.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[98970]: pgmap v1715: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 809 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:45:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[98970]: 1.7 scrub starts 2024-08-29T15:45:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[98970]: 1.7 scrub ok 2024-08-29T15:45:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[98970]: 1.e scrub starts 2024-08-29T15:45:26.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[98970]: 1.e scrub ok 2024-08-29T15:45:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[93921]: pgmap v1715: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 809 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:45:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[93921]: 1.7 scrub starts 2024-08-29T15:45:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[93921]: 1.7 scrub ok 2024-08-29T15:45:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[93921]: 1.e scrub starts 2024-08-29T15:45:26.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:26 smithi027 ceph-mon[93921]: 1.e scrub ok 2024-08-29T15:45:26.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:26.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:26.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:26.958 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:26.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:26 smithi171 ceph-mon[90031]: pgmap v1715: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 809 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-29T15:45:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:26 smithi171 ceph-mon[90031]: 1.7 scrub starts 2024-08-29T15:45:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:26 smithi171 ceph-mon[90031]: 1.7 scrub ok 2024-08-29T15:45:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:26 smithi171 ceph-mon[90031]: 1.e scrub starts 2024-08-29T15:45:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:26 smithi171 ceph-mon[90031]: 1.e scrub ok 2024-08-29T15:45:27.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:27.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:27.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:27.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:27.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:27.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:27.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:27.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:27.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:27.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:27.455 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:27.492 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:27.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:27.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:27.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:27.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:27.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:27.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 9 15 23780 23765 164.716 84.8125 0.00416886 0.00606619 2024-08-29T15:45:27.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:27.750 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:27.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:27 smithi027 ceph-mon[93921]: 1.1a scrub starts 2024-08-29T15:45:27.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:27 smithi027 ceph-mon[93921]: 1.1a scrub ok 2024-08-29T15:45:27.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:27 smithi027 ceph-mon[98970]: 1.1a scrub starts 2024-08-29T15:45:27.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:27 smithi027 ceph-mon[98970]: 1.1a scrub ok 2024-08-29T15:45:27.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:27.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:27.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:27.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:27.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.001 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:28.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:27 smithi171 ceph-mon[90031]: 1.1a scrub starts 2024-08-29T15:45:28.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:27 smithi171 ceph-mon[90031]: 1.1a scrub ok 2024-08-29T15:45:28.101 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:28.141 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:28.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:28.265 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:28.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:28.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:28.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:28.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:28.500 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:28.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:28.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:28.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 10 16 25271 25255 157.568 93.125 0.00547539 0.0063409 2024-08-29T15:45:28.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:28.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:28.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:28.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.818 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:28.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[93921]: 1.15 scrub starts 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[93921]: 1.15 scrub ok 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[93921]: pgmap v1716: 119 pgs: 119 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[93921]: 1.18 scrub starts 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[93921]: 1.18 scrub ok 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[98970]: 1.15 scrub starts 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[98970]: 1.15 scrub ok 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[98970]: pgmap v1716: 119 pgs: 119 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[98970]: 1.18 scrub starts 2024-08-29T15:45:28.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:28 smithi027 ceph-mon[98970]: 1.18 scrub ok 2024-08-29T15:45:28.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:28.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:28.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:28.993 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:28 smithi171 ceph-mon[90031]: 1.15 scrub starts 2024-08-29T15:45:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:28 smithi171 ceph-mon[90031]: 1.15 scrub ok 2024-08-29T15:45:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:28 smithi171 ceph-mon[90031]: pgmap v1716: 119 pgs: 119 active+clean; 1.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-29T15:45:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:28 smithi171 ceph-mon[90031]: 1.18 scrub starts 2024-08-29T15:45:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:28 smithi171 ceph-mon[90031]: 1.18 scrub ok 2024-08-29T15:45:29.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:29.091 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:29.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:29.192 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:29.232 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:29.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:29.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:29.270 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:45:29] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.51.0" 2024-08-29T15:45:29.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:29.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:29.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:29.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:29.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:29.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:29.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:29.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:29.646 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:29.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:29.667 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:45:29.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 11 16 27531 27515 156.085 141.25 0.00608476 0.00640153 2024-08-29T15:45:29.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:29.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:29.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:29.795 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[98970]: 1.17 scrub starts 2024-08-29T15:45:29.795 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[98970]: 1.17 scrub ok 2024-08-29T15:45:29.795 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:29.795 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:29.795 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[93921]: 1.17 scrub starts 2024-08-29T15:45:29.796 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[93921]: 1.17 scrub ok 2024-08-29T15:45:29.796 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:29.796 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:29.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:29.927 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:29.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:29.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:29 smithi171 ceph-mon[90031]: 1.17 scrub starts 2024-08-29T15:45:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:29 smithi171 ceph-mon[90031]: 1.17 scrub ok 2024-08-29T15:45:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:30.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:30.030 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:30.075 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:30.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:30.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:30.213 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:30.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:30.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:30.336 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:30.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:30.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:30.478 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:30.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:30.580 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:30.614 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:30.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:30.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:30.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 12 16 29654 29638 154.137 132.688 0.0040582 0.00648208 2024-08-29T15:45:30.716 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:30.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:30.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:30.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:30.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:45:30.784 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:30 smithi027 ceph-mon[93921]: pgmap v1717: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-08-29T15:45:30.784 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:30 smithi027 ceph-mon[98970]: pgmap v1717: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-08-29T15:45:30.851 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:30.883 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:30.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:30.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:30.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:30 smithi171 ceph-mon[90031]: pgmap v1717: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-08-29T15:45:31.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:31.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:31.151 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:31.187 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:31.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:31.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:31.250 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:31.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:31.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:31.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:31.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:31.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:31.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:31.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:31.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:31.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:45:31.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:31.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:31.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:31.668 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 13 16 31510 31494 151.207 116 0.0107585 0.00660832 2024-08-29T15:45:31.719 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:31.750 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:31.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:31.852 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:31.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:31.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:31.986 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:32.008 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:32.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:32.109 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:32.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.144 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:32.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:32.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:32.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:32.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:32.385 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:32.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:32.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:32.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:32.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:32.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 14 16 33280 33264 148.309 110.625 0.00569306 0.00673548 2024-08-29T15:45:32.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:32.741 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:32 smithi171 ceph-mon[90031]: pgmap v1718: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-08-29T15:45:32.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:32.814 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:32 smithi027 ceph-mon[93921]: pgmap v1718: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-08-29T15:45:32.814 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:32 smithi027 ceph-mon[98970]: pgmap v1718: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-08-29T15:45:32.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:32.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:32.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:32.979 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:32.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:32.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:33.041 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:45:33.042 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:45:33.072 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:45:33.074 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:45:33.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:33.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:33.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:33.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:33.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:33.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:33.255 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:33.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:33.356 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:33.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:33.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:33.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:33.528 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:33.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:33.631 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:33.667 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:33.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:33.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:33.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 15 16 35409 35393 147.293 133.062 0.00450488 0.00678417 2024-08-29T15:45:33.743 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:45:33.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:33.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:33.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:33.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:33.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:33.949 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:33.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:34.050 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:34.096 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:34.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:34.098 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:34.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:34.246 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:34.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:34.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:34.383 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:34.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:34.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:34.485 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:34.521 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:34.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:34.623 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:34.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:34.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:34.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 16 16 37481 37465 146.181 129.5 0.00436047 0.00683487 2024-08-29T15:45:34.747 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:34 smithi027 ceph-mon[93921]: from='client.56725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:45:34.748 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:34 smithi027 ceph-mon[93921]: pgmap v1719: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-29T15:45:34.748 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:34 smithi027 ceph-mon[98970]: from='client.56725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:45:34.748 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:34 smithi027 ceph-mon[98970]: pgmap v1719: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-29T15:45:34.758 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:34.788 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1718,"stamp":"2024-08-29T15:45:31.094740+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1670453221,"num_objects":25444,"num_object_clones":0,"num_object_copies":76332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25444,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":26324,"num_write_kb":1639249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":193,"num_bytes_recovered":12918369,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":31068,"ondisk_log_size":31068,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11970008,"kb_used_data":5984852,"kb_used_omap":43,"kb_used_meta":5985044,"kb_avail":737991208,"statfs":{"total":767960285184,"available":755702996992,"internally_reserved":0,"allocated":6128488448,"data_stored":6104246014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":6128685548},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1520382074,"num_objects":23197,"num_object_clones":0,"num_object_copies":69591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23197,"num_whiteouts":0,"num_read":-34,"num_read_kb":-75,"num_write":23176,"num_write_kb":1484324,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-292354,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004141"},"pg_stats":[{"pgid":"1.11","version":"90'149","reported_seq":894,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.150733+0000","last_change":"2024-08-29T15:44:59.240701+0000","last_active":"2024-08-29T15:45:15.150733+0000","last_peered":"2024-08-29T15:45:15.150733+0000","last_clean":"2024-08-29T15:45:15.150733+0000","last_became_active":"2024-08-29T15:44:59.240326+0000","last_became_peered":"2024-08-29T15:44:59.240326+0000","last_unstale":"2024-08-29T15:45:15.150733+0000","last_undegraded":"2024-08-29T15:45:15.150733+0000","last_fullsized":"2024-08-29T15:45:15.150733+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:36.699540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:36.699540+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:59:35.452250+0000","scrub_duration":0.0014941869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":882,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530557+0000","last_change":"2024-08-29T15:41:16.221956+0000","last_active":"2024-08-29T15:45:24.530557+0000","last_peered":"2024-08-29T15:45:24.530557+0000","last_clean":"2024-08-29T15:45:24.530557+0000","last_became_active":"2024-08-29T15:41:16.221533+0000","last_became_peered":"2024-08-29T15:41:16.221533+0000","last_unstale":"2024-08-29T15:45:24.530557+0000","last_undegraded":"2024-08-29T15:45:24.530557+0000","last_fullsized":"2024-08-29T15:45:24.530557+0000","mapping_epoch":439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:26:34.947614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":278,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.524346+0000","last_change":"2024-08-29T15:28:35.521036+0000","last_active":"2024-08-29T15:45:24.524346+0000","last_peered":"2024-08-29T15:45:24.524346+0000","last_clean":"2024-08-29T15:45:24.524346+0000","last_became_active":"2024-08-29T15:19:46.803496+0000","last_became_peered":"2024-08-29T15:19:46.803496+0000","last_unstale":"2024-08-29T15:45:24.524346+0000","last_undegraded":"2024-08-29T15:45:24.524346+0000","last_fullsized":"2024-08-29T15:45:24.524346+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:31:17.136173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":977,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.150670+0000","last_change":"2024-08-29T15:44:59.245746+0000","last_active":"2024-08-29T15:45:15.150670+0000","last_peered":"2024-08-29T15:45:15.150670+0000","last_clean":"2024-08-29T15:45:15.150670+0000","last_became_active":"2024-08-29T15:44:59.245400+0000","last_became_peered":"2024-08-29T15:44:59.245400+0000","last_unstale":"2024-08-29T15:45:15.150670+0000","last_undegraded":"2024-08-29T15:45:15.150670+0000","last_fullsized":"2024-08-29T15:45:15.150670+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.968809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.968809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:37.657803+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151397+0000","last_change":"2024-08-29T15:39:53.258205+0000","last_active":"2024-08-29T15:45:15.151397+0000","last_peered":"2024-08-29T15:45:15.151397+0000","last_clean":"2024-08-29T15:45:15.151397+0000","last_became_active":"2024-08-29T15:39:53.257784+0000","last_became_peered":"2024-08-29T15:39:53.257784+0000","last_unstale":"2024-08-29T15:45:15.151397+0000","last_undegraded":"2024-08-29T15:45:15.151397+0000","last_fullsized":"2024-08-29T15:45:15.151397+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:30:42.270982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":738,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530577+0000","last_change":"2024-08-29T15:43:44.388183+0000","last_active":"2024-08-29T15:45:24.530577+0000","last_peered":"2024-08-29T15:45:24.530577+0000","last_clean":"2024-08-29T15:45:24.530577+0000","last_became_active":"2024-08-29T15:43:44.387812+0000","last_became_peered":"2024-08-29T15:43:44.387812+0000","last_unstale":"2024-08-29T15:45:24.530577+0000","last_undegraded":"2024-08-29T15:45:24.530577+0000","last_fullsized":"2024-08-29T15:45:24.530577+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:06:27.607690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":991,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529501+0000","last_change":"2024-08-29T15:44:59.244961+0000","last_active":"2024-08-29T15:45:24.529501+0000","last_peered":"2024-08-29T15:45:24.529501+0000","last_clean":"2024-08-29T15:45:24.529501+0000","last_became_active":"2024-08-29T15:44:59.244557+0000","last_became_peered":"2024-08-29T15:44:59.244557+0000","last_unstale":"2024-08-29T15:45:24.529501+0000","last_undegraded":"2024-08-29T15:45:24.529501+0000","last_fullsized":"2024-08-29T15:45:24.529501+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.615446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.615446+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:04:28.864577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":801,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.531232+0000","last_change":"2024-08-29T15:43:44.377372+0000","last_active":"2024-08-29T15:45:24.531232+0000","last_peered":"2024-08-29T15:45:24.531232+0000","last_clean":"2024-08-29T15:45:24.531232+0000","last_became_active":"2024-08-29T15:43:44.377041+0000","last_became_peered":"2024-08-29T15:43:44.377041+0000","last_unstale":"2024-08-29T15:45:24.531232+0000","last_undegraded":"2024-08-29T15:45:24.531232+0000","last_fullsized":"2024-08-29T15:45:24.531232+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:05:58.536193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":271,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.525729+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:45:24.525729+0000","last_peered":"2024-08-29T15:45:24.525729+0000","last_clean":"2024-08-29T15:45:24.525729+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:45:24.525729+0000","last_undegraded":"2024-08-29T15:45:24.525729+0000","last_fullsized":"2024-08-29T15:45:24.525729+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":997,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.150871+0000","last_change":"2024-08-29T15:44:59.241957+0000","last_active":"2024-08-29T15:45:15.150871+0000","last_peered":"2024-08-29T15:45:15.150871+0000","last_clean":"2024-08-29T15:45:15.150871+0000","last_became_active":"2024-08-29T15:44:59.241187+0000","last_became_peered":"2024-08-29T15:44:59.241187+0000","last_unstale":"2024-08-29T15:45:15.150871+0000","last_undegraded":"2024-08-29T15:45:15.150871+0000","last_fullsized":"2024-08-29T15:45:15.150871+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.831081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.831081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:04:04.531194+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":582,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530581+0000","last_change":"2024-08-29T15:41:44.469235+0000","last_active":"2024-08-29T15:45:24.530581+0000","last_peered":"2024-08-29T15:45:24.530581+0000","last_clean":"2024-08-29T15:45:24.530581+0000","last_became_active":"2024-08-29T15:41:44.468857+0000","last_became_peered":"2024-08-29T15:41:44.468857+0000","last_unstale":"2024-08-29T15:45:24.530581+0000","last_undegraded":"2024-08-29T15:45:24.530581+0000","last_fullsized":"2024-08-29T15:45:24.530581+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:22:54.347105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":223,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530740+0000","last_change":"2024-08-29T15:43:44.389429+0000","last_active":"2024-08-29T15:45:24.530740+0000","last_peered":"2024-08-29T15:45:24.530740+0000","last_clean":"2024-08-29T15:45:24.530740+0000","last_became_active":"2024-08-29T15:43:44.375048+0000","last_became_peered":"2024-08-29T15:43:44.375048+0000","last_unstale":"2024-08-29T15:45:24.530740+0000","last_undegraded":"2024-08-29T15:45:24.530740+0000","last_fullsized":"2024-08-29T15:45:24.530740+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:51:36.427552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"484'1428","reported_seq":1441,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:27.489832+0000","last_change":"2024-08-29T15:45:06.877844+0000","last_active":"2024-08-29T15:45:27.489832+0000","last_peered":"2024-08-29T15:45:27.489832+0000","last_clean":"2024-08-29T15:45:27.489832+0000","last_became_active":"2024-08-29T15:45:06.877523+0000","last_became_peered":"2024-08-29T15:45:06.877523+0000","last_unstale":"2024-08-29T15:45:27.489832+0000","last_undegraded":"2024-08-29T15:45:27.489832+0000","last_fullsized":"2024-08-29T15:45:27.489832+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1428,"log_dups_size":0,"ondisk_log_size":1428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:37:30.523325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93585408,"num_objects":1428,"num_object_clones":0,"num_object_copies":4284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1428,"num_write_kb":91392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":915,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857408+0000","last_change":"2024-08-29T15:44:59.246665+0000","last_active":"2024-08-29T15:45:15.857408+0000","last_peered":"2024-08-29T15:45:15.857408+0000","last_clean":"2024-08-29T15:45:15.857408+0000","last_became_active":"2024-08-29T15:44:59.245987+0000","last_became_peered":"2024-08-29T15:44:59.245987+0000","last_unstale":"2024-08-29T15:45:15.857408+0000","last_undegraded":"2024-08-29T15:45:15.857408+0000","last_fullsized":"2024-08-29T15:45:15.857408+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.799642+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.799642+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:44:33.647503+0000","scrub_duration":0.0012023820000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":314,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.858319+0000","last_change":"2024-08-29T15:43:44.376246+0000","last_active":"2024-08-29T15:45:15.858319+0000","last_peered":"2024-08-29T15:45:15.858319+0000","last_clean":"2024-08-29T15:45:15.858319+0000","last_became_active":"2024-08-29T15:43:44.371925+0000","last_became_peered":"2024-08-29T15:43:44.371925+0000","last_unstale":"2024-08-29T15:45:15.858319+0000","last_undegraded":"2024-08-29T15:45:15.858319+0000","last_fullsized":"2024-08-29T15:45:15.858319+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:08:57.521364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.150827+0000","last_change":"2024-08-29T15:39:53.258473+0000","last_active":"2024-08-29T15:45:15.150827+0000","last_peered":"2024-08-29T15:45:15.150827+0000","last_clean":"2024-08-29T15:45:15.150827+0000","last_became_active":"2024-08-29T15:39:53.258055+0000","last_became_peered":"2024-08-29T15:39:53.258055+0000","last_unstale":"2024-08-29T15:45:15.150827+0000","last_undegraded":"2024-08-29T15:45:15.150827+0000","last_fullsized":"2024-08-29T15:45:15.150827+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:38:54.441868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"484'1404","reported_seq":1423,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:26.703040+0000","last_change":"2024-08-29T15:45:07.149089+0000","last_active":"2024-08-29T15:45:26.703040+0000","last_peered":"2024-08-29T15:45:26.703040+0000","last_clean":"2024-08-29T15:45:26.703040+0000","last_became_active":"2024-08-29T15:45:07.148534+0000","last_became_peered":"2024-08-29T15:45:07.148534+0000","last_unstale":"2024-08-29T15:45:26.703040+0000","last_undegraded":"2024-08-29T15:45:26.703040+0000","last_fullsized":"2024-08-29T15:45:26.703040+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1404,"log_dups_size":0,"ondisk_log_size":1404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:23:58.042421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92012544,"num_objects":1404,"num_object_clones":0,"num_object_copies":4212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1404,"num_write_kb":89856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'149","reported_seq":1044,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530418+0000","last_change":"2024-08-29T15:44:59.240657+0000","last_active":"2024-08-29T15:45:24.530418+0000","last_peered":"2024-08-29T15:45:24.530418+0000","last_clean":"2024-08-29T15:45:24.530418+0000","last_became_active":"2024-08-29T15:44:59.240261+0000","last_became_peered":"2024-08-29T15:44:59.240261+0000","last_unstale":"2024-08-29T15:45:24.530418+0000","last_undegraded":"2024-08-29T15:45:24.530418+0000","last_fullsized":"2024-08-29T15:45:24.530418+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.834998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.834998+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:19:43.756259+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":275,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.858377+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:45:15.858377+0000","last_peered":"2024-08-29T15:45:15.858377+0000","last_clean":"2024-08-29T15:45:15.858377+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:45:15.858377+0000","last_undegraded":"2024-08-29T15:45:15.858377+0000","last_fullsized":"2024-08-29T15:45:15.858377+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":307,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857699+0000","last_change":"2024-08-29T15:41:44.436101+0000","last_active":"2024-08-29T15:45:15.857699+0000","last_peered":"2024-08-29T15:45:15.857699+0000","last_clean":"2024-08-29T15:45:15.857699+0000","last_became_active":"2024-08-29T15:41:44.435769+0000","last_became_peered":"2024-08-29T15:41:44.435769+0000","last_unstale":"2024-08-29T15:45:15.857699+0000","last_undegraded":"2024-08-29T15:45:15.857699+0000","last_fullsized":"2024-08-29T15:45:15.857699+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:20:19.269808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"484'1742","reported_seq":1755,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:29.772407+0000","last_change":"2024-08-29T15:45:06.877033+0000","last_active":"2024-08-29T15:45:29.772407+0000","last_peered":"2024-08-29T15:45:29.772407+0000","last_clean":"2024-08-29T15:45:29.772407+0000","last_became_active":"2024-08-29T15:45:06.876771+0000","last_became_peered":"2024-08-29T15:45:06.876771+0000","last_unstale":"2024-08-29T15:45:29.772407+0000","last_undegraded":"2024-08-29T15:45:29.772407+0000","last_fullsized":"2024-08-29T15:45:29.772407+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1742,"log_dups_size":0,"ondisk_log_size":1742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:39:35.183951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114163712,"num_objects":1742,"num_object_clones":0,"num_object_copies":5226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1742,"num_write_kb":111488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":922,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151564+0000","last_change":"2024-08-29T15:44:59.247796+0000","last_active":"2024-08-29T15:45:15.151564+0000","last_peered":"2024-08-29T15:45:15.151564+0000","last_clean":"2024-08-29T15:45:15.151564+0000","last_became_active":"2024-08-29T15:44:59.246494+0000","last_became_peered":"2024-08-29T15:44:59.246494+0000","last_unstale":"2024-08-29T15:45:15.151564+0000","last_undegraded":"2024-08-29T15:45:15.151564+0000","last_fullsized":"2024-08-29T15:45:15.151564+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:21.844245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:21.844245+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:26.154513+0000","scrub_duration":0.0016201309999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":76,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151291+0000","last_change":"2024-08-29T15:39:53.262167+0000","last_active":"2024-08-29T15:45:15.151291+0000","last_peered":"2024-08-29T15:45:15.151291+0000","last_clean":"2024-08-29T15:45:15.151291+0000","last_became_active":"2024-08-29T15:39:53.258406+0000","last_became_peered":"2024-08-29T15:39:53.258406+0000","last_unstale":"2024-08-29T15:45:15.151291+0000","last_undegraded":"2024-08-29T15:45:15.151291+0000","last_fullsized":"2024-08-29T15:45:15.151291+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:05:19.731559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":462,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.525708+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:45:24.525708+0000","last_peered":"2024-08-29T15:45:24.525708+0000","last_clean":"2024-08-29T15:45:24.525708+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:45:24.525708+0000","last_undegraded":"2024-08-29T15:45:24.525708+0000","last_fullsized":"2024-08-29T15:45:24.525708+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"484'1885","reported_seq":1904,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:30.513541+0000","last_change":"2024-08-29T15:45:07.147672+0000","last_active":"2024-08-29T15:45:30.513541+0000","last_peered":"2024-08-29T15:45:30.513541+0000","last_clean":"2024-08-29T15:45:30.513541+0000","last_became_active":"2024-08-29T15:45:07.147363+0000","last_became_peered":"2024-08-29T15:45:07.147363+0000","last_unstale":"2024-08-29T15:45:30.513541+0000","last_undegraded":"2024-08-29T15:45:30.513541+0000","last_fullsized":"2024-08-29T15:45:30.513541+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1885,"log_dups_size":0,"ondisk_log_size":1885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:03.126615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123535360,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1885,"num_write_kb":120640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":784,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:25.870357+0000","last_change":"2024-08-29T15:45:25.871113+0000","last_active":"2024-08-29T15:45:25.870357+0000","last_peered":"2024-08-29T15:45:25.870357+0000","last_clean":"2024-08-29T15:45:25.870357+0000","last_became_active":"2024-08-29T15:44:59.246190+0000","last_became_peered":"2024-08-29T15:44:59.246190+0000","last_unstale":"2024-08-29T15:45:25.870357+0000","last_undegraded":"2024-08-29T15:45:25.870357+0000","last_fullsized":"2024-08-29T15:45:25.870357+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.870329+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.870329+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:42.080608+0000","scrub_duration":0.0042968349999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":379,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151227+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:45:15.151227+0000","last_peered":"2024-08-29T15:45:15.151227+0000","last_clean":"2024-08-29T15:45:15.151227+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:45:15.151227+0000","last_undegraded":"2024-08-29T15:45:15.151227+0000","last_fullsized":"2024-08-29T15:45:15.151227+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":841,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.533459+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:45:24.533459+0000","last_peered":"2024-08-29T15:45:24.533459+0000","last_clean":"2024-08-29T15:45:24.533459+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:45:24.533459+0000","last_undegraded":"2024-08-29T15:45:24.533459+0000","last_fullsized":"2024-08-29T15:45:24.533459+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"484'1337","reported_seq":1356,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:26.483034+0000","last_change":"2024-08-29T15:45:06.877939+0000","last_active":"2024-08-29T15:45:26.483034+0000","last_peered":"2024-08-29T15:45:26.483034+0000","last_clean":"2024-08-29T15:45:26.483034+0000","last_became_active":"2024-08-29T15:45:06.877593+0000","last_became_peered":"2024-08-29T15:45:06.877593+0000","last_unstale":"2024-08-29T15:45:26.483034+0000","last_undegraded":"2024-08-29T15:45:26.483034+0000","last_fullsized":"2024-08-29T15:45:26.483034+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1337,"log_dups_size":0,"ondisk_log_size":1337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:32.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87621632,"num_objects":1337,"num_object_clones":0,"num_object_copies":4011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1337,"num_write_kb":85568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":983,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529868+0000","last_change":"2024-08-29T15:44:30.836092+0000","last_active":"2024-08-29T15:45:24.529868+0000","last_peered":"2024-08-29T15:45:24.529868+0000","last_clean":"2024-08-29T15:45:24.529868+0000","last_became_active":"2024-08-29T15:44:22.624657+0000","last_became_peered":"2024-08-29T15:44:22.624657+0000","last_unstale":"2024-08-29T15:45:24.529868+0000","last_undegraded":"2024-08-29T15:45:24.529868+0000","last_fullsized":"2024-08-29T15:45:24.529868+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:30.836022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:30.836022+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:27:12.255056+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":869,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529996+0000","last_change":"2024-08-29T15:43:44.395045+0000","last_active":"2024-08-29T15:45:24.529996+0000","last_peered":"2024-08-29T15:45:24.529996+0000","last_clean":"2024-08-29T15:45:24.529996+0000","last_became_active":"2024-08-29T15:43:44.382969+0000","last_became_peered":"2024-08-29T15:43:44.382969+0000","last_unstale":"2024-08-29T15:45:24.529996+0000","last_undegraded":"2024-08-29T15:45:24.529996+0000","last_fullsized":"2024-08-29T15:45:24.529996+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:41.129154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":271,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857877+0000","last_change":"2024-08-29T15:19:46.807790+0000","last_active":"2024-08-29T15:45:15.857877+0000","last_peered":"2024-08-29T15:45:15.857877+0000","last_clean":"2024-08-29T15:45:15.857877+0000","last_became_active":"2024-08-29T15:19:46.807479+0000","last_became_peered":"2024-08-29T15:19:46.807479+0000","last_unstale":"2024-08-29T15:45:15.857877+0000","last_undegraded":"2024-08-29T15:45:15.857877+0000","last_fullsized":"2024-08-29T15:45:15.857877+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:45:44.170751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"484'1489","reported_seq":1502,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:27.493905+0000","last_change":"2024-08-29T15:45:07.146252+0000","last_active":"2024-08-29T15:45:27.493905+0000","last_peered":"2024-08-29T15:45:27.493905+0000","last_clean":"2024-08-29T15:45:27.493905+0000","last_became_active":"2024-08-29T15:45:07.145985+0000","last_became_peered":"2024-08-29T15:45:07.145985+0000","last_unstale":"2024-08-29T15:45:27.493905+0000","last_undegraded":"2024-08-29T15:45:27.493905+0000","last_fullsized":"2024-08-29T15:45:27.493905+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1489,"log_dups_size":0,"ondisk_log_size":1489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:22:20.603079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97583104,"num_objects":1489,"num_object_clones":0,"num_object_copies":4467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1489,"num_write_kb":95296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":958,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530026+0000","last_change":"2024-08-29T15:44:22.851383+0000","last_active":"2024-08-29T15:45:24.530026+0000","last_peered":"2024-08-29T15:45:24.530026+0000","last_clean":"2024-08-29T15:45:24.530026+0000","last_became_active":"2024-08-29T15:44:22.614765+0000","last_became_peered":"2024-08-29T15:44:22.614765+0000","last_unstale":"2024-08-29T15:45:24.530026+0000","last_undegraded":"2024-08-29T15:45:24.530026+0000","last_fullsized":"2024-08-29T15:45:24.530026+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.851293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.851293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:57:44.944567+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":267,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.524116+0000","last_change":"2024-08-29T15:41:44.443161+0000","last_active":"2024-08-29T15:45:24.524116+0000","last_peered":"2024-08-29T15:45:24.524116+0000","last_clean":"2024-08-29T15:45:24.524116+0000","last_became_active":"2024-08-29T15:41:44.442573+0000","last_became_peered":"2024-08-29T15:41:44.442573+0000","last_unstale":"2024-08-29T15:45:24.524116+0000","last_undegraded":"2024-08-29T15:45:24.524116+0000","last_fullsized":"2024-08-29T15:45:24.524116+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:31:08.897327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":271,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.858224+0000","last_change":"2024-08-29T15:19:46.804649+0000","last_active":"2024-08-29T15:45:15.858224+0000","last_peered":"2024-08-29T15:45:15.858224+0000","last_clean":"2024-08-29T15:45:15.858224+0000","last_became_active":"2024-08-29T15:19:46.804348+0000","last_became_peered":"2024-08-29T15:19:46.804348+0000","last_unstale":"2024-08-29T15:45:15.858224+0000","last_undegraded":"2024-08-29T15:45:15.858224+0000","last_fullsized":"2024-08-29T15:45:15.858224+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:57.715523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"484'1392","reported_seq":1411,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:26.493376+0000","last_change":"2024-08-29T15:45:06.875151+0000","last_active":"2024-08-29T15:45:26.493376+0000","last_peered":"2024-08-29T15:45:26.493376+0000","last_clean":"2024-08-29T15:45:26.493376+0000","last_became_active":"2024-08-29T15:45:06.874862+0000","last_became_peered":"2024-08-29T15:45:06.874862+0000","last_unstale":"2024-08-29T15:45:26.493376+0000","last_undegraded":"2024-08-29T15:45:26.493376+0000","last_fullsized":"2024-08-29T15:45:26.493376+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1392,"log_dups_size":0,"ondisk_log_size":1392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:54:54.225355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91226112,"num_objects":1392,"num_object_clones":0,"num_object_copies":4176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1392,"num_write_kb":89088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":1005,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.525801+0000","last_change":"2024-08-29T15:44:59.243847+0000","last_active":"2024-08-29T15:45:24.525801+0000","last_peered":"2024-08-29T15:45:24.525801+0000","last_clean":"2024-08-29T15:45:24.525801+0000","last_became_active":"2024-08-29T15:44:59.243399+0000","last_became_peered":"2024-08-29T15:44:59.243399+0000","last_unstale":"2024-08-29T15:45:24.525801+0000","last_undegraded":"2024-08-29T15:45:24.525801+0000","last_fullsized":"2024-08-29T15:45:24.525801+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.906140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.906140+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:24:25.970792+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":271,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857797+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:45:15.857797+0000","last_peered":"2024-08-29T15:45:15.857797+0000","last_clean":"2024-08-29T15:45:15.857797+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:45:15.857797+0000","last_undegraded":"2024-08-29T15:45:15.857797+0000","last_fullsized":"2024-08-29T15:45:15.857797+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":504,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.858398+0000","last_change":"2024-08-29T15:41:00.747697+0000","last_active":"2024-08-29T15:45:15.858398+0000","last_peered":"2024-08-29T15:45:15.858398+0000","last_clean":"2024-08-29T15:45:15.858398+0000","last_became_active":"2024-08-29T15:41:00.747350+0000","last_became_peered":"2024-08-29T15:41:00.747350+0000","last_unstale":"2024-08-29T15:45:15.858398+0000","last_undegraded":"2024-08-29T15:45:15.858398+0000","last_fullsized":"2024-08-29T15:45:15.858398+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:13:16.221880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"484'1775","reported_seq":1788,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:29.766907+0000","last_change":"2024-08-29T15:45:06.941691+0000","last_active":"2024-08-29T15:45:29.766907+0000","last_peered":"2024-08-29T15:45:29.766907+0000","last_clean":"2024-08-29T15:45:29.766907+0000","last_became_active":"2024-08-29T15:45:06.941400+0000","last_became_peered":"2024-08-29T15:45:06.941400+0000","last_unstale":"2024-08-29T15:45:29.766907+0000","last_undegraded":"2024-08-29T15:45:29.766907+0000","last_fullsized":"2024-08-29T15:45:29.766907+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1775,"log_dups_size":0,"ondisk_log_size":1775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:48:51.640282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1775,"num_write_kb":113600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":863,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.150998+0000","last_change":"2024-08-29T15:44:53.533605+0000","last_active":"2024-08-29T15:45:15.150998+0000","last_peered":"2024-08-29T15:45:15.150998+0000","last_clean":"2024-08-29T15:45:15.150998+0000","last_became_active":"2024-08-29T15:44:22.611492+0000","last_became_peered":"2024-08-29T15:44:22.611492+0000","last_unstale":"2024-08-29T15:45:15.150998+0000","last_undegraded":"2024-08-29T15:45:15.150998+0000","last_fullsized":"2024-08-29T15:45:15.150998+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:53.533488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:53.533488+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:10:06.648003+0000","scrub_duration":0.0059276559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":367,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.525748+0000","last_change":"2024-08-29T15:41:44.435967+0000","last_active":"2024-08-29T15:45:24.525748+0000","last_peered":"2024-08-29T15:45:24.525748+0000","last_clean":"2024-08-29T15:45:24.525748+0000","last_became_active":"2024-08-29T15:41:44.435570+0000","last_became_peered":"2024-08-29T15:41:44.435570+0000","last_unstale":"2024-08-29T15:45:24.525748+0000","last_undegraded":"2024-08-29T15:45:24.525748+0000","last_fullsized":"2024-08-29T15:45:24.525748+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:57:19.832668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":831,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530659+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:45:24.530659+0000","last_peered":"2024-08-29T15:45:24.530659+0000","last_clean":"2024-08-29T15:45:24.530659+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:45:24.530659+0000","last_undegraded":"2024-08-29T15:45:24.530659+0000","last_fullsized":"2024-08-29T15:45:24.530659+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"484'1411","reported_seq":1430,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:26.499415+0000","last_change":"2024-08-29T15:45:06.878140+0000","last_active":"2024-08-29T15:45:26.499415+0000","last_peered":"2024-08-29T15:45:26.499415+0000","last_clean":"2024-08-29T15:45:26.499415+0000","last_became_active":"2024-08-29T15:45:06.877878+0000","last_became_peered":"2024-08-29T15:45:06.877878+0000","last_unstale":"2024-08-29T15:45:26.499415+0000","last_undegraded":"2024-08-29T15:45:26.499415+0000","last_fullsized":"2024-08-29T15:45:26.499415+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1411,"log_dups_size":0,"ondisk_log_size":1411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:33:28.011162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92471296,"num_objects":1411,"num_object_clones":0,"num_object_copies":4233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1411,"num_write_kb":90304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":979,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.525509+0000","last_change":"2024-08-29T15:43:47.426690+0000","last_active":"2024-08-29T15:45:24.525509+0000","last_peered":"2024-08-29T15:45:24.525509+0000","last_clean":"2024-08-29T15:45:24.525509+0000","last_became_active":"2024-08-29T15:43:47.426071+0000","last_became_peered":"2024-08-29T15:43:47.426071+0000","last_unstale":"2024-08-29T15:45:24.525509+0000","last_undegraded":"2024-08-29T15:45:24.525509+0000","last_fullsized":"2024-08-29T15:45:24.525509+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.008533+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.008533+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:42:46.053321+0000","scrub_duration":111,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":964,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151177+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:45:15.151177+0000","last_peered":"2024-08-29T15:45:15.151177+0000","last_clean":"2024-08-29T15:45:15.151177+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:45:15.151177+0000","last_undegraded":"2024-08-29T15:45:15.151177+0000","last_fullsized":"2024-08-29T15:45:15.151177+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":245,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530480+0000","last_change":"2024-08-29T15:37:26.313495+0000","last_active":"2024-08-29T15:45:24.530480+0000","last_peered":"2024-08-29T15:45:24.530480+0000","last_clean":"2024-08-29T15:45:24.530480+0000","last_became_active":"2024-08-29T15:37:26.313188+0000","last_became_peered":"2024-08-29T15:37:26.313188+0000","last_unstale":"2024-08-29T15:45:24.530480+0000","last_undegraded":"2024-08-29T15:45:24.530480+0000","last_fullsized":"2024-08-29T15:45:24.530480+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:39.580490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":238,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.858148+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:45:15.858148+0000","last_peered":"2024-08-29T15:45:15.858148+0000","last_clean":"2024-08-29T15:45:15.858148+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:45:15.858148+0000","last_undegraded":"2024-08-29T15:45:15.858148+0000","last_fullsized":"2024-08-29T15:45:15.858148+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"484'1784","reported_seq":1803,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:30.498914+0000","last_change":"2024-08-29T15:45:07.146463+0000","last_active":"2024-08-29T15:45:30.498914+0000","last_peered":"2024-08-29T15:45:30.498914+0000","last_clean":"2024-08-29T15:45:30.498914+0000","last_became_active":"2024-08-29T15:45:07.146148+0000","last_became_peered":"2024-08-29T15:45:07.146148+0000","last_unstale":"2024-08-29T15:45:30.498914+0000","last_undegraded":"2024-08-29T15:45:30.498914+0000","last_fullsized":"2024-08-29T15:45:30.498914+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1784,"log_dups_size":0,"ondisk_log_size":1784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:04.649404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916224,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1784,"num_write_kb":114176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":983,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529324+0000","last_change":"2024-08-29T15:43:26.588054+0000","last_active":"2024-08-29T15:45:24.529324+0000","last_peered":"2024-08-29T15:45:24.529324+0000","last_clean":"2024-08-29T15:45:24.529324+0000","last_became_active":"2024-08-29T15:43:26.285267+0000","last_became_peered":"2024-08-29T15:43:26.285267+0000","last_unstale":"2024-08-29T15:45:24.529324+0000","last_undegraded":"2024-08-29T15:45:24.529324+0000","last_fullsized":"2024-08-29T15:45:24.529324+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.587986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.587986+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:08:35.477444+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1305,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.525689+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:45:24.525689+0000","last_peered":"2024-08-29T15:45:24.525689+0000","last_clean":"2024-08-29T15:45:24.525689+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:45:24.525689+0000","last_undegraded":"2024-08-29T15:45:24.525689+0000","last_fullsized":"2024-08-29T15:45:24.525689+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":864,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.524125+0000","last_change":"2024-08-29T15:41:44.432130+0000","last_active":"2024-08-29T15:45:24.524125+0000","last_peered":"2024-08-29T15:45:24.524125+0000","last_clean":"2024-08-29T15:45:24.524125+0000","last_became_active":"2024-08-29T15:41:44.431835+0000","last_became_peered":"2024-08-29T15:41:44.431835+0000","last_unstale":"2024-08-29T15:45:24.524125+0000","last_undegraded":"2024-08-29T15:45:24.524125+0000","last_fullsized":"2024-08-29T15:45:24.524125+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:07.995950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":521,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530039+0000","last_change":"2024-08-29T15:43:44.385638+0000","last_active":"2024-08-29T15:45:24.530039+0000","last_peered":"2024-08-29T15:45:24.530039+0000","last_clean":"2024-08-29T15:45:24.530039+0000","last_became_active":"2024-08-29T15:43:44.385042+0000","last_became_peered":"2024-08-29T15:43:44.385042+0000","last_unstale":"2024-08-29T15:45:24.530039+0000","last_undegraded":"2024-08-29T15:45:24.530039+0000","last_fullsized":"2024-08-29T15:45:24.530039+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:19:48.932998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"484'1719","reported_seq":1732,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:29.742671+0000","last_change":"2024-08-29T15:45:07.146250+0000","last_active":"2024-08-29T15:45:29.742671+0000","last_peered":"2024-08-29T15:45:29.742671+0000","last_clean":"2024-08-29T15:45:29.742671+0000","last_became_active":"2024-08-29T15:45:07.145937+0000","last_became_peered":"2024-08-29T15:45:07.145937+0000","last_unstale":"2024-08-29T15:45:29.742671+0000","last_undegraded":"2024-08-29T15:45:29.742671+0000","last_fullsized":"2024-08-29T15:45:29.742671+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1719,"log_dups_size":0,"ondisk_log_size":1719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:48:30.443435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112656384,"num_objects":1719,"num_object_clones":0,"num_object_copies":5157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1719,"num_write_kb":110016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":1001,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.524377+0000","last_change":"2024-08-29T15:43:27.841159+0000","last_active":"2024-08-29T15:45:24.524377+0000","last_peered":"2024-08-29T15:45:24.524377+0000","last_clean":"2024-08-29T15:45:24.524377+0000","last_became_active":"2024-08-29T15:43:26.281150+0000","last_became_peered":"2024-08-29T15:43:26.281150+0000","last_unstale":"2024-08-29T15:45:24.524377+0000","last_undegraded":"2024-08-29T15:45:24.524377+0000","last_fullsized":"2024-08-29T15:45:24.524377+0000","mapping_epoch":437,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.841106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.841106+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:28.542150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":873,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530201+0000","last_change":"2024-08-29T15:37:29.345555+0000","last_active":"2024-08-29T15:45:24.530201+0000","last_peered":"2024-08-29T15:45:24.530201+0000","last_clean":"2024-08-29T15:45:24.530201+0000","last_became_active":"2024-08-29T15:37:29.344738+0000","last_became_peered":"2024-08-29T15:37:29.344738+0000","last_unstale":"2024-08-29T15:45:24.530201+0000","last_undegraded":"2024-08-29T15:45:24.530201+0000","last_fullsized":"2024-08-29T15:45:24.530201+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:14.103195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":224,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.525632+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:45:15.525632+0000","last_peered":"2024-08-29T15:45:15.525632+0000","last_clean":"2024-08-29T15:45:15.525632+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:45:15.525632+0000","last_undegraded":"2024-08-29T15:45:15.525632+0000","last_fullsized":"2024-08-29T15:45:15.525632+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":398,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:22.503837+0000","last_change":"2024-08-29T15:43:44.373358+0000","last_active":"2024-08-29T15:45:22.503837+0000","last_peered":"2024-08-29T15:45:22.503837+0000","last_clean":"2024-08-29T15:45:22.503837+0000","last_became_active":"2024-08-29T15:43:44.372934+0000","last_became_peered":"2024-08-29T15:43:44.372934+0000","last_unstale":"2024-08-29T15:45:22.503837+0000","last_undegraded":"2024-08-29T15:45:22.503837+0000","last_fullsized":"2024-08-29T15:45:22.503837+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:42.042366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"484'1512","reported_seq":1525,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:27.499824+0000","last_change":"2024-08-29T15:45:07.146387+0000","last_active":"2024-08-29T15:45:27.499824+0000","last_peered":"2024-08-29T15:45:27.499824+0000","last_clean":"2024-08-29T15:45:27.499824+0000","last_became_active":"2024-08-29T15:45:07.146122+0000","last_became_peered":"2024-08-29T15:45:07.146122+0000","last_unstale":"2024-08-29T15:45:27.499824+0000","last_undegraded":"2024-08-29T15:45:27.499824+0000","last_fullsized":"2024-08-29T15:45:27.499824+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1512,"log_dups_size":0,"ondisk_log_size":1512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:43:44.859969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99090432,"num_objects":1512,"num_object_clones":0,"num_object_copies":4536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1512,"num_write_kb":96768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":878,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:22.503610+0000","last_change":"2024-08-29T15:44:26.709678+0000","last_active":"2024-08-29T15:45:22.503610+0000","last_peered":"2024-08-29T15:45:22.503610+0000","last_clean":"2024-08-29T15:45:22.503610+0000","last_became_active":"2024-08-29T15:43:47.407472+0000","last_became_peered":"2024-08-29T15:43:47.407472+0000","last_unstale":"2024-08-29T15:45:22.503610+0000","last_undegraded":"2024-08-29T15:45:22.503610+0000","last_fullsized":"2024-08-29T15:45:22.503610+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_clean_scrub_stamp":"2024-08-29T15:44:26.709569+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:55:44.124152+0000","scrub_duration":0.0031838119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":432,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151124+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:45:15.151124+0000","last_peered":"2024-08-29T15:45:15.151124+0000","last_clean":"2024-08-29T15:45:15.151124+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:45:15.151124+0000","last_undegraded":"2024-08-29T15:45:15.151124+0000","last_fullsized":"2024-08-29T15:45:15.151124+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":272,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857976+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:45:15.857976+0000","last_peered":"2024-08-29T15:45:15.857976+0000","last_clean":"2024-08-29T15:45:15.857976+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:45:15.857976+0000","last_undegraded":"2024-08-29T15:45:15.857976+0000","last_fullsized":"2024-08-29T15:45:15.857976+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":339,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151049+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:45:15.151049+0000","last_peered":"2024-08-29T15:45:15.151049+0000","last_clean":"2024-08-29T15:45:15.151049+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:45:15.151049+0000","last_undegraded":"2024-08-29T15:45:15.151049+0000","last_fullsized":"2024-08-29T15:45:15.151049+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"484'1651","reported_seq":1664,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:29.759672+0000","last_change":"2024-08-29T15:45:06.941778+0000","last_active":"2024-08-29T15:45:29.759672+0000","last_peered":"2024-08-29T15:45:29.759672+0000","last_clean":"2024-08-29T15:45:29.759672+0000","last_became_active":"2024-08-29T15:45:06.941435+0000","last_became_peered":"2024-08-29T15:45:06.941435+0000","last_unstale":"2024-08-29T15:45:29.759672+0000","last_undegraded":"2024-08-29T15:45:29.759672+0000","last_fullsized":"2024-08-29T15:45:29.759672+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1651,"log_dups_size":0,"ondisk_log_size":1651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:57:03.840050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108199936,"num_objects":1651,"num_object_clones":0,"num_object_copies":4953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1651,"num_write_kb":105664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":918,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:25.292218+0000","last_change":"2024-08-29T15:45:25.292326+0000","last_active":"2024-08-29T15:45:25.292218+0000","last_peered":"2024-08-29T15:45:25.292218+0000","last_clean":"2024-08-29T15:45:25.292218+0000","last_became_active":"2024-08-29T15:44:22.605576+0000","last_became_peered":"2024-08-29T15:44:22.605576+0000","last_unstale":"2024-08-29T15:45:25.292218+0000","last_undegraded":"2024-08-29T15:45:25.292218+0000","last_fullsized":"2024-08-29T15:45:25.292218+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.292191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.292191+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:07:30.936596+0000","scrub_duration":0.0075757899999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":364,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151342+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:45:15.151342+0000","last_peered":"2024-08-29T15:45:15.151342+0000","last_clean":"2024-08-29T15:45:15.151342+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:45:15.151342+0000","last_undegraded":"2024-08-29T15:45:15.151342+0000","last_fullsized":"2024-08-29T15:45:15.151342+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1270,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529990+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:45:24.529990+0000","last_peered":"2024-08-29T15:45:24.529990+0000","last_clean":"2024-08-29T15:45:24.529990+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:45:24.529990+0000","last_undegraded":"2024-08-29T15:45:24.529990+0000","last_fullsized":"2024-08-29T15:45:24.529990+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.507904+0000","last_change":"2024-08-29T15:41:16.222377+0000","last_active":"2024-08-29T15:45:24.507904+0000","last_peered":"2024-08-29T15:45:24.507904+0000","last_clean":"2024-08-29T15:45:24.507904+0000","last_became_active":"2024-08-29T15:41:16.221775+0000","last_became_peered":"2024-08-29T15:41:16.221775+0000","last_unstale":"2024-08-29T15:45:24.507904+0000","last_undegraded":"2024-08-29T15:45:24.507904+0000","last_fullsized":"2024-08-29T15:45:24.507904+0000","mapping_epoch":439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:43:52.584991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"484'1725","reported_seq":1738,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:29.768016+0000","last_change":"2024-08-29T15:45:06.874772+0000","last_active":"2024-08-29T15:45:29.768016+0000","last_peered":"2024-08-29T15:45:29.768016+0000","last_clean":"2024-08-29T15:45:29.768016+0000","last_became_active":"2024-08-29T15:45:06.874516+0000","last_became_peered":"2024-08-29T15:45:06.874516+0000","last_unstale":"2024-08-29T15:45:29.768016+0000","last_undegraded":"2024-08-29T15:45:29.768016+0000","last_fullsized":"2024-08-29T15:45:29.768016+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1725,"log_dups_size":0,"ondisk_log_size":1725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:59:18.795330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113049600,"num_objects":1725,"num_object_clones":0,"num_object_copies":5175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1725,"num_write_kb":110400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":839,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.531401+0000","last_change":"2024-08-29T15:44:23.864259+0000","last_active":"2024-08-29T15:45:24.531401+0000","last_peered":"2024-08-29T15:45:24.531401+0000","last_clean":"2024-08-29T15:45:24.531401+0000","last_became_active":"2024-08-29T15:44:22.619475+0000","last_became_peered":"2024-08-29T15:44:22.619475+0000","last_unstale":"2024-08-29T15:45:24.531401+0000","last_undegraded":"2024-08-29T15:45:24.531401+0000","last_fullsized":"2024-08-29T15:45:24.531401+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:23.864183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:23.864183+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:07:39.942477+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":852,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529863+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:45:24.529863+0000","last_peered":"2024-08-29T15:45:24.529863+0000","last_clean":"2024-08-29T15:45:24.529863+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:45:24.529863+0000","last_undegraded":"2024-08-29T15:45:24.529863+0000","last_fullsized":"2024-08-29T15:45:24.529863+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":971,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.526171+0000","last_change":"2024-08-29T15:44:20.524258+0000","last_active":"2024-08-29T15:45:15.526171+0000","last_peered":"2024-08-29T15:45:15.526171+0000","last_clean":"2024-08-29T15:45:15.526171+0000","last_became_active":"2024-08-29T15:43:44.383379+0000","last_became_peered":"2024-08-29T15:43:44.383379+0000","last_unstale":"2024-08-29T15:45:15.526171+0000","last_undegraded":"2024-08-29T15:45:15.526171+0000","last_fullsized":"2024-08-29T15:45:15.526171+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:28:34.359806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":843,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.533367+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:45:24.533367+0000","last_peered":"2024-08-29T15:45:24.533367+0000","last_clean":"2024-08-29T15:45:24.533367+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:45:24.533367+0000","last_undegraded":"2024-08-29T15:45:24.533367+0000","last_fullsized":"2024-08-29T15:45:24.533367+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"484'1766","reported_seq":1779,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:29.771711+0000","last_change":"2024-08-29T15:45:07.146261+0000","last_active":"2024-08-29T15:45:29.771711+0000","last_peered":"2024-08-29T15:45:29.771711+0000","last_clean":"2024-08-29T15:45:29.771711+0000","last_became_active":"2024-08-29T15:45:07.145979+0000","last_became_peered":"2024-08-29T15:45:07.145979+0000","last_unstale":"2024-08-29T15:45:29.771711+0000","last_undegraded":"2024-08-29T15:45:29.771711+0000","last_fullsized":"2024-08-29T15:45:29.771711+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1766,"log_dups_size":0,"ondisk_log_size":1766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:28.876377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115736576,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1766,"num_write_kb":113024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":899,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.524592+0000","last_change":"2024-08-29T15:44:22.861452+0000","last_active":"2024-08-29T15:45:24.524592+0000","last_peered":"2024-08-29T15:45:24.524592+0000","last_clean":"2024-08-29T15:45:24.524592+0000","last_became_active":"2024-08-29T15:44:22.611166+0000","last_became_peered":"2024-08-29T15:44:22.611166+0000","last_unstale":"2024-08-29T15:45:24.524592+0000","last_undegraded":"2024-08-29T15:45:24.524592+0000","last_fullsized":"2024-08-29T15:45:24.524592+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.861280+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.861280+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:45.254970+0000","scrub_duration":0.0040975610000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":836,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.537174+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:45:24.537174+0000","last_peered":"2024-08-29T15:45:24.537174+0000","last_clean":"2024-08-29T15:45:24.537174+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:45:24.537174+0000","last_undegraded":"2024-08-29T15:45:24.537174+0000","last_fullsized":"2024-08-29T15:45:24.537174+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1488,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530376+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:45:24.530376+0000","last_peered":"2024-08-29T15:45:24.530376+0000","last_clean":"2024-08-29T15:45:24.530376+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:45:24.530376+0000","last_undegraded":"2024-08-29T15:45:24.530376+0000","last_fullsized":"2024-08-29T15:45:24.530376+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":175,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529931+0000","last_change":"2024-08-29T15:43:44.383123+0000","last_active":"2024-08-29T15:45:24.529931+0000","last_peered":"2024-08-29T15:45:24.529931+0000","last_clean":"2024-08-29T15:45:24.529931+0000","last_became_active":"2024-08-29T15:43:44.382729+0000","last_became_peered":"2024-08-29T15:43:44.382729+0000","last_unstale":"2024-08-29T15:45:24.529931+0000","last_undegraded":"2024-08-29T15:45:24.529931+0000","last_fullsized":"2024-08-29T15:45:24.529931+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:14:30.890056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"484'1396","reported_seq":1415,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:26.704506+0000","last_change":"2024-08-29T15:45:07.148809+0000","last_active":"2024-08-29T15:45:26.704506+0000","last_peered":"2024-08-29T15:45:26.704506+0000","last_clean":"2024-08-29T15:45:26.704506+0000","last_became_active":"2024-08-29T15:45:07.148514+0000","last_became_peered":"2024-08-29T15:45:07.148514+0000","last_unstale":"2024-08-29T15:45:26.704506+0000","last_undegraded":"2024-08-29T15:45:26.704506+0000","last_fullsized":"2024-08-29T15:45:26.704506+0000","mapping_epoch":480,"log_start":"0'0","ondisk_log_start":"0'0","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":1396,"log_dups_size":0,"ondisk_log_size":1396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:42:28.486131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91488256,"num_objects":1396,"num_object_clones":0,"num_object_copies":4188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1396,"num_write_kb":89344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":1015,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530368+0000","last_change":"2024-08-29T15:43:26.775058+0000","last_active":"2024-08-29T15:45:24.530368+0000","last_peered":"2024-08-29T15:45:24.530368+0000","last_clean":"2024-08-29T15:45:24.530368+0000","last_became_active":"2024-08-29T15:43:26.284121+0000","last_became_peered":"2024-08-29T15:43:26.284121+0000","last_unstale":"2024-08-29T15:45:24.530368+0000","last_undegraded":"2024-08-29T15:45:24.530368+0000","last_fullsized":"2024-08-29T15:45:24.530368+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.774861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.774861+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:37:57.962524+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":210,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530131+0000","last_change":"2024-08-29T15:43:44.393371+0000","last_active":"2024-08-29T15:45:24.530131+0000","last_peered":"2024-08-29T15:45:24.530131+0000","last_clean":"2024-08-29T15:45:24.530131+0000","last_became_active":"2024-08-29T15:43:44.387134+0000","last_became_peered":"2024-08-29T15:43:44.387134+0000","last_unstale":"2024-08-29T15:45:24.530131+0000","last_undegraded":"2024-08-29T15:45:24.530131+0000","last_fullsized":"2024-08-29T15:45:24.530131+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:33.515571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":128,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.536553+0000","last_change":"2024-08-29T15:41:00.761163+0000","last_active":"2024-08-29T15:45:24.536553+0000","last_peered":"2024-08-29T15:45:24.536553+0000","last_clean":"2024-08-29T15:45:24.536553+0000","last_became_active":"2024-08-29T15:41:00.749422+0000","last_became_peered":"2024-08-29T15:41:00.749422+0000","last_unstale":"2024-08-29T15:45:24.536553+0000","last_undegraded":"2024-08-29T15:45:24.536553+0000","last_fullsized":"2024-08-29T15:45:24.536553+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:31:21.012113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":272,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857930+0000","last_change":"2024-08-29T15:19:46.808467+0000","last_active":"2024-08-29T15:45:15.857930+0000","last_peered":"2024-08-29T15:45:15.857930+0000","last_clean":"2024-08-29T15:45:15.857930+0000","last_became_active":"2024-08-29T15:19:46.805269+0000","last_became_peered":"2024-08-29T15:19:46.805269+0000","last_unstale":"2024-08-29T15:45:15.857930+0000","last_undegraded":"2024-08-29T15:45:15.857930+0000","last_fullsized":"2024-08-29T15:45:15.857930+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:25:25.993314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"90'149","reported_seq":843,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529931+0000","last_change":"2024-08-29T15:44:59.239507+0000","last_active":"2024-08-29T15:45:24.529931+0000","last_peered":"2024-08-29T15:45:24.529931+0000","last_clean":"2024-08-29T15:45:24.529931+0000","last_became_active":"2024-08-29T15:44:59.239049+0000","last_became_peered":"2024-08-29T15:44:59.239049+0000","last_unstale":"2024-08-29T15:45:24.529931+0000","last_undegraded":"2024-08-29T15:45:24.529931+0000","last_fullsized":"2024-08-29T15:45:24.529931+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:28.746327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:28.746327+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:22:01.855005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":271,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.524383+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:45:24.524383+0000","last_peered":"2024-08-29T15:45:24.524383+0000","last_clean":"2024-08-29T15:45:24.524383+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:45:24.524383+0000","last_undegraded":"2024-08-29T15:45:24.524383+0000","last_fullsized":"2024-08-29T15:45:24.524383+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":280,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857957+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:45:15.857957+0000","last_peered":"2024-08-29T15:45:15.857957+0000","last_clean":"2024-08-29T15:45:15.857957+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:45:15.857957+0000","last_undegraded":"2024-08-29T15:45:15.857957+0000","last_fullsized":"2024-08-29T15:45:15.857957+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"90'149","reported_seq":922,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151441+0000","last_change":"2024-08-29T15:44:59.242106+0000","last_active":"2024-08-29T15:45:15.151441+0000","last_peered":"2024-08-29T15:45:15.151441+0000","last_clean":"2024-08-29T15:45:15.151441+0000","last_became_active":"2024-08-29T15:44:59.241743+0000","last_became_peered":"2024-08-29T15:44:59.241743+0000","last_unstale":"2024-08-29T15:45:15.151441+0000","last_undegraded":"2024-08-29T15:45:15.151441+0000","last_fullsized":"2024-08-29T15:45:15.151441+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:37:16.688893+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:37:16.688893+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.016432537000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":32,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.150790+0000","last_change":"2024-08-29T15:43:44.375331+0000","last_active":"2024-08-29T15:45:15.150790+0000","last_peered":"2024-08-29T15:45:15.150790+0000","last_clean":"2024-08-29T15:45:15.150790+0000","last_became_active":"2024-08-29T15:43:44.375027+0000","last_became_peered":"2024-08-29T15:43:44.375027+0000","last_unstale":"2024-08-29T15:45:15.150790+0000","last_undegraded":"2024-08-29T15:45:15.150790+0000","last_fullsized":"2024-08-29T15:45:15.150790+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:04:37.416657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":599,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.533361+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:45:24.533361+0000","last_peered":"2024-08-29T15:45:24.533361+0000","last_clean":"2024-08-29T15:45:24.533361+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:45:24.533361+0000","last_undegraded":"2024-08-29T15:45:24.533361+0000","last_fullsized":"2024-08-29T15:45:24.533361+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'149","reported_seq":1018,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.521987+0000","last_change":"2024-08-29T15:44:59.244921+0000","last_active":"2024-08-29T15:45:24.521987+0000","last_peered":"2024-08-29T15:45:24.521987+0000","last_clean":"2024-08-29T15:45:24.521987+0000","last_became_active":"2024-08-29T15:44:59.244391+0000","last_became_peered":"2024-08-29T15:44:59.244391+0000","last_unstale":"2024-08-29T15:45:24.521987+0000","last_undegraded":"2024-08-29T15:45:24.521987+0000","last_fullsized":"2024-08-29T15:45:24.521987+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.778004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.778004+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:01.098771+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":360,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151435+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:45:15.151435+0000","last_peered":"2024-08-29T15:45:15.151435+0000","last_clean":"2024-08-29T15:45:15.151435+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:45:15.151435+0000","last_undegraded":"2024-08-29T15:45:15.151435+0000","last_fullsized":"2024-08-29T15:45:15.151435+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":799,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529978+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:45:24.529978+0000","last_peered":"2024-08-29T15:45:24.529978+0000","last_clean":"2024-08-29T15:45:24.529978+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:45:24.529978+0000","last_undegraded":"2024-08-29T15:45:24.529978+0000","last_fullsized":"2024-08-29T15:45:24.529978+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"90'149","reported_seq":794,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:26.882653+0000","last_change":"2024-08-29T15:45:26.882752+0000","last_active":"2024-08-29T15:45:26.882653+0000","last_peered":"2024-08-29T15:45:26.882653+0000","last_clean":"2024-08-29T15:45:26.882653+0000","last_became_active":"2024-08-29T15:44:59.248534+0000","last_became_peered":"2024-08-29T15:44:59.248534+0000","last_unstale":"2024-08-29T15:45:26.882653+0000","last_undegraded":"2024-08-29T15:45:26.882653+0000","last_fullsized":"2024-08-29T15:45:26.882653+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.882638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.882638+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:29:07.801722+0000","scrub_duration":0.001489043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":301,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151640+0000","last_change":"2024-08-29T15:43:44.372361+0000","last_active":"2024-08-29T15:45:15.151640+0000","last_peered":"2024-08-29T15:45:15.151640+0000","last_clean":"2024-08-29T15:45:15.151640+0000","last_became_active":"2024-08-29T15:43:44.371982+0000","last_became_peered":"2024-08-29T15:43:44.371982+0000","last_unstale":"2024-08-29T15:45:15.151640+0000","last_undegraded":"2024-08-29T15:45:15.151640+0000","last_fullsized":"2024-08-29T15:45:15.151640+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:38:20.070361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":851,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530129+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:45:24.530129+0000","last_peered":"2024-08-29T15:45:24.530129+0000","last_clean":"2024-08-29T15:45:24.530129+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:45:24.530129+0000","last_undegraded":"2024-08-29T15:45:24.530129+0000","last_fullsized":"2024-08-29T15:45:24.530129+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"90'149","reported_seq":726,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:26.912193+0000","last_change":"2024-08-29T15:45:26.912257+0000","last_active":"2024-08-29T15:45:26.912193+0000","last_peered":"2024-08-29T15:45:26.912193+0000","last_clean":"2024-08-29T15:45:26.912193+0000","last_became_active":"2024-08-29T15:44:59.244250+0000","last_became_peered":"2024-08-29T15:44:59.244250+0000","last_unstale":"2024-08-29T15:45:26.912193+0000","last_undegraded":"2024-08-29T15:45:26.912193+0000","last_fullsized":"2024-08-29T15:45:26.912193+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.912179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.912179+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:51.014992+0000","scrub_duration":0.0012045789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":342,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151614+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:45:15.151614+0000","last_peered":"2024-08-29T15:45:15.151614+0000","last_clean":"2024-08-29T15:45:15.151614+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:45:15.151614+0000","last_undegraded":"2024-08-29T15:45:15.151614+0000","last_fullsized":"2024-08-29T15:45:15.151614+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":96,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529729+0000","last_change":"2024-08-29T15:43:44.374180+0000","last_active":"2024-08-29T15:45:24.529729+0000","last_peered":"2024-08-29T15:45:24.529729+0000","last_clean":"2024-08-29T15:45:24.529729+0000","last_became_active":"2024-08-29T15:43:44.373767+0000","last_became_peered":"2024-08-29T15:43:44.373767+0000","last_unstale":"2024-08-29T15:45:24.529729+0000","last_undegraded":"2024-08-29T15:45:24.529729+0000","last_fullsized":"2024-08-29T15:45:24.529729+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:07.501547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'149","reported_seq":994,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151042+0000","last_change":"2024-08-29T15:44:59.245945+0000","last_active":"2024-08-29T15:45:15.151042+0000","last_peered":"2024-08-29T15:45:15.151042+0000","last_clean":"2024-08-29T15:45:15.151042+0000","last_became_active":"2024-08-29T15:44:59.245584+0000","last_became_peered":"2024-08-29T15:44:59.245584+0000","last_unstale":"2024-08-29T15:45:15.151042+0000","last_undegraded":"2024-08-29T15:45:15.151042+0000","last_fullsized":"2024-08-29T15:45:15.151042+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:23.954739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:23.954739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:58:16.364501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":812,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530235+0000","last_change":"2024-08-29T15:43:44.384584+0000","last_active":"2024-08-29T15:45:24.530235+0000","last_peered":"2024-08-29T15:45:24.530235+0000","last_clean":"2024-08-29T15:45:24.530235+0000","last_became_active":"2024-08-29T15:43:44.383996+0000","last_became_peered":"2024-08-29T15:43:44.383996+0000","last_unstale":"2024-08-29T15:45:24.530235+0000","last_undegraded":"2024-08-29T15:45:24.530235+0000","last_fullsized":"2024-08-29T15:45:24.530235+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:01:14.990354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":831,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.525452+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:45:24.525452+0000","last_peered":"2024-08-29T15:45:24.525452+0000","last_clean":"2024-08-29T15:45:24.525452+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:45:24.525452+0000","last_undegraded":"2024-08-29T15:45:24.525452+0000","last_fullsized":"2024-08-29T15:45:24.525452+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'149","reported_seq":738,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:27.312553+0000","last_change":"2024-08-29T15:45:27.312630+0000","last_active":"2024-08-29T15:45:27.312553+0000","last_peered":"2024-08-29T15:45:27.312553+0000","last_clean":"2024-08-29T15:45:27.312553+0000","last_became_active":"2024-08-29T15:44:59.247776+0000","last_became_peered":"2024-08-29T15:44:59.247776+0000","last_unstale":"2024-08-29T15:45:27.312553+0000","last_undegraded":"2024-08-29T15:45:27.312553+0000","last_fullsized":"2024-08-29T15:45:27.312553+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:27.312539+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:27.312539+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:07.441954+0000","scrub_duration":0.0034059530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":835,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.533196+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:45:24.533196+0000","last_peered":"2024-08-29T15:45:24.533196+0000","last_clean":"2024-08-29T15:45:24.533196+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:45:24.533196+0000","last_undegraded":"2024-08-29T15:45:24.533196+0000","last_fullsized":"2024-08-29T15:45:24.533196+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":270,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857894+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:45:15.857894+0000","last_peered":"2024-08-29T15:45:15.857894+0000","last_clean":"2024-08-29T15:45:15.857894+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:45:15.857894+0000","last_undegraded":"2024-08-29T15:45:15.857894+0000","last_fullsized":"2024-08-29T15:45:15.857894+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'149","reported_seq":992,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530835+0000","last_change":"2024-08-29T15:44:59.243109+0000","last_active":"2024-08-29T15:45:24.530835+0000","last_peered":"2024-08-29T15:45:24.530835+0000","last_clean":"2024-08-29T15:45:24.530835+0000","last_became_active":"2024-08-29T15:44:59.242614+0000","last_became_peered":"2024-08-29T15:44:59.242614+0000","last_unstale":"2024-08-29T15:45:24.530835+0000","last_undegraded":"2024-08-29T15:45:24.530835+0000","last_fullsized":"2024-08-29T15:45:24.530835+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:31.851590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:31.851590+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:16:37.507078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":876,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530048+0000","last_change":"2024-08-29T15:39:11.832387+0000","last_active":"2024-08-29T15:45:24.530048+0000","last_peered":"2024-08-29T15:45:24.530048+0000","last_clean":"2024-08-29T15:45:24.530048+0000","last_became_active":"2024-08-29T15:39:11.832056+0000","last_became_peered":"2024-08-29T15:39:11.832056+0000","last_unstale":"2024-08-29T15:45:24.530048+0000","last_undegraded":"2024-08-29T15:45:24.530048+0000","last_fullsized":"2024-08-29T15:45:24.530048+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:47:42.049015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":147,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.858111+0000","last_change":"2024-08-29T15:43:44.378845+0000","last_active":"2024-08-29T15:45:15.858111+0000","last_peered":"2024-08-29T15:45:15.858111+0000","last_clean":"2024-08-29T15:45:15.858111+0000","last_became_active":"2024-08-29T15:43:44.378491+0000","last_became_peered":"2024-08-29T15:43:44.378491+0000","last_unstale":"2024-08-29T15:45:15.858111+0000","last_undegraded":"2024-08-29T15:45:15.858111+0000","last_fullsized":"2024-08-29T15:45:15.858111+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:13:25.137263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"90'149","reported_seq":848,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.531141+0000","last_change":"2024-08-29T15:44:59.241503+0000","last_active":"2024-08-29T15:45:24.531141+0000","last_peered":"2024-08-29T15:45:24.531141+0000","last_clean":"2024-08-29T15:45:24.531141+0000","last_became_active":"2024-08-29T15:44:59.241088+0000","last_became_peered":"2024-08-29T15:44:59.241088+0000","last_unstale":"2024-08-29T15:45:24.531141+0000","last_undegraded":"2024-08-29T15:45:24.531141+0000","last_fullsized":"2024-08-29T15:45:24.531141+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:27.788587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:27.788587+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:13:46.377483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":297,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857893+0000","last_change":"2024-08-29T15:41:44.446288+0000","last_active":"2024-08-29T15:45:15.857893+0000","last_peered":"2024-08-29T15:45:15.857893+0000","last_clean":"2024-08-29T15:45:15.857893+0000","last_became_active":"2024-08-29T15:41:44.445958+0000","last_became_peered":"2024-08-29T15:41:44.445958+0000","last_unstale":"2024-08-29T15:45:15.857893+0000","last_undegraded":"2024-08-29T15:45:15.857893+0000","last_fullsized":"2024-08-29T15:45:15.857893+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:47:53.474043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":403,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151175+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:45:15.151175+0000","last_peered":"2024-08-29T15:45:15.151175+0000","last_clean":"2024-08-29T15:45:15.151175+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:45:15.151175+0000","last_undegraded":"2024-08-29T15:45:15.151175+0000","last_fullsized":"2024-08-29T15:45:15.151175+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":270,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857966+0000","last_change":"2024-08-29T15:19:46.809473+0000","last_active":"2024-08-29T15:45:15.857966+0000","last_peered":"2024-08-29T15:45:15.857966+0000","last_clean":"2024-08-29T15:45:15.857966+0000","last_became_active":"2024-08-29T15:19:46.809074+0000","last_became_peered":"2024-08-29T15:19:46.809074+0000","last_unstale":"2024-08-29T15:45:15.857966+0000","last_undegraded":"2024-08-29T15:45:15.857966+0000","last_fullsized":"2024-08-29T15:45:15.857966+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:48:25.157709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":403,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151440+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:45:15.151440+0000","last_peered":"2024-08-29T15:45:15.151440+0000","last_clean":"2024-08-29T15:45:15.151440+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:45:15.151440+0000","last_undegraded":"2024-08-29T15:45:15.151440+0000","last_fullsized":"2024-08-29T15:45:15.151440+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"90'149","reported_seq":1033,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.529640+0000","last_change":"2024-08-29T15:44:59.243558+0000","last_active":"2024-08-29T15:45:24.529640+0000","last_peered":"2024-08-29T15:45:24.529640+0000","last_clean":"2024-08-29T15:45:24.529640+0000","last_became_active":"2024-08-29T15:44:59.243114+0000","last_became_peered":"2024-08-29T15:44:59.243114+0000","last_unstale":"2024-08-29T15:45:24.529640+0000","last_undegraded":"2024-08-29T15:45:24.529640+0000","last_fullsized":"2024-08-29T15:45:24.529640+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:25.805561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:25.805561+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:54:30.325522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":867,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.530309+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:45:24.530309+0000","last_peered":"2024-08-29T15:45:24.530309+0000","last_clean":"2024-08-29T15:45:24.530309+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:45:24.530309+0000","last_undegraded":"2024-08-29T15:45:24.530309+0000","last_fullsized":"2024-08-29T15:45:24.530309+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":870,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.531306+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:45:24.531306+0000","last_peered":"2024-08-29T15:45:24.531306+0000","last_clean":"2024-08-29T15:45:24.531306+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:45:24.531306+0000","last_undegraded":"2024-08-29T15:45:24.531306+0000","last_fullsized":"2024-08-29T15:45:24.531306+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'149","reported_seq":811,"reported_epoch":484,"state":"active+clean","last_fresh":"2024-08-29T15:45:24.970545+0000","last_change":"2024-08-29T15:45:24.970545+0000","last_active":"2024-08-29T15:45:24.970545+0000","last_peered":"2024-08-29T15:45:24.970545+0000","last_clean":"2024-08-29T15:45:24.970545+0000","last_became_active":"2024-08-29T15:44:59.247230+0000","last_became_peered":"2024-08-29T15:44:59.247230+0000","last_unstale":"2024-08-29T15:45:24.970545+0000","last_undegraded":"2024-08-29T15:45:24.970545+0000","last_fullsized":"2024-08-29T15:45:24.970545+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:24.970456+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:45:24.970456+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:15:54.900538+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":270,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.857528+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:45:15.857528+0000","last_peered":"2024-08-29T15:45:15.857528+0000","last_clean":"2024-08-29T15:45:15.857528+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:45:15.857528+0000","last_undegraded":"2024-08-29T15:45:15.857528+0000","last_fullsized":"2024-08-29T15:45:15.857528+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":308,"reported_epoch":483,"state":"active+clean","last_fresh":"2024-08-29T15:45:15.151580+0000","last_change":"2024-08-29T15:43:44.391961+0000","last_active":"2024-08-29T15:45:15.151580+0000","last_peered":"2024-08-29T15:45:15.151580+0000","last_clean":"2024-08-29T15:45:15.151580+0000","last_became_active":"2024-08-29T15:43:44.375278+0000","last_became_peered":"2024-08-29T15:43:44.375278+0000","last_unstale":"2024-08-29T15:45:15.151580+0000","last_undegraded":"2024-08-29T15:45:15.151580+0000","last_fullsized":"2024-08-29T15:45:15.151580+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:51:42.693350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1665662976,"num_objects":25416,"num_object_clones":0,"num_object_copies":76248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":25416,"num_write_kb":1626624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5218369536,"data_stored":5218369536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":25416,"ondisk_log_size":25416,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":7758,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":129,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234610,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717348,"kb_used_data":869980,"kb_used_omap":1,"kb_used_meta":847358,"kb_avail":92027804,"statfs":{"total":95995035648,"available":94236471296,"internally_reserved":0,"allocated":890859520,"data_stored":887815031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":867695051},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496241,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375708,"kb_used_data":785108,"kb_used_omap":1,"kb_used_meta":590590,"kb_avail":92369444,"statfs":{"total":95995035648,"available":94586310656,"internally_reserved":0,"allocated":803950592,"data_stored":800914003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":604764615},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426827,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1141848,"kb_used_data":559248,"kb_used_omap":8,"kb_used_meta":582583,"kb_avail":92603304,"statfs":{"total":95995035648,"available":94825783296,"internally_reserved":0,"allocated":572669952,"data_stored":569633211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":596565898},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019520,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1839964,"kb_used_data":1032788,"kb_used_omap":1,"kb_used_meta":807166,"kb_avail":91905188,"statfs":{"total":95995035648,"available":94110912512,"internally_reserved":0,"allocated":1057574912,"data_stored":1054537798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":826538438},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435704,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1322060,"kb_used_data":451908,"kb_used_omap":4,"kb_used_meta":870139,"kb_avail":92423092,"statfs":{"total":95995035648,"available":94641246208,"internally_reserved":0,"allocated":462753792,"data_stored":459728535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":891022601},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903556,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1451404,"kb_used_data":814340,"kb_used_omap":10,"kb_used_meta":637045,"kb_avail":92293748,"statfs":{"total":95995035648,"available":94508797952,"internally_reserved":0,"allocated":833884160,"data_stored":830858793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":652334608},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762757,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1611596,"kb_used_data":683072,"kb_used_omap":7,"kb_used_meta":928504,"kb_avail":92133556,"statfs":{"total":95995035648,"available":94344761344,"internally_reserved":0,"allocated":699465728,"data_stored":696443351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":950788450},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926281,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1510080,"kb_used_data":788408,"kb_used_omap":7,"kb_used_meta":721656,"kb_avail":92235072,"statfs":{"total":95995035648,"available":94448713728,"internally_reserved":0,"allocated":807329792,"data_stored":804315292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":738975887},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":835911680,"data_stored":835911680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":364118016,"data_stored":364118016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":845873152,"data_stored":845873152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":466681856,"data_stored":466681856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1133838336,"data_stored":1133838336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":290258944,"data_stored":290258944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":835780608,"data_stored":835780608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":445906944,"data_stored":445906944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:45:34.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool set .mgr pgp_num 31 2024-08-29T15:45:34.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:34.795 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:34.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:34.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:34.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:34.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:34 smithi171 ceph-mon[90031]: from='client.56725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:45:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:34 smithi171 ceph-mon[90031]: pgmap v1719: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-08-29T15:45:35.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:35.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:35.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:35.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:35.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:35.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:35.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:35.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:35.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:35.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:35.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:35.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:35.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:35.561 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:35.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:35.646 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2745652364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:35.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2745652364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:35.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:35.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 17 16 39177 39161 143.82 106 0.00575255 0.00694722 2024-08-29T15:45:35.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:45:35.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:45:35.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:45:35.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:45:35.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:45:35.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:45:35.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:45:35.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:35.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:35.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:35.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:35.845 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:35.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:35.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:35.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:35.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:35.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:35.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2745652364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:36.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:36.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:36.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:36.226 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:36.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:36.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:36.364 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:36.401 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:36.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:36.503 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:36.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:36.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:36.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:36.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:36.666 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:36.668 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:36.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:36.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:36.670 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 18 16 40983 40967 142.101 112.875 0.00517668 0.0070309 2024-08-29T15:45:36.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:36 smithi027 ceph-mon[98970]: pgmap v1720: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 155 MiB/s wr, 2.47k op/s 2024-08-29T15:45:36.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:36 smithi027 ceph-mon[93921]: pgmap v1720: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 155 MiB/s wr, 2.47k op/s 2024-08-29T15:45:36.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:36.804 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:36.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:36.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:36.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:36.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:36.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:36 smithi171 ceph-mon[90031]: pgmap v1720: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 155 MiB/s wr, 2.47k op/s 2024-08-29T15:45:37.045 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:37.087 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:37.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:37.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:37.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:37.227 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:37.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:37.228 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:45:37.228 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:45:37.229 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:45:37.229 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:45:37.229 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:45:37.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:37.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:37.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:37.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:37.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:37.493 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:37.501 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:37.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:37.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:37.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:37.646 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:37.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:37.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:37.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 19 16 42825 42809 140.682 115.125 0.00805386 0.00710319 2024-08-29T15:45:37.749 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:37.782 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:37.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:37.884 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:37.920 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:37.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:37.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:37.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:45:38.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:38.088 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:38.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:38.190 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:38.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:38.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:38.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:38.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:38.336 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:38.384 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:38.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:38.486 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:38.519 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:38.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:38.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:38.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:38.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:38.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:38.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:45:38.669516+0000 min lat: 0.00100862 max lat: 0.313675 avg lat: 0.00715839 2024-08-29T15:45:38.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:45:38.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 20 16 44726 44710 139.589 118.812 0.00520686 0.00715839 2024-08-29T15:45:38.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:38.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:38.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:38.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:38.815 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:38 smithi171 ceph-mon[90031]: pgmap v1721: 119 pgs: 119 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-29T15:45:38.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:38 smithi027 ceph-mon[98970]: pgmap v1721: 119 pgs: 119 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-29T15:45:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:38 smithi027 ceph-mon[93921]: pgmap v1721: 119 pgs: 119 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-29T15:45:38.909 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:38.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:38.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:38.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:39.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:39.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:39.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:39.193 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:39.238 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:39.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:39.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:39.246 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:45:39.290 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:45:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:45:39] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.51.0" 2024-08-29T15:45:39.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:39.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:39.391 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:39.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:39.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:39.529 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:39.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:39.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:39.667 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:39.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:39.669 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 21 16 46523 46507 138.291 112.312 0.00692893 0.00722591 2024-08-29T15:45:39.769 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:39.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:39.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:39.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:39.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:39.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:39.966 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:39.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:40.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:40.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:40.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:40.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:40.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:40.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:40.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:40.328 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:45:40.328 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:45:40.328 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:45:40.328 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:45:40.328 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.328 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:180121 8.6 e372/344 8:6f0f00f8:::benchmark_data_smithi027_306999_object53342:head v 372'11323, mlcod=372'11319)", 2024-08-29T15:45:40.328 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:30.556359+0000", 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 489.76870996500003, 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049309257000000002, 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556359+0000", 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556359+0000", 2024-08-29T15:45:40.329 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556358+0000", 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556363+0000", 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.395e-06 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.330 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556378+0000", 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4881e-05 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556385+0000", 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8499999999999996e-06 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556471+0000", 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5739e-05 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.331 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.556489+0000", 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8725000000000001e-05 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605522+0000", 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.049032294999999997 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605668+0000", 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014615599999999999 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.332 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:180122 8.6 e372/344 8:64fe0a2b:::benchmark_data_smithi027_306999_object53343:head v 372'11324, mlcod=372'11321)", 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 489.76173616599999, 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042496968000000003, 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.333 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563332+0000", 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563337+0000", 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4129999999999997e-06 2024-08-29T15:45:40.334 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563345+0000", 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2910000000000002e-06 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.563349+0000", 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6029999999999998e-06 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.570758+0000", 2024-08-29T15:45:40.335 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.007409044 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.570777+0000", 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.842e-05 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605672+0000", 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034895109000000001 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.336 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605829+0000", 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015755599999999999 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:180138 8.6 e372/344 8:6a769c0c:::benchmark_data_smithi027_306999_object53359:head v 372'11325, mlcod=372'11321)", 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:30.564453+0000", 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 489.76061571399998, 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041501364999999998, 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.337 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564453+0000", 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564453+0000", 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564452+0000", 2024-08-29T15:45:40.338 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564458+0000", 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8590000000000001e-06 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564469+0000", 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0536e-05 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.564475+0000", 2024-08-29T15:45:40.339 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9830000000000001e-06 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.575560+0000", 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011084946 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.575605+0000", 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4928000000000001e-05 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.340 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605834+0000", 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030229096 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:30.605954+0000", 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000120667 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:236681 8.6 e393/344 8:68c8102a:::benchmark_data_smithi027_306999_object109902:head v 393'14929, mlcod=393'14926)", 2024-08-29T15:45:40.341 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:46.096960+0000", 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 414.22810850100001, 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036404883999999998, 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096960+0000", 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096960+0000", 2024-08-29T15:45:40.342 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096959+0000", 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096971+0000", 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2201000000000001e-05 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.343 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096988+0000", 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7407000000000001e-05 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.096998+0000", 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0107e-05 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.097070+0000", 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1593000000000004e-05 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.344 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.097094+0000", 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3966999999999999e-05 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.133154+0000", 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036059846 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.133365+0000", 2024-08-29T15:45:40.345 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000211239 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:236689 8.6 e393/344 8:60741c53:::benchmark_data_smithi027_306999_object109910:head v 393'14931, mlcod=393'14926)", 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:46.101218+0000", 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 414.22385089400001, 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.042136763000000001, 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.346 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101218+0000", 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101218+0000", 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101217+0000", 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.347 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101225+0000", 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0909999999999993e-06 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101240+0000", 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5225000000000001e-05 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.101247+0000", 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4499999999999998e-06 2024-08-29T15:45:40.348 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.114281+0000", 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013033916 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.114305+0000", 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3921999999999999e-05 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.349 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143195+0000", 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028890078 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143355+0000", 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000159149 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46475.0:236691 8.6 e393/344 8:672d18b6:::benchmark_data_smithi027_306999_object109912:head v 393'14932, mlcod=393'14926)", 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:46.104680+0000", 2024-08-29T15:45:40.350 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 414.22038909999998, 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.038806661999999999, 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104680+0000", 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104680+0000", 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.351 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104678+0000", 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104690+0000", 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1099e-05 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104709+0000", 2024-08-29T15:45:40.352 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9902999999999999e-05 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.104718+0000", 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5059999999999995e-06 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.121620+0000", 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016901596000000001 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.121667+0000", 2024-08-29T15:45:40.353 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7104999999999997e-05 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143359+0000", 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021692052 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:46.143486+0000", 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012745399999999999 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.354 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:75304 9.b e432/420 9:d35b9d25:::benchmark_data_smithi027_341149_object75303:head v 432'4759, mlcod=432'4755)", 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:40:38.926391+0000", 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 301.39867790400001, 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017142794999999999, 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926391+0000", 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.355 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.356 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926391+0000", 2024-08-29T15:45:40.356 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.356 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.356 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:40.357 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.357 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926390+0000", 2024-08-29T15:45:40.357 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926430+0000", 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9685999999999999e-05 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926450+0000", 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0497e-05 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.358 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926458+0000", 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1799999999999996e-06 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926515+0000", 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6993000000000001e-05 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.926536+0000", 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0548e-05 2024-08-29T15:45:40.359 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.942032+0000", 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015496535000000001 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:38.943534+0000", 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0015011720000000001 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.360 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:90597 9.1 e432/420 9:8fcb4ce9:::benchmark_data_smithi027_341149_object90596:head v 432'5659, mlcod=432'5656)", 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:40:51.469906+0000", 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 288.855163038, 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016407099000000001, 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.469906+0000", 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.361 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.469906+0000", 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.469905+0000", 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.470024+0000", 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000118823 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.362 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.470037+0000", 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3277e-05 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.470044+0000", 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4839999999999998e-06 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.471459+0000", 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014141970000000001 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.363 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.471506+0000", 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7055999999999998e-05 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.486274+0000", 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01476889 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:40:51.486313+0000", 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8219000000000001e-05 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.364 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:114831 9.1 e447/420 9:8308c35c:::benchmark_data_smithi027_341149_object12401:head v 447'7168, mlcod=447'7166)", 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:51.926509+0000", 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 228.39856003200001, 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033561088000000003, 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926509+0000", 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.365 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926509+0000", 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926508+0000", 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926516+0000", 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8510000000000005e-06 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.366 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926530+0000", 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3522e-05 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926536+0000", 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3199999999999996e-06 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926575+0000", 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9122999999999998e-05 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.367 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.926592+0000", 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6526000000000001e-05 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.959986+0000", 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033394090000000001 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.960070+0000", 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3647999999999996e-05 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.368 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:114832 9.0 e447/446 9:06227d88:::benchmark_data_smithi027_341149_object12402:head v 447'6974, mlcod=429'517)", 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:51.927353+0000", 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 228.397715855, 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028675895, 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927353+0000", 2024-08-29T15:45:40.369 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927353+0000", 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927352+0000", 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.370 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927359+0000", 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.199e-06 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927371+0000", 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1878e-05 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927377+0000", 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4710000000000004e-06 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.371 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927426+0000", 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8463e-05 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.927448+0000", 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2396e-05 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.955983+0000", 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028535151000000002 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.372 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956029+0000", 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5198999999999999e-05 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:114834 9.b e447/420 9:d707837b:::benchmark_data_smithi027_341149_object12404:head v 447'7158, mlcod=447'7156)", 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:41:51.931036+0000", 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 228.39403230400001, 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.029067564000000001, 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.373 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931036+0000", 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931036+0000", 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931035+0000", 2024-08-29T15:45:40.374 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.931043+0000", 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7700000000000001e-06 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956560+0000", 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025517387999999998 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.375 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956571+0000", 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0098000000000001e-05 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956615+0000", 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3946999999999998e-05 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.956635+0000", 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0295000000000001e-05 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.376 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.960078+0000", 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003442826 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:41:51.960104+0000", 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6239000000000001e-05 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119373 9.0 e447/446 9:00090137:::benchmark_data_smithi027_341149_object16943:head v 447'7240, mlcod=429'517)", 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.436935+0000", 2024-08-29T15:45:40.377 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 219.88813353800001, 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018171396999999999, 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436935+0000", 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436935+0000", 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.378 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436934+0000", 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.379 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436944+0000", 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9399999999999997e-06 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436964+0000", 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0052e-05 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.436973+0000", 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6690000000000001e-06 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.380 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437141+0000", 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000168018 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437213+0000", 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1418e-05 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455035+0000", 2024-08-29T15:45:40.381 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017822489 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455106+0000", 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1141000000000002e-05 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119378 9.0 e447/446 9:052ba21d:::benchmark_data_smithi027_341149_object16948:head v 447'7241, mlcod=429'517)", 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.437597+0000", 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 219.88747177600001, 2024-08-29T15:45:40.382 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017570830999999999, 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437597+0000", 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437597+0000", 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.383 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437596+0000", 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437603+0000", 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5549999999999999e-06 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437616+0000", 2024-08-29T15:45:40.384 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3407999999999999e-05 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.437622+0000", 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.02e-06 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.441222+0000", 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035998969999999999 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.385 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.441247+0000", 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5188e-05 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455110+0000", 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013862670000000001 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.455168+0000", 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7708999999999999e-05 2024-08-29T15:45:40.386 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119456 9.b e447/420 9:d0fbc3a5:::benchmark_data_smithi027_341149_object17026:head v 447'7429, mlcod=447'7427)", 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.589097+0000", 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 219.73597149400001, 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015616016999999999, 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589097+0000", 2024-08-29T15:45:40.387 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589097+0000", 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589096+0000", 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.388 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589106+0000", 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7240000000000004e-06 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589122+0000", 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6192e-05 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589129+0000", 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3860000000000001e-06 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.389 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589193+0000", 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3207000000000007e-05 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589218+0000", 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5778999999999998e-05 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604639+0000", 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015420496000000001 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.390 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604713+0000", 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4313999999999994e-05 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119461 9.b e447/420 9:de9ccd52:::benchmark_data_smithi027_341149_object17031:head v 447'7430, mlcod=447'7427)", 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.589711+0000", 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 219.73535758400001, 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015031308, 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.391 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589711+0000", 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589711+0000", 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589710+0000", 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.392 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589717+0000", 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0269999999999998e-06 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589730+0000", 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2801e-05 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.589737+0000", 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4810000000000003e-06 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.393 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.592962+0000", 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032251810000000001 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.593010+0000", 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7922e-05 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604722+0000", 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011712482999999999 2024-08-29T15:45:40.394 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.604742+0000", 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0250000000000001e-05 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:119673 9.b e447/420 9:d3d729fb:::benchmark_data_smithi027_341149_object17243:head v 447'7443, mlcod=447'7441)", 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:00.891670+0000", 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 219.43339883100001, 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015279612999999999, 2024-08-29T15:45:40.395 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891670+0000", 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891670+0000", 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.396 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891668+0000", 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891676+0000", 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6389999999999992e-06 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891692+0000", 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5679e-05 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.397 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891699+0000", 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2450000000000001e-06 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891756+0000", 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6913999999999998e-05 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.891777+0000", 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1030999999999999e-05 2024-08-29T15:45:40.398 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.906904+0000", 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015127108 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:00.906949+0000", 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5419e-05 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.399 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191023 9.b e467/463 9:db7e643b:::benchmark_data_smithi027_341149_object88593:head v 467'11978, mlcod=467'11976)", 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.564639+0000", 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 94.760429751999993, 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016074029, 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564639+0000", 2024-08-29T15:45:40.400 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564639+0000", 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564638+0000", 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.401 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564648+0000", 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.047e-05 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564671+0000", 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2852000000000001e-05 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564679+0000", 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1580000000000005e-06 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.402 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564735+0000", 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5846e-05 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.564763+0000", 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8459e-05 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.580666+0000", 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015902823999999999 2024-08-29T15:45:40.403 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.580713+0000", 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6687000000000002e-05 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:422 10.f e483/480 10:f443d05d:::benchmark_data_smithi027_374610_object421:head v 483'33, mlcod=483'31)", 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:18.761422+0000", 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 21.563646701, 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018122649000000001, 2024-08-29T15:45:40.404 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.761422+0000", 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.761422+0000", 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.405 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.761421+0000", 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.778680+0000", 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.017259192 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.778697+0000", 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6269e-05 2024-08-29T15:45:40.406 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.778703+0000", 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8919999999999997e-06 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.778770+0000", 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7063000000000005e-05 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.778795+0000", 2024-08-29T15:45:40.407 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5755e-05 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779508+0000", 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00071282400000000003 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779545+0000", 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6387000000000002e-05 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.408 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:428 10.0 e483/480 10:07c2b8e0:::benchmark_data_smithi027_374610_object427:head v 483'39, mlcod=483'34)", 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:18.764004+0000", 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 21.561064701999999, 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015841468000000001, 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.764004+0000", 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.409 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.764004+0000", 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.764004+0000", 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779041+0000", 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015037712 2024-08-29T15:45:40.410 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779055+0000", 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3307e-05 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779061+0000", 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9619999999999998e-06 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779224+0000", 2024-08-29T15:45:40.411 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016301400000000001 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779241+0000", 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7068e-05 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779813+0000", 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00057216299999999999 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.412 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:18.779845+0000", 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2539000000000001e-05 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:43857 10.0 e484/480 10:06f10599:::benchmark_data_smithi027_374610_object43856:head v 484'2783, mlcod=484'2781)", 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:38.205499+0000", 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 2.1195696310000001, 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014848452999999999, 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:40.413 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.205499+0000", 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.205499+0000", 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.205498+0000", 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:40.414 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.205532+0000", 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3924999999999998e-05 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.205540+0000", 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0760000000000003e-06 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.205546+0000", 2024-08-29T15:45:40.415 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3170000000000001e-06 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.209325+0000", 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003779536 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.209345+0000", 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0024999999999999e-05 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:40.416 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.220152+0000", 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.010806614000000001 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:38.220347+0000", 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019547600000000001 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:40.417 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:45:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:40.444 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:40.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:40.545 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:40.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:40.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:40.670 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 22 16 47930 47914 136.003 87.9375 0.0096271 0.00734748 2024-08-29T15:45:40.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:40.721 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:40.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:40.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:40.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:40 smithi027 ceph-mon[93921]: pgmap v1722: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:45:40.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:40 smithi027 ceph-mon[98970]: pgmap v1722: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:45:40.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:40.864 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:40.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:40.966 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:40.987 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:40.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:40 smithi171 ceph-mon[90031]: pgmap v1722: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-29T15:45:41.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:41.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:41.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:41.227 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:41.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:41.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:41.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:41.402 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:41.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:41.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:41.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:41.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:41.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:41.611 INFO:teuthology.orchestra.run.smithi027.stderr:set pool 1 pgp_num to 31 2024-08-29T15:45:41.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:41.671 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 23 16 49487 49471 134.315 97.3125 0.00581893 0.00743964 2024-08-29T15:45:41.680 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:41.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:41.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:41.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:41.803 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:41.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:41.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:41.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:41.939 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:41.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:41.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:42.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3069888444' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-29T15:45:42.040 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:42.068 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:42.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:42.072 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3069888444' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-29T15:45:42.072 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3069888444' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-29T15:45:42.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:42.207 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:42.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:42.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:42.349 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:42.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:42.451 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:42.485 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:42.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:42.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:42.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:42.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:42.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:42.671 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 24 16 50987 50971 132.626 93.75 0.00527925 0.00753283 2024-08-29T15:45:42.683 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:45:42.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:42.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:42.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:42.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:42.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:42.883 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:42.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:42.912 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:42.985 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:43.006 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:45:43.006 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[98970]: pgmap v1723: 119 pgs: 119 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-29T15:45:43.006 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3069888444' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-08-29T15:45:43.006 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[98970]: osdmap e485: 8 total, 8 up, 8 in 2024-08-29T15:45:43.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3739241021' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:43.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:43.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[93921]: pgmap v1723: 119 pgs: 119 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-29T15:45:43.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3069888444' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-08-29T15:45:43.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[93921]: osdmap e485: 8 total, 8 up, 8 in 2024-08-29T15:45:43.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3739241021' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:43.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:42 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.008 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:43.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:42 smithi171 ceph-mon[90031]: pgmap v1723: 119 pgs: 119 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-29T15:45:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3069888444' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-08-29T15:45:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:42 smithi171 ceph-mon[90031]: osdmap e485: 8 total, 8 up, 8 in 2024-08-29T15:45:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3739241021' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:42 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:45:43.109 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:43.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:43.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:43.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:43.286 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:43.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:43.387 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:43.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:43.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:43.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:43.661 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:43.671 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 25 16 52174 52158 130.29 74.1875 0.00722848 0.00767011 2024-08-29T15:45:43.696 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:43.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:43.833 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:43.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:43.869 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:45:43.869 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:45:43.869 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:45:43.870 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[93921]: osdmap e486: 8 total, 8 up, 8 in 2024-08-29T15:45:43.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:45:43.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:45:43.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:45:43.870 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:43 smithi027 ceph-mon[98970]: osdmap e486: 8 total, 8 up, 8 in 2024-08-29T15:45:43.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:43.974 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:43.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:43.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:43 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:45:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:43 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:45:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:45:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:43 smithi171 ceph-mon[90031]: osdmap e486: 8 total, 8 up, 8 in 2024-08-29T15:45:44.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:44.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:44.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:44.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:44.255 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:44.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:44.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:44.357 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:44.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:44.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:44.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:44.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:44.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:44.641 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:44.671 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 26 16 53849 53833 129.305 104.688 0.00702831 0.00772872 2024-08-29T15:45:44.671 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:44.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:44.774 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:44.813 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:44.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:44.913 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:44 smithi171 ceph-mon[90031]: pgmap v1726: 119 pgs: 119 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-29T15:45:44.913 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:44.913 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:44.913 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:45:44.913 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:45:44.913 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.913 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:54221 10.9 e486/480 10:91e46143:::benchmark_data_smithi027_374610_object54220:head v 486'3417, mlcod=486'3415)", 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:44.908806+0000", 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0026995700000000001, 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002727148, 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908806+0000", 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:44.914 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908806+0000", 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908805+0000", 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908834+0000", 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8102e-05 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.915 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908846+0000", 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2682e-05 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908852+0000", 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7799999999999997e-06 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908940+0000", 2024-08-29T15:45:44.916 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7738000000000006e-05 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.908968+0000", 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8467e-05 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:54223 10.1 e486/480 10:8f46a963:::benchmark_data_smithi027_374610_object54222:head v 486'3476, mlcod=486'3474)", 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:44.911037+0000", 2024-08-29T15:45:44.917 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.00046869199999999999, 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00056499299999999996, 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911037+0000", 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911037+0000", 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:44.918 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911036+0000", 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911112+0000", 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5545999999999998e-05 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:44.919 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911129+0000", 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6801000000000001e-05 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911135+0000", 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4269999999999996e-06 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911174+0000", 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8979000000000001e-05 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.920 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911192+0000", 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7969e-05 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:54222 10.9 e486/480 10:96c96056:::benchmark_data_smithi027_374610_object54221:head v 486'3418, mlcod=486'3415)", 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:44.911320+0000", 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.000185587, 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00033966500000000001, 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:44.921 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "started", 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911320+0000", 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911320+0000", 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911320+0000", 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911344+0000", 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3876999999999999e-05 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911362+0000", 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8904999999999999e-05 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911368+0000", 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3770000000000004e-06 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911494+0000", 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012645700000000001 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:44.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:44.911549+0000", 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4644000000000001e-05 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 3 2024-08-29T15:45:44.925 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:45:44.925 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:44.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:44.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:44.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:45.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:44 smithi027 ceph-mon[93921]: pgmap v1726: 119 pgs: 119 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-29T15:45:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:45.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:45.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:44 smithi027 ceph-mon[98970]: pgmap v1726: 119 pgs: 119 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-08-29T15:45:45.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:45:45.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:45:45.100 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:45.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:45.202 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:45.239 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:45.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:45.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:45.275 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:45.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:45.386 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:45.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:45.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:45.523 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:45.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:45.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:45.626 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:45.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:45.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:45.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:45.671 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 27 16 55467 55451 128.262 101.125 0.00921948 0.00779105 2024-08-29T15:45:45.713 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:45:45.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:45.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:45.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:45.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:45.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:45.948 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:45.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:46.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:46.069 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:46.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:46.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:46.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:46.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:46.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:46.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:46.367 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:46.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:46.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:46.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:46.500 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:46.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:46.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:46.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:46.651 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:46.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:46.671 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 28 16 57073 57057 127.266 100.375 0.00377066 0.00785207 2024-08-29T15:45:46.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:46.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:46 smithi027 ceph-mon[98970]: pgmap v1727: 119 pgs: 119 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:45:46.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:46 smithi027 ceph-mon[93921]: pgmap v1727: 119 pgs: 119 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:45:46.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:46.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:46.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:46.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:46.924 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:46.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:46.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:46 smithi171 ceph-mon[90031]: pgmap v1727: 119 pgs: 119 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-08-29T15:45:47.026 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:47.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:47.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:47.160 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:47.194 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:47.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:47.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:47.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:45:47.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:47.321 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:45:47.321 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:45:47.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:47.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:47.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:47.363 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:45:47.363 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:45:47.397 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:45:47.397 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:45:47.423 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:45:47.423 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:45:47.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:47.463 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:45:47.463 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:45:47.472 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:47.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:47.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:47.497 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:45:47.497 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:45:47.527 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:45:47.528 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:45:47.567 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:45:47.568 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:45:47.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:47.577 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:47.620 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:47.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:47.672 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 29 16 58361 58345 125.652 80.5 0.0239279 0.0079538 2024-08-29T15:45:47.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:47.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:47.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:47.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:47.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:47.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:47.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:47.948 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/928755491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:47.948 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/928755491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:48.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/928755491' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:48.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:48.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:48.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:48.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:48.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:48.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:48.306 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:48.407 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:48.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:48.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:48.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:48.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:48.672 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 30 16 59936 59920 124.745 98.4375 0.0120978 0.00801117 2024-08-29T15:45:48.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:48.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:48.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:48.824 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:48.829 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:48 smithi027 ceph-mon[98970]: pgmap v1728: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:45:48.830 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:48 smithi027 ceph-mon[93921]: pgmap v1728: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:45:48.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:48.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:48.959 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:48.996 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:48.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:48.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:49.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:48 smithi171 ceph-mon[90031]: pgmap v1728: 119 pgs: 119 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-29T15:45:49.098 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:49.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:49.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:49.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:49.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:49.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:49.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:45:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:45:49] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.51.0" 2024-08-29T15:45:49.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:49.399 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:49.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:49.501 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:49.534 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:49.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:49.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:49.672 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 31 16 61526 61510 123.927 99.375 0.0100097 0.00806328 2024-08-29T15:45:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:49.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:49.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:49.727 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:45:49.727 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:45:49.728 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-08-29T15:45:49.728 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:45:49.728 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:45:49.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-08-29T15:45:49.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:49.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:49.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:49.866 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:49 smithi027 ceph-mon[93921]: pgmap v1729: 119 pgs: 119 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-29T15:45:49.867 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:49 smithi027 ceph-mon[98970]: pgmap v1729: 119 pgs: 119 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-29T15:45:49.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:49.946 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:49.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:50.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:49 smithi171 ceph-mon[90031]: pgmap v1729: 119 pgs: 119 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-29T15:45:50.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:50.078 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:50.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:50.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:50.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:50.220 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:50.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:50.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:50.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:50.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:45:50.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:50.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:50.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:50.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:50.515 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:50.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:50.616 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:50.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:50.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:50.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:50.672 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 32 16 62806 62790 122.554 80 0.00669702 0.00815417 2024-08-29T15:45:50.741 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:50.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:50.781 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:50.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:50.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:50.795 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:45:50.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3306168167' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:45:50.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:50 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:50.876 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3306168167' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:45:50.877 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:50 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:50.883 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:50.912 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:50.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:50.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:51.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:51.047 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:51.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:51.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:51.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:51.186 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:51.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3306168167' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:45:51.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:50 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:45:51.289 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:51.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:51.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:51.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:51.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:51.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:51.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:51.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:51.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:51.621 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:51.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:51.672 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 33 16 64251 64235 121.578 90.3125 0.011794 0.00822007 2024-08-29T15:45:51.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:51.761 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:51.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:51.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:51.899 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:51.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:52.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:52.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:52.037 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:52.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3306168167' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:45:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:51 smithi027 ceph-mon[98970]: osdmap e487: 8 total, 8 up, 8 in 2024-08-29T15:45:52.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:51 smithi027 ceph-mon[98970]: pgmap v1731: 119 pgs: 119 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:45:52.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3306168167' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:45:52.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:51 smithi027 ceph-mon[93921]: osdmap e487: 8 total, 8 up, 8 in 2024-08-29T15:45:52.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:51 smithi027 ceph-mon[93921]: pgmap v1731: 119 pgs: 119 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:45:52.109 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3306168167' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:45:52.109 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:51 smithi171 ceph-mon[90031]: osdmap e487: 8 total, 8 up, 8 in 2024-08-29T15:45:52.109 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:51 smithi171 ceph-mon[90031]: pgmap v1731: 119 pgs: 119 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-29T15:45:52.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:52.179 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:52.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:52.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:52.268 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:52.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:52.291 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:45:52.291 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:45:52.322 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:52.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:52.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:52.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:52.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:52.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:52.570 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:52.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:52.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:52.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:52.672 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 34 16 65556 65540 120.401 81.5625 0.00873835 0.00829942 2024-08-29T15:45:52.710 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:52.755 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:52.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:52.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:52.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:52.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:52.997 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:53.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:53.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:53.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:53.178 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:53.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:53.280 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:53.317 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:53.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:53.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:53.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:53.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:53.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:53.545 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:45:53.546 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:45:53.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:53.591 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:45:53.591 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:45:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:53.610 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:53.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:53.630 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:45:53.631 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:45:53.664 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:45:53.664 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:45:53.672 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 35 16 66922 66906 119.401 85.375 0.0108531 0.00837074 2024-08-29T15:45:53.702 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:45:53.702 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:45:53.711 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:53.737 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:45:53.737 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:45:53.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:53.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:53.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:53.775 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:45:53.776 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:45:53.809 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:45:53.810 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:45:53.848 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:53.878 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:53.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:53.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:53.979 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:54.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:54.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:54.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:54.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:54.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:54.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:45:54.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:54.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:54.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:54 smithi027 ceph-mon[98970]: pgmap v1732: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-29T15:45:54.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:54 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3659912728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:54 smithi027 ceph-mon[93921]: pgmap v1732: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-29T15:45:54.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:54 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3659912728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:54.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:54.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:54.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:54 smithi171 ceph-mon[90031]: pgmap v1732: 119 pgs: 119 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-29T15:45:54.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:54 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3659912728' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:45:54.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:54.598 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:54.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:54.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:54.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 36 16 68231 68215 118.357 81.8125 0.00842226 0.00844426 2024-08-29T15:45:54.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:54.740 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:54.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:54.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:54.762 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:45:54.842 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:54.882 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:54.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:54.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:45:54.984 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:55.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:55.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:55.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:55.147 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:55.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:55.249 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:55.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:55.313 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:45:55.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:55.436 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:45:55.436 INFO:teuthology.orchestra.run.smithi171.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:45:55.437 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:45:55.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:55.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:55.544 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:55.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:55.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 37 16 69437 69421 117.196 75.375 0.00836927 0.00852676 2024-08-29T15:45:55.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:55.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:55.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:55.851 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:55.953 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:55.996 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:55.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:55.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:56.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:56.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:56.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:56.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:56.276 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:56.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:56.378 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:56.415 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:56.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:56.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:56.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:56.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:56 smithi171 ceph-mon[90031]: pgmap v1733: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-29T15:45:56.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:56.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:56.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:45:56.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:56 smithi027 ceph-mon[98970]: pgmap v1733: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-29T15:45:56.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:56 smithi027 ceph-mon[93921]: pgmap v1733: 119 pgs: 119 active+clean; 4.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-29T15:45:56.656 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:56.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 38 16 70886 70870 116.495 90.5625 0.0206521 0.00857887 2024-08-29T15:45:56.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:56.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:56.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:56.834 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:56.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:56.935 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:45:56.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:56.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:56.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:45:57.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:57.107 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:57.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:57.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:57.244 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:57.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:57.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:57.346 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:57.412 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:57.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:57.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:57.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:57.559 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:57.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:57.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:57.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 39 16 72147 72131 115.528 78.8125 0.00741013 0.00865095 2024-08-29T15:45:57.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:57.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:57.800 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:57.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:57.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:57.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:57.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:57.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:58.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:58.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:58.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:58.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:58.213 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:45:58.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:58.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:58.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:45:58.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:58.385 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:58.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:58.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:58.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:58 smithi171 ceph-mon[90031]: pgmap v1734: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-08-29T15:45:58.525 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:58.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:58.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:58 smithi027 ceph-mon[93921]: pgmap v1734: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-08-29T15:45:58.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:58 smithi027 ceph-mon[98970]: pgmap v1734: 119 pgs: 119 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-08-29T15:45:58.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:45:58.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:58.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:58.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:45:58.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:45:58.673424+0000 min lat: 0.00100862 max lat: 0.322213 avg lat: 0.00871002 2024-08-29T15:45:58.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:45:58.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 40 16 73479 73463 114.721 83.25 0.00295566 0.00871002 2024-08-29T15:45:58.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:45:58.802 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:58.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:45:58.904 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:45:58.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:45:58.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:58.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:45:59.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:59.067 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:59.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:59.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:59.205 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:59.205 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:45:59.205 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148094 9.2 e453/440 9:414bc4a8:::benchmark_data_smithi027_341149_object45664:head v 453'9218, mlcod=430'2855)", 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.380994+0000", 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 183.82080578700001, 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033578950000000003, 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.380994+0000", 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.380994+0000", 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.380993+0000", 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.381000+0000", 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0489999999999998e-06 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.381011+0000", 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1219e-05 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.381017+0000", 2024-08-29T15:45:59.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.365e-06 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.394752+0000", 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013734379999999999 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.394778+0000", 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.658e-05 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414517+0000", 2024-08-29T15:45:59.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019738215999999999 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414573+0000", 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6048999999999998e-05 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148100 9.2 e453/440 9:413568bd:::benchmark_data_smithi027_341149_object45670:head v 453'9219, mlcod=430'2855)", 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.387969+0000", 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 183.813830118, 2024-08-29T15:45:59.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036568690000000001, 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387969+0000", 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387969+0000", 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387968+0000", 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387975+0000", 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9589999999999998e-06 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387987+0000", 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1392999999999999e-05 2024-08-29T15:45:59.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387993+0000", 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.888e-06 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.403310+0000", 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015316695999999999 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.403334+0000", 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.355e-05 2024-08-29T15:45:59.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424496+0000", 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021162258 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424538+0000", 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2094000000000001e-05 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148105 9.2 e453/440 9:4d9f105c:::benchmark_data_smithi027_341149_object45675:head v 453'9220, mlcod=430'2855)", 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.390364+0000", 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 183.811435106, 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034200055, 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390364+0000", 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390364+0000", 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390363+0000", 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390371+0000", 2024-08-29T15:45:59.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9680000000000006e-06 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390386+0000", 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5197999999999999e-05 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390394+0000", 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8690000000000004e-06 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.217 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414615+0000", 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024220742 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414638+0000", 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2867e-05 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424543+0000", 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099052010000000006 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424564+0000", 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1467e-05 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148250 9.2 e453/440 9:475ad16a:::benchmark_data_smithi027_341149_object45820:head v 453'9230, mlcod=430'2866)", 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.662960+0000", 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 183.53883979700001, 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035166784, 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662960+0000", 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662960+0000", 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662959+0000", 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662967+0000", 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2239999999999995e-06 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662982+0000", 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5234e-05 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662989+0000", 2024-08-29T15:45:59.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5069999999999997e-06 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.663035+0000", 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6383e-05 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.663059+0000", 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3623999999999999e-05 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.697510+0000", 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034451341000000003 2024-08-29T15:45:59.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.698126+0000", 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00061651999999999996 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:152289 9.8 e454/435 9:1a4c53e5:::benchmark_data_smithi027_341149_object49859:head v 454'9556, mlcod=431'3113)", 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 175.95186539599999, 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041027240999999999, 2024-08-29T15:45:59.223 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249937+0000", 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2399999999999999e-06 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249942+0000", 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9949999999999996e-06 2024-08-29T15:45:59.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249946+0000", 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.997e-06 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.282663+0000", 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032717165999999999 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.282691+0000", 2024-08-29T15:45:59.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7729000000000002e-05 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.290934+0000", 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0082430369999999999 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.290961+0000", 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7282000000000001e-05 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:167613 9.a e460/424 9:5041f107:::benchmark_data_smithi027_341149_object65183:head v 460'10529, mlcod=460'10527)", 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:31.020831+0000", 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 148.18096881299999, 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034174152999999999, 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020831+0000", 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020831+0000", 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020830+0000", 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020836+0000", 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4980000000000003e-06 2024-08-29T15:45:59.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020845+0000", 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5320000000000006e-06 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020850+0000", 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.451e-06 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.022801+0000", 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019502860000000001 2024-08-29T15:45:59.230 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.022821+0000", 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0160999999999999e-05 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.054929+0000", 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032108593999999997 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.055005+0000", 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5332000000000003e-05 2024-08-29T15:45:59.231 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:172219 9.1 e461/420 9:85c0e43c:::benchmark_data_smithi027_341149_object69789:head v 461'10776, mlcod=461'10774)", 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:38.225280+0000", 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 140.97651939299999, 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039086323999999999, 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.232 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225280+0000", 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225280+0000", 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.233 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:59.234 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225279+0000", 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225287+0000", 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8480000000000001e-06 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225299+0000", 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1595e-05 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.235 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225306+0000", 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1949999999999997e-06 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233238+0000", 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079318259999999995 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233257+0000", 2024-08-29T15:45:59.236 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9757000000000001e-05 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264299+0000", 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031041632999999999 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264366+0000", 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7428000000000006e-05 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.237 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:172221 9.1 e461/420 9:8160b39d:::benchmark_data_smithi027_341149_object69791:head v 461'10777, mlcod=461'10774)", 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 140.97223699200001, 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034854593000000003, 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.238 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229570+0000", 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9370000000000004e-06 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.239 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233292+0000", 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003722347 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233300+0000", 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8529999999999992e-06 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.236669+0000", 2024-08-29T15:45:59.240 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033689480000000001 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.236687+0000", 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8088e-05 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264369+0000", 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027682521000000002 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.241 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264417+0000", 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7741000000000003e-05 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:172224 9.1 e461/420 9:8d0cf34f:::benchmark_data_smithi027_341149_object69794:head v 461'10778, mlcod=461'10774)", 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:38.233364+0000", 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 140.96843561899999, 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031082248999999999, 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.242 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233364+0000", 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233364+0000", 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233364+0000", 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.243 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233368+0000", 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.155e-06 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233373+0000", 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2920000000000003e-06 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233377+0000", 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6500000000000002e-06 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.245425+0000", 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012048052 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.245451+0000", 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6729000000000001e-05 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264421+0000", 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.01896989 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264446+0000", 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.4769999999999998e-05 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:185992 9.2 e467/440 9:47f34658:::benchmark_data_smithi027_341149_object83562:head v 467'11563, mlcod=432'4648)", 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 120.32721436600001, 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032982351, 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.246 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874590+0000", 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4419999999999997e-06 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874599+0000", 2024-08-29T15:45:59.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8289999999999997e-06 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874603+0000", 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4039999999999997e-06 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.880284+0000", 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056811320000000002 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.249 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.880312+0000", 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7317999999999998e-05 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.907528+0000", 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027216081 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.907567+0000", 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9746999999999997e-05 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.250 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:186001 9.a e467/424 9:5329eecb:::benchmark_data_smithi027_341149_object83571:head v 467'11686, mlcod=467'11683)", 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:58.884314+0000", 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 120.317485643, 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030772368000000001, 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.884314+0000", 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.884314+0000", 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.884313+0000", 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.884321+0000", 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0139999999999998e-06 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.884336+0000", 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5150000000000001e-05 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.884342+0000", 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2360000000000001e-06 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.894331+0000", 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099888649999999995 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.894351+0000", 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0237999999999998e-05 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.915020+0000", 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.020668945000000001 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.915086+0000", 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5857000000000003e-05 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:186823 9.8 e467/435 9:1424c12e:::benchmark_data_smithi027_341149_object84393:head v 467'11728, mlcod=432'5285)", 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 119.03718618000001, 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.037339148000000003, 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.255 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:45:59.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164619+0000", 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7290000000000003e-06 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164631+0000", 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1591e-05 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.257 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164637+0000", 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8839999999999999e-06 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.175974+0000", 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011337017 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.175994+0000", 2024-08-29T15:45:59.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9862000000000001e-05 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.201931+0000", 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025936858 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.201952+0000", 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1928000000000001e-05 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.259 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:186850 9.8 e467/435 9:1aed1321:::benchmark_data_smithi027_341149_object84420:head v 467'11729, mlcod=432'5285)", 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:00.170774+0000", 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 119.031025302, 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031199290000000001, 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.170774+0000", 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.260 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.170774+0000", 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.170773+0000", 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.261 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.170780+0000", 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2280000000000003e-06 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.170788+0000", 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3960000000000006e-06 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.262 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.170793+0000", 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7890000000000002e-06 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.179520+0000", 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087273730000000001 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.179548+0000", 2024-08-29T15:45:59.263 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7535e-05 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.201955+0000", 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.022407660999999999 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.201973+0000", 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8202999999999998e-05 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.264 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:189430 9.8 e467/435 9:19f93844:::benchmark_data_smithi027_341149_object87000:head v 467'11889, mlcod=432'5447)", 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:03.621554+0000", 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 115.58024587600001, 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.03492348, 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.265 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621554+0000", 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621554+0000", 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621553+0000", 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.266 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621558+0000", 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1290000000000004e-06 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621564+0000", 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6899999999999997e-06 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621568+0000", 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0380000000000001e-06 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.267 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.630821+0000", 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0092526529999999996 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.630843+0000", 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2084000000000001e-05 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.656440+0000", 2024-08-29T15:45:59.268 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.02559707 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.656477+0000", 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7143e-05 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:190138 9.1 e467/420 9:8604df0f:::benchmark_data_smithi027_341149_object87708:head v 467'11907, mlcod=467'11904)", 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:04.424851+0000", 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 114.77694885, 2024-08-29T15:45:59.269 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034214171000000002, 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424851+0000", 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424851+0000", 2024-08-29T15:45:59.270 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424850+0000", 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424858+0000", 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0559999999999999e-06 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.271 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424863+0000", 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2469999999999999e-06 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424866+0000", 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6839999999999998e-06 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.428269+0000", 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034026820000000002 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.272 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.428288+0000", 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8555000000000001e-05 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459029+0000", 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030741764000000001 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459065+0000", 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5340999999999999e-05 2024-08-29T15:45:59.273 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:190139 9.1 e467/420 9:8027c676:::benchmark_data_smithi027_341149_object87709:head v 467'11908, mlcod=467'11904)", 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:04.427334+0000", 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 114.774465706, 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031759969999999998, 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427334+0000", 2024-08-29T15:45:59.274 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427334+0000", 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427333+0000", 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427340+0000", 2024-08-29T15:45:59.275 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4400000000000002e-06 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427354+0000", 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.455e-05 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427360+0000", 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1600000000000003e-06 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.434094+0000", 2024-08-29T15:45:59.276 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067339219999999998 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.434112+0000", 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7737e-05 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.277 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459070+0000", 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024958270000000001 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459094+0000", 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3598999999999999e-05 2024-08-29T15:45:59.278 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191156 9.2 e467/440 9:408459e1:::benchmark_data_smithi027_341149_object88726:head v 467'11903, mlcod=432'4648)", 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.723860+0000", 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 113.477939714, 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035658529000000001, 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.279 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723860+0000", 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723860+0000", 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723859+0000", 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.280 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723867+0000", 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6980000000000004e-06 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723880+0000", 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2942000000000001e-05 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723895+0000", 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5211000000000001e-05 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.281 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.727039+0000", 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031432750000000001 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.727060+0000", 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1707000000000001e-05 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759474+0000", 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032413497999999999 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.282 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759518+0000", 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4399e-05 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191159 9.2 e467/440 9:4454a5d8:::benchmark_data_smithi027_341149_object88729:head v 467'11904, mlcod=432'4648)", 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.724627+0000", 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 113.477172736, 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034913665000000003, 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.283 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724627+0000", 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724627+0000", 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.284 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724626+0000", 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724632+0000", 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1290000000000001e-06 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724645+0000", 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2469999999999999e-05 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.285 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724651+0000", 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9669999999999998e-06 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.732733+0000", 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0080824529999999999 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.732760+0000", 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7226999999999999e-05 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.286 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759522+0000", 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026761686 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759540+0000", 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8464000000000001e-05 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191166 9.2 e467/440 9:4270f1af:::benchmark_data_smithi027_341149_object88736:head v 467'11905, mlcod=432'4648)", 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.729757+0000", 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 113.472042909, 2024-08-29T15:45:59.287 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035565313000000001, 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729757+0000", 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729757+0000", 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.288 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729756+0000", 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729764+0000", 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1720000000000007e-06 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729779+0000", 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4827999999999999e-05 2024-08-29T15:45:59.289 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729785+0000", 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4219999999999997e-06 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.738213+0000", 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.008427604 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.738232+0000", 2024-08-29T15:45:59.290 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8895000000000001e-05 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.765289+0000", 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.02705747 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.765322+0000", 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2833000000000002e-05 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.291 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191169 9.2 e467/440 9:40d1e6e4:::benchmark_data_smithi027_341149_object88739:head v 467'11906, mlcod=432'4648)", 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.734206+0000", 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 113.46759374600001, 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031170539000000001, 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.734206+0000", 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.292 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.734206+0000", 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.734205+0000", 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.734212+0000", 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7719999999999997e-06 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.293 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.734221+0000", 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3370000000000006e-06 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.734227+0000", 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6930000000000004e-06 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.746697+0000", 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012470645000000001 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.294 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.746730+0000", 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2511000000000001e-05 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.765355+0000", 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018625472000000001 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.765376+0000", 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0801999999999998e-05 2024-08-29T15:45:59.295 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.296 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.296 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.296 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:45:59.296 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:45:59.296 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:45:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:45:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:45:59] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.51.0" 2024-08-29T15:45:59.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:59.419 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:59.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:59.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:59.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:59.544 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:59.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:45:59.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:45:59.673 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 41 16 74669 74653 113.738 74.375 0.00798339 0.00878734 2024-08-29T15:45:59.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:59.683 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:59.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:45:59.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:45:59.815 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:59.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:45:59.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:45:59.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:45:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:45:59.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:45:59 smithi171 ceph-mon[90031]: pgmap v1735: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-29T15:46:00.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:00.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:45:59 smithi027 ceph-mon[93921]: pgmap v1735: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-29T15:46:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:00.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:45:59 smithi027 ceph-mon[98970]: pgmap v1735: 119 pgs: 119 active+clean; 4.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-08-29T15:46:00.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:00.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:00.194 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:00.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:00.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:00.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:00.332 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:00.368 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:00.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:00.470 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:00.505 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:00.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:00.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:00.645 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:00.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:00.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:00.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 42 16 75992 75976 112.998 82.6875 0.00612715 0.0088444 2024-08-29T15:46:00.747 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:00.786 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:00.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:00.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:46:00.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:00.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:00.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:00.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:01.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:01.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:01.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:01.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:01.180 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:01.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:01.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:01.239 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:46:01.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:01.369 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:01.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:01.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:01.504 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:01.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:01.606 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:01.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:01.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:01.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:01.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 43 16 77307 77291 112.282 82.1875 0.0113438 0.00890198 2024-08-29T15:46:01.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:01.779 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:01.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:01.847 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:01.881 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:01.912 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:01.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:02.011 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:46:02.011 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:46:02.012 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-08-29T15:46:02.012 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:46:02.012 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:46:02.012 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-08-29T15:46:02.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:02.060 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:02.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:02.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:02.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:02.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:02.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:02.350 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:02.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:02.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:02.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:02 smithi027 ceph-mon[93921]: pgmap v1736: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:46:02.385 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:02 smithi027 ceph-mon[98970]: pgmap v1736: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:46:02.444 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:02 smithi171 ceph-mon[90031]: pgmap v1736: 119 pgs: 119 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-29T15:46:02.452 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:02.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:02.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:02.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:02.594 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:02.654 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:02.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:02.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:02.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 44 16 78517 78501 111.449 75.625 0.0079989 0.00896822 2024-08-29T15:46:02.756 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:02.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:02.801 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:02.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:02.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:02.945 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:02.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:03.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:03.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:03.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:03.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:03.184 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:03 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2879252841' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:03.184 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:03 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:03.184 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:46:03.185 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:03 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2879252841' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:03.185 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:03 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:03.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:03.193 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:46:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:03.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:03.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:03.243 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:46:03.243 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:46:03.243 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.243 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:79225 10.3 10:c99d2e8f:::benchmark_data_smithi027_374610_object79224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2024-08-29T15:46:03.243 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:03.231338+0000", 2024-08-29T15:46:03.243 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.0096087550000000001, 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0096417810000000003, 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 79225 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231338+0000", 2024-08-29T15:46:03.244 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231338+0000", 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231335+0000", 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.245 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231399+0000", 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3855000000000004e-05 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231416+0000", 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7108999999999998e-05 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231425+0000", 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3979999999999999e-06 2024-08-29T15:46:03.246 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231477+0000", 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1938000000000003e-05 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231633+0000", 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015629500000000001 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,4", 2024-08-29T15:46:03.247 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.231793+0000", 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016020100000000001 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.235744+0000", 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039510389999999999 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.240726+0000", 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049815110000000001 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:03.248 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:79228 10.6 10:66b58ca3:::benchmark_data_smithi027_374610_object79227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:03.233446+0000", 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.007501349, 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0076241809999999998, 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 79228 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.249 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233446+0000", 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233446+0000", 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233444+0000", 2024-08-29T15:46:03.250 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233512+0000", 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7725000000000006e-05 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233525+0000", 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3256e-05 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233533+0000", 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2779999999999996e-06 2024-08-29T15:46:03.251 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233583+0000", 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.007e-05 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233754+0000", 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000171297 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.233905+0000", 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000150612 2024-08-29T15:46:03.252 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.237879+0000", 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039742019999999996 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.240739+0000", 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.002859753 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.253 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:79232 10.a 10:5b9407b1:::benchmark_data_smithi027_374610_object79231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:03.239731+0000", 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 0.001215619, 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0014192040000000001, 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 79232 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.239731+0000", 2024-08-29T15:46:03.254 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.239731+0000", 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.239729+0000", 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.255 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.239797+0000", 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7293999999999996e-05 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.239811+0000", 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4681000000000001e-05 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:03.256 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.239819+0000", 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8029999999999997e-06 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.239913+0000", 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3745000000000005e-05 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.257 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.240041+0000", 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012806800000000001 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:03.240172+0000", 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013064500000000001 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:03.258 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:03.259 INFO:teuthology.orchestra.run.smithi171.stdout: ], 2024-08-29T15:46:03.259 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 3 2024-08-29T15:46:03.259 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:46:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:03 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2879252841' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:03.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:03 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:03.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:03.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:03.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:03.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:03.484 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:03.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:03.516 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:03.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:03.617 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:03.652 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:03.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:03.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:03.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 45 16 79813 79797 110.772 81 0.011383 0.00902302 2024-08-29T15:46:03.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:03.795 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:03.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:03.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:03.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:03.944 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:03.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:04.046 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:04.082 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:04.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:04.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:04.184 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:04.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:04.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:04.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[93921]: pgmap v1737: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-29T15:46:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:04.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[93921]: osdmap e488: 8 total, 8 up, 8 in 2024-08-29T15:46:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[98970]: pgmap v1737: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-29T15:46:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:04.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:04 smithi027 ceph-mon[98970]: osdmap e488: 8 total, 8 up, 8 in 2024-08-29T15:46:04.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:04.354 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:04.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:04.444 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:04 smithi171 ceph-mon[90031]: pgmap v1737: 119 pgs: 119 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-29T15:46:04.444 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:04 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:04.444 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:04 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:04.444 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:04 smithi171 ceph-mon[90031]: osdmap e488: 8 total, 8 up, 8 in 2024-08-29T15:46:04.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:04.506 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:04.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:04.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:04.647 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:04.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:04.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:04.661 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:46:04.661 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:46:04.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 46 16 81082 81066 110.088 79.3125 0.00623615 0.00907831 2024-08-29T15:46:04.696 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:46:04.696 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:46:04.735 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:46:04.735 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:46:04.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:04.767 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:46:04.767 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:46:04.791 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:04.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:04.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:04.803 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:46:04.803 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:46:04.839 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:46:04.839 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:46:04.878 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:46:04.878 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:46:04.893 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:04.915 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:46:04.916 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:46:04.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:04.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:05.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:05.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:05.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:05.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:05.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:05.203 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:05.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:05.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:05.340 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:05.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:05.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:05.352 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:46:05.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:05.473 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/11839359' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:05.482 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:05.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:05.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/11839359' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:05.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/11839359' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:05.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:05.620 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:05.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:05.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:05.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 47 16 82338 82322 109.416 78.5 0.00562957 0.00913338 2024-08-29T15:46:05.723 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:05.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:05.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:05.855 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:05.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:05.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:05.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:05.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:06.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:06.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:06.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:06.164 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:06.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:06.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:06.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:06.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:06.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:06.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:06 smithi027 ceph-mon[98970]: pgmap v1739: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:46:06.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:06 smithi027 ceph-mon[93921]: pgmap v1739: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:46:06.369 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:46:06.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:06.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:06.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:06.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:06 smithi171 ceph-mon[90031]: pgmap v1739: 119 pgs: 119 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-29T15:46:06.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:06.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:06.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:06.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 48 16 83403 83387 108.524 66.5625 0.0117572 0.00921005 2024-08-29T15:46:06.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:06.740 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:06.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:06.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:06.750 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:06.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:06.884 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:06.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:06.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:06.986 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:07.025 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:07.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:07.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:07.169 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:07.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:07.271 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:07.308 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:07.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:07.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:07.554 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:07.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:07.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:07.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 49 16 84502 84486 107.711 68.6875 0.0125757 0.00927873 2024-08-29T15:46:07.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:07.714 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:07.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:46:07.751 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:46:07.751 INFO:teuthology.orchestra.run.smithi171.stdout: "success": "" 2024-08-29T15:46:07.752 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:46:07.816 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:07.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:07.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:07.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:07.960 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:07.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:08.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:08.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:08.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:08.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:08.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:08.229 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:08.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:08.312 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:08.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:08.362 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:08.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:08.384 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:08 smithi171 ceph-mon[90031]: pgmap v1740: 119 pgs: 119 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-08-29T15:46:08.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:08.495 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:08.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:08.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:08.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:08 smithi027 ceph-mon[98970]: pgmap v1740: 119 pgs: 119 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-08-29T15:46:08.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:08 smithi027 ceph-mon[93921]: pgmap v1740: 119 pgs: 119 active+clean; 5.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-08-29T15:46:08.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:08.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:08.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:08.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:08.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 50 16 85514 85498 106.822 63.25 0.0166433 0.00935625 2024-08-29T15:46:08.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:08.742 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:46:08.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:08.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:08.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:08.943 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:08.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:08.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:09.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:09.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:09.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:09.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:09.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:09.223 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:09.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:09.246 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:46:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:46:09] "GET /metrics HTTP/1.1" 200 36620 "" "Prometheus/2.51.0" 2024-08-29T15:46:09.325 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:09.369 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:09.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:09.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:09.505 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:09.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:09.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:09.638 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:09.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:09.674 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 51 16 86658 86642 106.129 71.5 0.00677399 0.00941745 2024-08-29T15:46:09.739 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:09.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:09.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:09.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:09.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:09.919 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:09.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:09.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:10.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:10.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:10.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:10.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:10.208 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:10 smithi171 ceph-mon[90031]: pgmap v1741: 119 pgs: 119 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:46:10.209 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2207402800' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:46:10.210 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:46:10.211 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:10.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:10.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:10.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:10.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:10.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:10.450 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:10.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:10 smithi027 ceph-mon[98970]: pgmap v1741: 119 pgs: 119 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:46:10.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2207402800' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:46:10.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:10 smithi027 ceph-mon[93921]: pgmap v1741: 119 pgs: 119 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-29T15:46:10.451 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2207402800' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:46:10.484 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:10.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:10.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:10.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:10.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:10.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:10.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 52 16 87795 87779 105.455 71.0625 0.0104466 0.00947737 2024-08-29T15:46:10.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:10.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:10.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:10.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:10.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:10.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:10.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:10.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:11.024 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:11.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:11.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:11.150 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:46:11.150 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:46:11.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:11.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:11.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:11 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2207402800' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:46:11.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:11 smithi171 ceph-mon[90031]: osdmap e489: 8 total, 8 up, 8 in 2024-08-29T15:46:11.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:11.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:11.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:11.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:11.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:11.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:11.458 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:11 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:11.458 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2207402800' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:46:11.458 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:11 smithi027 ceph-mon[93921]: osdmap e489: 8 total, 8 up, 8 in 2024-08-29T15:46:11.459 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:11 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:11.459 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2207402800' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:46:11.459 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:11 smithi027 ceph-mon[98970]: osdmap e489: 8 total, 8 up, 8 in 2024-08-29T15:46:11.536 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:11.575 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:11.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:11.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:11.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 53 16 88939 88923 104.815 71.5 0.0148598 0.00953524 2024-08-29T15:46:11.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:11.711 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:11.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:11.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:11.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:46:11.858 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:46:11.858 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:46:11.862 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:11.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:11.894 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:46:11.894 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:46:11.926 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:46:11.926 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:46:11.961 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:46:11.961 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:46:11.964 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:11.998 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:46:11.998 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:46:12.006 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:12.033 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:46:12.033 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:46:12.067 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:46:12.067 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:46:12.103 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:46:12.103 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:46:12.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.153 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:12.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:12.197 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:46:12.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:12.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:12.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:12 smithi027 ceph-mon[98970]: pgmap v1743: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-29T15:46:12.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/732482842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:12 smithi027 ceph-mon[93921]: pgmap v1743: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-29T15:46:12.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/732482842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:12.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:12.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:12.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:12.513 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:12 smithi171 ceph-mon[90031]: pgmap v1743: 119 pgs: 119 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-08-29T15:46:12.514 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/732482842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:12.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:12.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:12.671 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:12.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 54 16 90132 90116 104.254 74.5625 0.0164376 0.00958646 2024-08-29T15:46:12.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:12.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:12.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:12.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:12.950 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:12.988 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:12.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:12.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:13.090 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:13.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:13.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:13.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:13.267 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:13.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:13.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:13.369 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:13.408 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:13.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:13.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:46:13.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:13.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:13.542 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:13.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:13.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:13.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 55 16 91363 91347 103.758 76.9375 0.00798664 0.00963149 2024-08-29T15:46:13.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:13.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:13.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:13.773 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:46:13.773 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:46:13.773 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-08-29T15:46:13.773 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-29T15:46:13.773 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-29T15:46:13.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-08-29T15:46:13.780 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:13.808 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:13.811 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:13.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:13.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:13.965 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:13.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:13.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:14.068 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:14.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:14.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:14.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:14.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:14.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:14.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:14.340 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:14.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:14.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:14.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:14.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:14 smithi171 ceph-mon[90031]: pgmap v1744: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-29T15:46:14.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:14.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:14.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:14 smithi027 ceph-mon[98970]: pgmap v1744: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-29T15:46:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:14 smithi027 ceph-mon[93921]: pgmap v1744: 119 pgs: 119 active+clean; 5.4 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-29T15:46:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:14.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:14.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:14.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:14.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 56 16 92368 92352 103.027 62.8125 0.0173457 0.00970109 2024-08-29T15:46:14.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:14.805 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:14.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:14.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:46:14.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:14.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:14.959 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:14.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:15.060 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:15.089 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:185367 8.3 8:cd5481a4:::benchmark_data_smithi027_306999_object58588:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e373)", 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:37:36.370533+0000", 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 518.71518004500001, 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8547635389999999, 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.090 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 185367 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.370533+0000", 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.091 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.370533+0000", 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.370532+0000", 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.372144+0000", 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001611152 2024-08-29T15:46:15.092 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.372157+0000", 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3047999999999999e-05 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.372164+0000", 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7409999999999992e-06 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.093 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.373826+0000", 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001661299 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op must wait for map", 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.373833+0000", 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.002e-06 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.380905+0000", 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070721150000000003 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.094 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:36.380915+0000", 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0475000000000001e-05 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:37.085614+0000", 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.70469882399999995 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:37.085629+0000", 2024-08-29T15:46:15.095 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4654999999999999e-05 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.222484+0000", 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1368552919999999 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:37:38.225297+0000", 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0028127019999999998 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.096 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210349 8.3 8:c3141fcc:::benchmark_data_smithi027_306999_object83570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.047770+0000", 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 485.03794371200001, 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.025306083, 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210349 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.097 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047770+0000", 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047770+0000", 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047769+0000", 2024-08-29T15:46:15.098 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047771+0000", 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7129999999999999e-06 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047774+0000", 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.281e-06 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.099 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047777+0000", 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5770000000000001e-06 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047803+0000", 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6010999999999999e-05 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.047817+0000", 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4493e-05 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.100 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.060774+0000", 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.012957026 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.060914+0000", 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000139209 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.061041+0000", 2024-08-29T15:46:15.101 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000127868 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.064982+0000", 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0039400709999999999 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.068568+0000", 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0035868390000000001 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.102 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.072987+0000", 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0044185099999999996 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.073039+0000", 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2249000000000001e-05 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.073076+0000", 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6508999999999999e-05 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.103 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.104 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210381 8.3 8:cd9a4a2c:::benchmark_data_smithi027_306999_object83602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.104 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.080751+0000", 2024-08-29T15:46:15.104 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 485.00496207800001, 2024-08-29T15:46:15.104 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99721405500000004, 2024-08-29T15:46:15.104 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210381 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080751+0000", 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.105 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080751+0000", 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080750+0000", 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080757+0000", 2024-08-29T15:46:15.106 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8589999999999999e-06 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080767+0000", 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0681e-05 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080774+0000", 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8009999999999998e-06 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080818+0000", 2024-08-29T15:46:15.107 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4070999999999999e-05 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.080826+0000", 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7300000000000005e-06 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.065021+0000", 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98419544699999995 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.108 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.065138+0000", 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011656500000000001 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.065251+0000", 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000112676 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.073113+0000", 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078620919999999993 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.109 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.077803+0000", 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046899000000000003 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.077863+0000", 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0847999999999997e-05 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.077926+0000", 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2089999999999997e-05 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.110 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.077965+0000", 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9711e-05 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210384 8.3 8:cb22450b:::benchmark_data_smithi027_306999_object83605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.082924+0000", 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 485.00278914, 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99520572399999996, 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.111 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210384 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082924+0000", 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082924+0000", 2024-08-29T15:46:15.112 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082923+0000", 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082929+0000", 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5110000000000003e-06 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082940+0000", 2024-08-29T15:46:15.113 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1518999999999999e-05 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082946+0000", 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1970000000000001e-06 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082988+0000", 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1072999999999998e-05 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.082994+0000", 2024-08-29T15:46:15.114 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8170000000000001e-06 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.068624+0000", 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.98563009099999999 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.068745+0000", 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000120994 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.068863+0000", 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117817 2024-08-29T15:46:15.115 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.073164+0000", 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.004300679 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.077976+0000", 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0048116119999999998 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.078057+0000", 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1644000000000006e-05 2024-08-29T15:46:15.116 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.078096+0000", 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9196000000000002e-05 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.078130+0000", 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3479999999999998e-05 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.117 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210406 8.3 8:ceef6976:::benchmark_data_smithi027_306999_object83627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.113028+0000", 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 484.97268492199998, 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97733929399999997, 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210406 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.118 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113028+0000", 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113028+0000", 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113027+0000", 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:46:15.119 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113033+0000", 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8119999999999996e-06 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113046+0000", 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3478999999999999e-05 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.120 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113053+0000", 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1099999999999999e-06 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113088+0000", 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5896999999999999e-05 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113094+0000", 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7400000000000001e-06 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.121 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.073340+0000", 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96024593000000003 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.073455+0000", 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000114647 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.073572+0000", 2024-08-29T15:46:15.122 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117537 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.078140+0000", 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045680210000000002 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.081826+0000", 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0036855659999999999 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090282+0000", 2024-08-29T15:46:15.123 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0084559149999999996 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090328+0000", 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6218999999999999e-05 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090368+0000", 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9666000000000003e-05 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.124 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210408 8.3 8:c0225c48:::benchmark_data_smithi027_306999_object83629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.113322+0000", 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 484.972390797, 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.99136295399999996, 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210408 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.125 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113322+0000", 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113322+0000", 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.126 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113322+0000", 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113327+0000", 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.31e-06 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113340+0000", 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3523000000000001e-05 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.127 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113348+0000", 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3200000000000002e-06 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113384+0000", 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.591e-05 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.113391+0000", 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5669999999999997e-06 2024-08-29T15:46:15.128 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.078165+0000", 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.96477332199999999 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.078283+0000", 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000118273 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.129 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.078352+0000", 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8702999999999996e-05 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090403+0000", 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.012051287000000001 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104543+0000", 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014139999 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.130 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104600+0000", 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6796000000000003e-05 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104651+0000", 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1687999999999997e-05 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104685+0000", 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4124000000000001e-05 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.131 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210419 8.3 8:c601530e:::benchmark_data_smithi027_306999_object83640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.128392+0000", 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 484.95732096199998, 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97650066800000002, 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.132 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210419 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128392+0000", 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128392+0000", 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.133 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128391+0000", 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128396+0000", 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9300000000000002e-06 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128409+0000", 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2328e-05 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.134 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128415+0000", 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6800000000000004e-06 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128468+0000", 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3047000000000002e-05 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.128476+0000", 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.858e-06 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.135 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.081854+0000", 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95337771999999998 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.081966+0000", 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000111798 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.082046+0000", 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0065e-05 2024-08-29T15:46:15.136 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090432+0000", 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0083866070000000008 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104548+0000", 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.014115213 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104815+0000", 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00026771999999999999 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.137 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104859+0000", 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3148000000000001e-05 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104893+0000", 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4493000000000003e-05 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210422 8.3 8:c26cf211:::benchmark_data_smithi027_306999_object83643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.133166+0000", 2024-08-29T15:46:15.138 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 484.95254696699999, 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.97190030400000005, 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210422 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133166+0000", 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.139 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133166+0000", 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133165+0000", 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133171+0000", 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5269999999999998e-06 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.140 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133182+0000", 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1617000000000001e-05 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133190+0000", 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4610000000000002e-06 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133227+0000", 2024-08-29T15:46:15.141 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7444000000000003e-05 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.133235+0000", 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.498e-06 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.085702+0000", 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95246758099999995 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.142 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.085801+0000", 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8980000000000004e-05 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.085894+0000", 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2967999999999995e-05 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104558+0000", 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018664258999999999 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.143 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104780+0000", 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000221673 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104957+0000", 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00017705100000000001 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105034+0000", 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7263999999999998e-05 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.144 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105067+0000", 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2157999999999999e-05 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.145 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:15.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:15.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:15.147 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210430 8.3 8:c3b9e612:::benchmark_data_smithi027_306999_object83651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.147 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.149106+0000", 2024-08-29T15:46:15.147 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 484.93660736200002, 2024-08-29T15:46:15.147 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.956144879, 2024-08-29T15:46:15.147 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.147 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210430 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149106+0000", 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.148 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149106+0000", 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149105+0000", 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149111+0000", 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7180000000000001e-06 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.149 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149118+0000", 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2790000000000001e-06 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149122+0000", 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4209999999999997e-06 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149170+0000", 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8096999999999997e-05 2024-08-29T15:46:15.150 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.149177+0000", 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.46e-06 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090452+0000", 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94127539699999996 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.151 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090555+0000", 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010264600000000001 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.090654+0000", 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9079999999999993e-05 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104564+0000", 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013909978999999999 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.152 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104930+0000", 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00036608399999999997 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105181+0000", 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000250788 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105219+0000", 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8624999999999997e-05 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.153 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105251+0000", 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1331999999999997e-05 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210433 8.3 8:c9a52173:::benchmark_data_smithi027_306999_object83654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.154106+0000", 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 484.93160705899999, 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95123712599999999, 2024-08-29T15:46:15.154 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210433 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154106+0000", 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.155 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154106+0000", 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154105+0000", 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.156 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154110+0000", 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8930000000000004e-06 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154123+0000", 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3055e-05 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154130+0000", 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7859999999999999e-06 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.157 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154162+0000", 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1735e-05 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.154168+0000", 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9240000000000004e-06 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.094933+0000", 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94076504800000005 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.158 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.095050+0000", 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117143 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.095164+0000", 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000114784 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.104699+0000", 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0095340310000000001 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.159 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105140+0000", 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00044099900000000002 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105284+0000", 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014467000000000001 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105315+0000", 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0877000000000003e-05 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.160 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105343+0000", 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8291999999999999e-05 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46475.0:210451 8.3 8:c2646687:::benchmark_data_smithi027_306999_object83672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:38:10.234679+0000", 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 484.85103459999999, 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.88657108200000001, 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.161 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46475", 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/1260191779", 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 210451 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234679+0000", 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234679+0000", 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.162 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234678+0000", 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234683+0000", 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.293e-06 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234694+0000", 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0936e-05 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.163 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234701+0000", 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1459999999999999e-06 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234754+0000", 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2743e-05 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:10.234762+0000", 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6129999999999998e-06 2024-08-29T15:46:15.164 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.099715+0000", 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86495381699999996 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.099811+0000", 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5332000000000001e-05 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.165 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.099906+0000", 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.5810999999999995e-05 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.105150+0000", 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0052433480000000001 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.114412+0000", 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0092617740000000004 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.166 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.121140+0000", 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067287249999999996 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.121212+0000", 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.2146999999999999e-05 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:38:11.121250+0000", 2024-08-29T15:46:15.167 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7319e-05 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129360 9.d 9:bde52ebc:::benchmark_data_smithi027_341149_object26930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.272298+0000", 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.813415789, 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95432824800000005, 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.168 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129360 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272298+0000", 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272298+0000", 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.169 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272296+0000", 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272302+0000", 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7180000000000001e-06 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272313+0000", 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0728e-05 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.170 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272318+0000", 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9239999999999998e-06 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272365+0000", 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7036000000000002e-05 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272376+0000", 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1297999999999999e-05 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.171 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.214809+0000", 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94243309600000003 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.214927+0000", 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117853 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.215003+0000", 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6551000000000006e-05 2024-08-29T15:46:15.172 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219227+0000", 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042237120000000001 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222490+0000", 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003262788 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226533+0000", 2024-08-29T15:46:15.173 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040426460000000004 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226582+0000", 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9709000000000001e-05 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226626+0000", 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3510000000000002e-05 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.174 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129372 9.d 9:b04251a7:::benchmark_data_smithi027_341149_object26942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.287652+0000", 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.79806125900001, 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94708441799999998, 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129372 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.175 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287652+0000", 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287652+0000", 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287651+0000", 2024-08-29T15:46:15.176 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287658+0000", 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0690000000000004e-06 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287671+0000", 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2781e-05 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.177 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287677+0000", 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9500000000000004e-06 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287722+0000", 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4787999999999999e-05 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287730+0000", 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0430000000000008e-06 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.178 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219285+0000", 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93155439900000003 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219422+0000", 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013689600000000001 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219511+0000", 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9679000000000007e-05 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.179 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226757+0000", 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072459239999999999 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226813+0000", 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5693999999999999e-05 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234663+0000", 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078496350000000006 2024-08-29T15:46:15.180 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234694+0000", 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1883e-05 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234736+0000", 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2015999999999999e-05 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.181 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129383 9.d 9:b062f382:::benchmark_data_smithi027_341149_object26953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.304378+0000", 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.78133520099999, 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93048768100000001, 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129383 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304378+0000", 2024-08-29T15:46:15.182 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304378+0000", 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304376+0000", 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.183 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304384+0000", 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7200000000000006e-06 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304400+0000", 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5546999999999999e-05 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304407+0000", 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4379999999999998e-06 2024-08-29T15:46:15.184 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.306628+0000", 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022207770000000002 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.306649+0000", 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0724e-05 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.185 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222519+0000", 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91587083400000002 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222661+0000", 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014186799999999999 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222790+0000", 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012872900000000001 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.186 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226804+0000", 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040143399999999999 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226820+0000", 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5138e-05 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234817+0000", 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079972889999999994 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.187 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234843+0000", 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6574999999999999e-05 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234866+0000", 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2379e-05 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.188 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129386 9.d 9:b11ef5c9:::benchmark_data_smithi027_341149_object26956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.310122+0000", 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.77559138999999, 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.92487085199999997, 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129386 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.189 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310122+0000", 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310122+0000", 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310120+0000", 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:46:15.190 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310129+0000", 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8000000000000004e-06 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310144+0000", 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5815e-05 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.191 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310153+0000", 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7329999999999992e-06 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310209+0000", 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5673e-05 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310219+0000", 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0550000000000001e-05 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.192 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226852+0000", 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91663254900000002 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226988+0000", 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136339 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.227083+0000", 2024-08-29T15:46:15.193 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4705999999999998e-05 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234786+0000", 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0077026150000000003 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234795+0000", 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2709999999999998e-06 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.194 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234943+0000", 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014839000000000001 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234967+0000", 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3359000000000001e-05 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234993+0000", 2024-08-29T15:46:15.195 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6163999999999999e-05 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129392 9.d 9:b5b3a1de:::benchmark_data_smithi027_341149_object26962:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.320301+0000", 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.76541199299999, 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91480801899999997, 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.196 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129392 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320301+0000", 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320301+0000", 2024-08-29T15:46:15.197 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320300+0000", 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320306+0000", 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6230000000000002e-06 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.198 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320315+0000", 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3070000000000008e-06 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320320+0000", 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0420000000000002e-06 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320360+0000", 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0213000000000003e-05 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.199 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320367+0000", 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.691e-06 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.230939+0000", 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91057188499999997 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.231056+0000", 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011665799999999999 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.200 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.231130+0000", 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4276000000000005e-05 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234905+0000", 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037745859999999999 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234912+0000", 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5290000000000003e-06 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.201 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235038+0000", 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012573 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235079+0000", 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1551999999999998e-05 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235109+0000", 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9969999999999999e-05 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.202 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129399 9.d 9:b72fd645:::benchmark_data_smithi027_341149_object26969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.327813+0000", 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.75790009100001, 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.92127104000000004, 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129399 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.203 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327813+0000", 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327813+0000", 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327812+0000", 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.204 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327819+0000", 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1389999999999998e-06 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327834+0000", 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.467e-05 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327841+0000", 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7230000000000004e-06 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.205 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327895+0000", 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3366999999999999e-05 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327905+0000", 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.076e-05 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235139+0000", 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.90723388999999999 2024-08-29T15:46:15.206 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235216+0000", 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6254000000000005e-05 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235292+0000", 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6412e-05 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.207 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248835+0000", 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013543026 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248879+0000", 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3544999999999997e-05 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248981+0000", 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000102285 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.208 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249045+0000", 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4367000000000002e-05 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249084+0000", 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9029000000000002e-05 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129409 9.d 9:b9830034:::benchmark_data_smithi027_341149_object26979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.356970+0000", 2024-08-29T15:46:15.209 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.728743307, 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.89225831899999997, 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129409 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356970+0000", 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.210 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356970+0000", 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356969+0000", 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356976+0000", 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0299999999999996e-06 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.211 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356989+0000", 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2847000000000001e-05 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356995+0000", 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5699999999999998e-06 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.357046+0000", 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0504999999999998e-05 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.212 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.357055+0000", 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1339999999999993e-06 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.239754+0000", 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.88269909199999996 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.239912+0000", 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015809299999999999 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.213 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.240041+0000", 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012887200000000001 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248888+0000", 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.008847644 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248924+0000", 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5528000000000003e-05 2024-08-29T15:46:15.214 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.215 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.215 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249130+0000", 2024-08-29T15:46:15.215 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00020563600000000001 2024-08-29T15:46:15.215 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.215 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249194+0000", 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4634000000000005e-05 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249228+0000", 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4020999999999998e-05 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.216 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129444 9.d 9:b42f07b7:::benchmark_data_smithi027_341149_object27014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.417884+0000", 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.667829622, 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.83963427700000004, 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129444 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.217 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417884+0000", 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417884+0000", 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417882+0000", 2024-08-29T15:46:15.218 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417890+0000", 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1759999999999993e-06 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417907+0000", 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7054999999999998e-05 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.219 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417916+0000", 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4339999999999992e-06 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417993+0000", 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7540000000000006e-05 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.418003+0000", 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7950000000000002e-06 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.220 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.244378+0000", 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.82637549499999996 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.244510+0000", 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013137899999999999 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.244598+0000", 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8317000000000002e-05 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.221 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249273+0000", 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046749579999999999 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249279+0000", 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0090000000000004e-06 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257450+0000", 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081709120000000007 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.222 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257488+0000", 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8463000000000001e-05 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257518+0000", 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9573e-05 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129460 9.d 9:b66c46df:::benchmark_data_smithi027_341149_object27030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:46:15.223 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.450107+0000", 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 235.63560619399999, 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.80755162700000005, 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129460 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450107+0000", 2024-08-29T15:46:15.224 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450107+0000", 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450105+0000", 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:46:15.225 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450113+0000", 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3599999999999998e-06 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450124+0000", 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1460999999999999e-05 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450131+0000", 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5740000000000001e-06 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.226 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450190+0000", 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9763999999999999e-05 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450201+0000", 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0549e-05 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249301+0000", 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.79910004899999998 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.227 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249448+0000", 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000147362 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249581+0000", 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013289500000000001 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257568+0000", 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079868119999999994 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.228 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257574+0000", 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6169999999999998e-06 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257600+0000", 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6412000000000001e-05 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257627+0000", 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6397000000000002e-05 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:46:15.229 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257659+0000", 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2181999999999998e-05 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:46:15.230 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:46:15.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:15.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:15.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:15.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:15.427 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:15.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:15.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:15.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:15.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:15.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:15.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:15.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 57 16 93521 93505 102.484 72.0625 0.00804714 0.00975257 2024-08-29T15:46:15.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:15.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:15.815 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:15.853 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:15.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:15.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:15.991 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:15.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:16.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:16.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:16.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:16.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:16.234 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:16.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:16.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:16.366 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:16.402 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:16.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:16.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:16.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:16.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:16 smithi171 ceph-mon[90031]: pgmap v1745: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-29T15:46:16.526 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:16.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:16.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:16 smithi027 ceph-mon[93921]: pgmap v1745: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-29T15:46:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:16 smithi027 ceph-mon[98970]: pgmap v1745: 119 pgs: 119 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-29T15:46:16.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:16.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:16.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:16.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 58 16 94253 94237 101.506 45.75 0.00280598 0.00980731 2024-08-29T15:46:16.751 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:16.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:16.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:16.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:16.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:16.931 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:16.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:17.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:17.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:46:17.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:17.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:17.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:17.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:17.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:17.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:17.303 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:17.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:17.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:17.452 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:17.478 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:17.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:17.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:17.487 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:17.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:17.625 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:17.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:17.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 59 16 95362 95346 100.96 69.3125 0.0139279 0.00990014 2024-08-29T15:46:17.727 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:17.760 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:17.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:17.862 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:17.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:17.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:17.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:17.998 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:18.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:18.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:18.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:18.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:18.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:18.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:18.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:18.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:18.323 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:18.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:18.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:18.468 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:18.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:18.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:18.475 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:18 smithi027 ceph-mon[98970]: pgmap v1746: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:46:18.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:18 smithi027 ceph-mon[93921]: pgmap v1746: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:46:18.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:18 smithi171 ceph-mon[90031]: pgmap v1746: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-29T15:46:18.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:18.613 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:18.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:18.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:46:18.675598+0000 min lat: 0.00100862 max lat: 0.322213 avg lat: 0.00994847 2024-08-29T15:46:18.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:46:18.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 60 16 96497 96481 100.46 70.9375 0.0272487 0.00994847 2024-08-29T15:46:18.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:18.775 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:18.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:18.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:18.920 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:18.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:19.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:19.044 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:19.047 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:19.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:19.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:19.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:19.152 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:46:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:46:19] "GET /metrics HTTP/1.1" 200 36619 "" "Prometheus/2.51.0" 2024-08-29T15:46:19.208 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:19.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:19.310 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:19.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:19.342 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:19.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:19.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:19.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:19.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:19.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:19.618 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:19.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:19.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:19.675 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 61 16 97622 97606 99.9656 70.3125 0.0119308 0.00999846 2024-08-29T15:46:19.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:19.764 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:19.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:19.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:19.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:19.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:19.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:19.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:20.020 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:20.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:20.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:20.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:20.193 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:20.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:20.247 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:46:20.294 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:20.325 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:20 smithi027 ceph-mon[93921]: pgmap v1747: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-29T15:46:20.325 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:20 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4125501719' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:20.325 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:20 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:20.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:20 smithi027 ceph-mon[98970]: pgmap v1747: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-29T15:46:20.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:20 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4125501719' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:20.325 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:20 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:20.327 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:20.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:20.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:20.463 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:20.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:20 smithi171 ceph-mon[90031]: pgmap v1747: 119 pgs: 119 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-29T15:46:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:20 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4125501719' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:20.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:20 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:20.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:20.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:20.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:20.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:20.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 62 16 98730 98714 99.4702 69.25 0.0170386 0.0100481 2024-08-29T15:46:20.701 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:20.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:20.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:20.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:20.771 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:46:20.771 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-29T15:46:20.771 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:46:20.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:20.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:20.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:20.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:21.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:21.114 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:21.145 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:21.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:21.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.286 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:21.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:21.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:21.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:21.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[98970]: osdmap e490: 8 total, 8 up, 8 in 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:21.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:21 smithi027 ceph-mon[93921]: osdmap e490: 8 total, 8 up, 8 in 2024-08-29T15:46:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.576 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:21.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:21.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 63 16 99829 99813 98.9817 68.6875 0.00996358 0.0100974 2024-08-29T15:46:21.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:21.722 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:21 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:46:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:21 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:46:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:21 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:21 smithi171 ceph-mon[90031]: osdmap e490: 8 total, 8 up, 8 in 2024-08-29T15:46:21.824 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:21.845 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:21.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:21.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:21.985 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:21.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:22.086 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:22.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:22.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:22.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:22.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:22.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:22.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:22.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:22.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:22.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:22 smithi171 ceph-mon[90031]: pgmap v1749: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:46:22.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:22 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:46:22.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:22.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:22 smithi027 ceph-mon[98970]: pgmap v1749: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:46:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:22 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:46:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:22 smithi027 ceph-mon[93921]: pgmap v1749: 119 pgs: 119 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:46:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:22 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:46:22.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:22.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 64 16 100742 100726 98.3269 57.0625 0.012238 0.0101652 2024-08-29T15:46:22.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:22.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:22.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:22.814 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:22.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:22.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:22.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:22.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:23.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:23.097 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:23.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:23.199 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:23.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:23.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:23.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:23.241 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:23.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:23.377 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:23.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:23.479 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:23.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:23.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:23.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:23.666 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:23.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:23.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 65 16 101867 101851 97.8957 70.3125 0.0189417 0.0102087 2024-08-29T15:46:23.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:23.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:23.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:23.915 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:23.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:23.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:24.056 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:24.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:24.095 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:24.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:24.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:24.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:24.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:24.322 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:24.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:24.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:24.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:24.364 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:24 smithi027 ceph-mon[98970]: pgmap v1750: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:46:24.364 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:24 smithi027 ceph-mon[93921]: pgmap v1750: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:46:24.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:24.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:24.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:24.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:24.614 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:24.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:24.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 66 16 102644 102628 97.1484 48.5625 0.00464347 0.010258 2024-08-29T15:46:24.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:24.751 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:46:24.751 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:46:24.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:24.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:24.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:24 smithi171 ceph-mon[90031]: pgmap v1750: 119 pgs: 119 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-08-29T15:46:24.783 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:46:24.783 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:46:24.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:46:24.817 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:46:24.842 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:46:24.842 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:46:24.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:24.859 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:46:24.859 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:46:24.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:24.892 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:24.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:24.907 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:46:24.907 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:46:24.941 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:46:24.941 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:46:24.974 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:46:24.975 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:46:24.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:25.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:25.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.163 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:25.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:25.264 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:25.297 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:25.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:25.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:25.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:25.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/425979458' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:25.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/425979458' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.598 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:25.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:25.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 67 16 103530 103514 96.525 55.375 0.030278 0.0103549 2024-08-29T15:46:25.699 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.733 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:25.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/425979458' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:25.834 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:25.867 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:25.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:25.995 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:25.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:26.097 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:26.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:26.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:26.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:46:26.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:26.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:26.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:26.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:26.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:26.392 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:26.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:26.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:26.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:26.539 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:26.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:26.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:26 smithi027 ceph-mon[98970]: pgmap v1751: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:46:26.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:26 smithi027 ceph-mon[93921]: pgmap v1751: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:46:26.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:26.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 68 16 104474 104458 95.9732 59 0.0063407 0.0104139 2024-08-29T15:46:26.677 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:26.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:26.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:26 smithi171 ceph-mon[90031]: pgmap v1751: 119 pgs: 119 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-29T15:46:26.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:26.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:26.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:26.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:26.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:26.962 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:26.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:27.064 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:27.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:27.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:27.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:27.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:27.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:27.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:27.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:27.397 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:27.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:27.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:27.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:27.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:27.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:27.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:27.637 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:27.676 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 69 16 105592 105576 95.5949 69.875 0.0139474 0.0104554 2024-08-29T15:46:27.683 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:27.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:27.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:27.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:27.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:27.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:27.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:27.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:27.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:27.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:28.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:28.230 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:46:28.230 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:46:28.231 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-29T15:46:28.231 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.766546 2024-08-29T15:46:28.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 4 0.7665458254447958 2024-08-29T15:46:28.237 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:28.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:28.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:28.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:28.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:28.396 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:28.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:28.437 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:28.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:28.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:28.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:28.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:28.564 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:28 smithi027 ceph-mon[93921]: pgmap v1752: 119 pgs: 119 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:46:28.565 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:28 smithi027 ceph-mon[98970]: pgmap v1752: 119 pgs: 119 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:46:28.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:28.677 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 70 16 106579 106563 95.1104 61.6875 0.0163414 0.0105085 2024-08-29T15:46:28.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:28.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:28.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:28.755 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:46:28.755 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:46:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:28 smithi171 ceph-mon[90031]: pgmap v1752: 119 pgs: 119 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:46:28.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:28.832 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:28.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:28.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:28.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:28.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:28.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:28.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:29.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:29.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:29.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:29.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:29.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:29.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:29.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:46:29] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.51.0" 2024-08-29T15:46:29.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:29.393 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:29.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:29.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:29.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:29.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:29.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:29.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:46:29.636 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:29.637 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:29.637 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:29.637 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:29.637 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:29.638 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:29.638 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:29.638 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:29.638 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:29.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:29.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:29.677 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 71 16 107640 107624 94.7048 66.3125 0.0212895 0.0105544 2024-08-29T15:46:29.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:29.806 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:29.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:29.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:46:29.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:29.967 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:29.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:30.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:30.120 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:30.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:30.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:30.222 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:30.257 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:30.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:30.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:30.309 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:30.359 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:30.391 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:30.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:30.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:30.493 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:30.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:30.527 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:30.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:30.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:30.657 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:30.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:30.678 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 72 16 108634 108618 94.2515 62.125 0.0095873 0.0106044 2024-08-29T15:46:30.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:30 smithi027 ceph-mon[98970]: pgmap v1753: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:46:30.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:30 smithi027 ceph-mon[93921]: pgmap v1753: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:46:30.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:30.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:30.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:30.890 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:30.922 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:30.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:30 smithi171 ceph-mon[90031]: pgmap v1753: 119 pgs: 119 active+clean; 6.3 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-29T15:46:31.025 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:31.058 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:31.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:31.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:31.198 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:31.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:31.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:31.332 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:31.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:31.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:31.474 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:31.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:31.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:31.576 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:31.611 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:31.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:31.678 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 73 16 109738 109722 93.9056 69 0.011428 0.0106438 2024-08-29T15:46:31.713 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:31.746 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:31.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:31.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:31.889 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:31.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:31.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:32.032 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:32.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:32.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:32.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:32.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:32.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:32.288 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:32.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:32.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:32.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:32.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:32.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:32.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:32.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:32.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:32 smithi027 ceph-mon[93921]: pgmap v1754: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-29T15:46:32.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:32 smithi027 ceph-mon[98970]: pgmap v1754: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-29T15:46:32.597 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:32.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:32.678 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 74 15 110866 110851 93.5902 70.5625 0.0106381 0.01068 2024-08-29T15:46:32.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:32.737 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:32.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:32.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:32.879 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:32.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:32.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:33.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:32 smithi171 ceph-mon[90031]: pgmap v1754: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-29T15:46:33.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:33.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:33.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:33.283 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:33.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:33.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:33.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:33.525 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:33.542 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:33.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:33.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:33.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 75 16 111929 111913 93.2237 66.375 0.0286166 0.0107212 2024-08-29T15:46:33.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:33.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:33.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:33.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:33.965 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:33.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:34.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:34.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:34.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:34.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:34.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:34.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:34.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:34.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:34.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:34.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:34.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:34.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:34.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:34.548 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:46:34.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:34.676 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:34.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:34.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:34.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 76 16 112915 112899 92.808 61.625 0.0131831 0.0107701 2024-08-29T15:46:34.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:34 smithi027 ceph-mon[98970]: pgmap v1755: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:46:34.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2240894454' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:46:34.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:34 smithi027 ceph-mon[93921]: pgmap v1755: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:46:34.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2240894454' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:46:34.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:34.818 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:34.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:34.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:34.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:34.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:34 smithi171 ceph-mon[90031]: pgmap v1755: 119 pgs: 119 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-29T15:46:35.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2240894454' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:46:35.049 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:35.083 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:35.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:35.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:35.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.224 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:35.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:35.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:35.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:35.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:35.500 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:35.582 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.4 primary-affinity to 0.766546 (8502362) 2024-08-29T15:46:35.603 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:35.626 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:35.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:35.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 77 16 113981 113965 92.4679 66.625 0.00472304 0.0107999 2024-08-29T15:46:35.727 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:35.749 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:35.799 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:35.800 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2240894454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:46:35.800 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[98970]: osdmap e491: 8 total, 8 up, 8 in 2024-08-29T15:46:35.800 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2338460411' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 0.7665458254447958} : dispatch 2024-08-29T15:46:35.800 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:35.800 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2240894454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:46:35.800 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[93921]: osdmap e491: 8 total, 8 up, 8 in 2024-08-29T15:46:35.800 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2338460411' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 0.7665458254447958} : dispatch 2024-08-29T15:46:35.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:35.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.873 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:35.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:35.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:35.997 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:35.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:35 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2240894454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:46:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:35 smithi171 ceph-mon[90031]: osdmap e491: 8 total, 8 up, 8 in 2024-08-29T15:46:36.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2338460411' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 0.7665458254447958} : dispatch 2024-08-29T15:46:36.100 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:36.123 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:36.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:36.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:36.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:36.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:36.340 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:36.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:36.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:36.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:36.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:36.499 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:36.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:36.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:36.631 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:36.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:36.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:36.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 78 16 114006 113990 91.3028 1.5625 0.00640646 0.0109395 2024-08-29T15:46:36.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:36.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:36.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:36.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:36.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:36.921 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:36.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:36 smithi171 ceph-mon[90031]: pgmap v1757: 119 pgs: 119 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:46:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2338460411' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7665458254447958}]': finished 2024-08-29T15:46:37.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:36 smithi171 ceph-mon[90031]: osdmap e492: 8 total, 8 up, 8 in 2024-08-29T15:46:37.022 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:37.050 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:37.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:37.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:37.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:36 smithi027 ceph-mon[98970]: pgmap v1757: 119 pgs: 119 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:46:37.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2338460411' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7665458254447958}]': finished 2024-08-29T15:46:37.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:36 smithi027 ceph-mon[98970]: osdmap e492: 8 total, 8 up, 8 in 2024-08-29T15:46:37.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:36 smithi027 ceph-mon[93921]: pgmap v1757: 119 pgs: 119 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-29T15:46:37.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2338460411' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.7665458254447958}]': finished 2024-08-29T15:46:37.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:36 smithi027 ceph-mon[93921]: osdmap e492: 8 total, 8 up, 8 in 2024-08-29T15:46:37.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:37.188 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:37.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:37.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:37.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:37.330 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:37.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:37.431 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:37.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:37.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:37.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:37.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:46:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:37.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:37.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:37.683 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 79 16 114941 114925 90.8868 58.4375 0.0157575 0.0109979 2024-08-29T15:46:37.720 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:37.756 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:37.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:37.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:37.886 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:37.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:37.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:38.010 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:38.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:37 smithi171 ceph-mon[90031]: osdmap e493: 8 total, 8 up, 8 in 2024-08-29T15:46:38.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:37 smithi027 ceph-mon[93921]: osdmap e493: 8 total, 8 up, 8 in 2024-08-29T15:46:38.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:37 smithi027 ceph-mon[98970]: osdmap e493: 8 total, 8 up, 8 in 2024-08-29T15:46:38.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:38.152 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:38.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:38.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:38.294 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:38.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:38.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:38.539 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:38.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:38.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:38.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:38.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:2024-08-29T15:46:38.681357+0000 min lat: 0.00100862 max lat: 1.03562 avg lat: 0.0110398 2024-08-29T15:46:38.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-29T15:46:38.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 80 16 115936 115920 90.5281 62.1875 0.0706639 0.0110398 2024-08-29T15:46:38.711 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:38.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:38.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:38.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:38.918 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:38 smithi171 ceph-mon[90031]: pgmap v1760: 119 pgs: 3 peering, 116 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:46:38.918 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:38 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:46:38.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:38.994 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:38.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:39.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:38 smithi027 ceph-mon[98970]: pgmap v1760: 119 pgs: 3 peering, 116 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:46:39.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:38 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:46:39.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:38 smithi027 ceph-mon[93921]: pgmap v1760: 119 pgs: 3 peering, 116 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-29T15:46:39.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:38 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:46:39.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:39.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:39.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:39.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:39.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:39.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:46:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:46:39] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.51.0" 2024-08-29T15:46:39.383 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:39.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:39.425 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:39.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:39.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:39.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:39.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:39.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:39.681 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 81 16 117051 117035 90.2705 69.6875 0.00784675 0.0110723 2024-08-29T15:46:39.725 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:39.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:39.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:39.880 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:39.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:39.981 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:40.015 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:40.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:40.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:40.158 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:40.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:40.301 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:40.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:40.403 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:40.438 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:40.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:40.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:40.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.579 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:40.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:40.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:40.683 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 82 16 118096 118080 89.9661 65.3125 0.00857676 0.0111104 2024-08-29T15:46:40.715 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:40.817 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:40.852 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:40.954 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:40.986 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:40.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:40.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:40 smithi027 ceph-mon[93921]: pgmap v1761: 119 pgs: 3 peering, 116 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-08-29T15:46:40.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:40 smithi027 ceph-mon[98970]: pgmap v1761: 119 pgs: 3 peering, 116 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-08-29T15:46:41.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:40 smithi171 ceph-mon[90031]: pgmap v1761: 119 pgs: 3 peering, 116 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2024-08-29T15:46:41.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:41.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:41.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:41.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:41.230 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:41.265 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:41.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:41.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:41.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:41.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:41.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:41.508 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:41.546 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:41.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:41.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:41.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:41.682 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 83 16 119146 119130 89.6729 65.625 0.0149772 0.011147 2024-08-29T15:46:41.685 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:41.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:41.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:41.787 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:41.815 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:41.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:41.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:41.963 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:41.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:42.011 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:42.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:42.140 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:42.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:42.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:42.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:42.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:42.297 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:42.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:42.399 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:42.424 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:42.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:42.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:42.442 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:46:42.442 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:46:42.485 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:46:42.485 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:46:42.487 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:46:42.487 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:46:42.523 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:46:42.524 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:46:42.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:42.561 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:46:42.561 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:46:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:42.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:42.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:42.593 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:46:42.593 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:46:42.624 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:46:42.624 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:46:42.660 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:46:42.661 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:46:42.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:42.682 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 84 16 120116 120100 89.3271 60.625 0.0236895 0.0111894 2024-08-29T15:46:42.696 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:46:42.696 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:46:42.706 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:42.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:42.808 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:42.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:42.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:42.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:42.944 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:42.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:42 smithi027 ceph-mon[93921]: pgmap v1762: 119 pgs: 1 peering, 118 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:46:42.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3836611348' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:42.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:42 smithi027 ceph-mon[98970]: pgmap v1762: 119 pgs: 1 peering, 118 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:46:42.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3836611348' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:42.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:42.976 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:42.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:42 smithi171 ceph-mon[90031]: pgmap v1762: 119 pgs: 1 peering, 118 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-29T15:46:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3836611348' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:46:43.078 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:43.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:43.111 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:43.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:43.211 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:43.249 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:43.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:43.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:43.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:43.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:43.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:43.492 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:43.525 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:43.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:43.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:43.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:43.665 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:43.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:43.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:43.682 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 85 16 121125 121109 89.0181 63.0625 0.0204934 0.0112283 2024-08-29T15:46:43.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:43.803 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:43.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:43.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:43.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:43.939 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:43.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:44.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:43 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:46:44.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:44.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:43 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:46:44.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:43 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:46:44.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:44.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:44.179 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:44.215 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:44.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:44.316 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:44.354 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:44.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:44.456 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:44.490 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 5, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:46:44.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 0 1 2024-08-29T15:46:44.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:44.492 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:44.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:44.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:44.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:44.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:44.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:44.682 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 86 16 122120 122104 88.7057 62.1875 0.015008 0.0112682 2024-08-29T15:46:44.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:44.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:44.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:44.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:44 smithi027 ceph-mon[98970]: pgmap v1763: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-08-29T15:46:44.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:44.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:44.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:44 smithi027 ceph-mon[93921]: pgmap v1763: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-08-29T15:46:44.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:44.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:44.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:44.907 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:44.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:44.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:45.009 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:44 smithi171 ceph-mon[90031]: pgmap v1763: 119 pgs: 119 active+clean; 7.3 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-08-29T15:46:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:45.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:45.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:45.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:45.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:45.159 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:45.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:45.195 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:45.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:45.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:46:45.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:45.344 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:45.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:45.445 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:45.481 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:45.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:45.583 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:45.613 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:45.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:45.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:45.682 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 87 16 123198 123182 88.4605 67.375 0.0102324 0.0113001 2024-08-29T15:46:45.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:45.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:45.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:45.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:45.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:45.889 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:45.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:45.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:46.030 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:46.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:46.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:46.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:46.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:46.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:46.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:46.284 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:46.316 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:46.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:46.418 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:46.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:46.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:46.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:46.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:46.616 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:46.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:46.684 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 88 16 124193 124177 88.1608 62.1875 0.0155728 0.0113374 2024-08-29T15:46:46.717 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:46.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:46.753 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:46.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:46.854 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:46.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:46.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:46.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:47.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:46 smithi171 ceph-mon[90031]: pgmap v1764: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2024-08-29T15:46:47.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:47.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:47.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:46 smithi027 ceph-mon[93921]: pgmap v1764: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2024-08-29T15:46:47.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:46 smithi027 ceph-mon[98970]: pgmap v1764: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2024-08-29T15:46:47.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:47.171 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:47.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:47.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:47.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:47.315 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:47.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:47.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:47.378 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:47.384 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:47.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:47.463 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:47.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:47.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:47.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:47.609 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:47.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:47.684 INFO:tasks.radosbench.radosbench.0.smithi027.stdout: 89 16 125211 125195 87.8852 63.625 0.0157358 0.0113741 2024-08-29T15:46:47.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:47.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:46:47.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:47.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:47.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:47.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:47.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:47.904 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:47.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:47.906 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:47 smithi027 ceph-mon[98970]: pgmap v1765: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-08-29T15:46:47.906 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:47 smithi027 ceph-mon[93921]: pgmap v1765: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-08-29T15:46:48.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:48.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:47 smithi171 ceph-mon[90031]: pgmap v1765: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2024-08-29T15:46:48.056 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:48.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:48.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:48.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:48.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:48.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:48.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:48.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:48.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:48.446 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:48.473 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:48.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:48.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:48.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:48.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:48.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:48.684 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total time run: 90.0077 2024-08-29T15:46:48.684 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Total writes made: 126127 2024-08-29T15:46:48.684 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Write size: 65536 2024-08-29T15:46:48.684 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Object size: 65536 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Bandwidth (MB/sec): 87.5807 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Bandwidth: 35.7187 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max bandwidth (MB/sec): 215.438 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min bandwidth (MB/sec): 1.5625 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average IOPS: 1401 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev IOPS: 571.499 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max IOPS: 3447 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min IOPS: 25 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Average Latency(s): 0.0114138 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Stddev Latency(s): 0.0157913 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Max latency(s): 1.03562 2024-08-29T15:46:48.685 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Min latency(s): 0.00100862 2024-08-29T15:46:48.689 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Cleaning up (deleting benchmark objects) 2024-08-29T15:46:48.700 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:48.728 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:48.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:48.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:48.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:48.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:48.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:49.015 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:46:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:46:49] "GET /metrics HTTP/1.1" 200 36627 "" "Prometheus/2.51.0" 2024-08-29T15:46:49.018 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:49.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.160 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:49.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:49.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.300 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:49.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:49.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:49.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:49.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:49.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:49.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:49 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:49.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:49.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:49 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:49.579 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:49.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:49.681 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:49.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:49.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:49.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:49 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:49.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:49.836 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:49.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:49.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:49.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:49.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:50.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:50.113 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:50.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:50.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:50.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:50.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:50.255 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:50.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:50.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:50.396 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:50.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:50.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:50.522 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:50.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:50.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:50.624 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:50.661 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:50.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:50.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:50.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:50 smithi171 ceph-mon[90031]: pgmap v1766: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2024-08-29T15:46:50.793 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:50.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:50.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:50 smithi027 ceph-mon[93921]: pgmap v1766: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2024-08-29T15:46:50.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:50 smithi027 ceph-mon[98970]: pgmap v1766: 119 pgs: 119 active+clean; 7.6 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2024-08-29T15:46:50.895 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:50.927 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:50.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:51.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:51.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:51.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:51.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:51.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:51.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:51.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:51.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:51.360 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:51.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:51.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:51.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:51.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:51.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:51.611 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:51.638 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:51.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:51.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:51.771 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:51.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:51.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:51.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:51.915 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:51.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:52.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:52.055 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:52.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:52.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:52.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:52.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:52.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:52.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:52.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:52.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:52.340 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:46:52.427 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:52.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:52.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:52.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:52.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:52.596 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:52.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:52.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:52.738 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:52.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:52 smithi171 ceph-mon[90031]: pgmap v1767: 119 pgs: 119 active+clean; 7.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 58 MiB/s wr, 978 op/s 2024-08-29T15:46:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/294532181' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:52 smithi027 ceph-mon[98970]: pgmap v1767: 119 pgs: 119 active+clean; 7.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 58 MiB/s wr, 978 op/s 2024-08-29T15:46:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/294532181' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:52.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:52 smithi027 ceph-mon[93921]: pgmap v1767: 119 pgs: 119 active+clean; 7.6 GiB data, 30 GiB used, 686 GiB / 715 GiB avail; 58 MiB/s wr, 978 op/s 2024-08-29T15:46:52.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/294532181' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:46:52.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:52.883 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:52.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:52.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:52.985 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:53.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:53.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:53.132 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:53.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:53.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:53.293 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:53.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:53.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:53.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:53.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:53.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:53.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/294532181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:53.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[93921]: osdmap e494: 8 total, 8 up, 8 in 2024-08-29T15:46:53.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:46:53.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:46:53.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:53.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:53.654 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/294532181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:53.654 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[98970]: osdmap e494: 8 total, 8 up, 8 in 2024-08-29T15:46:53.654 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:46:53.654 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:46:53.654 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:53.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:53.678 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:46:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.727 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:53.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:53 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:46:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/294532181' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:46:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:53 smithi171 ceph-mon[90031]: osdmap e494: 8 total, 8 up, 8 in 2024-08-29T15:46:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:46:53.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:46:53.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:46:53.828 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:53.856 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:53.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:53.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:53.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:53.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:53.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:54.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:54.136 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:54.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:54.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:54.281 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:54.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:54.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:54.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:54.424 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:54.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:54.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:54.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:54 smithi027 ceph-mon[93921]: pgmap v1769: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 46 MiB/s wr, 1.03k op/s 2024-08-29T15:46:54.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:54 smithi027 ceph-mon[98970]: pgmap v1769: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 46 MiB/s wr, 1.03k op/s 2024-08-29T15:46:54.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:54.575 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:54.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:54.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:54.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:54.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:54 smithi171 ceph-mon[90031]: pgmap v1769: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 46 MiB/s wr, 1.03k op/s 2024-08-29T15:46:54.812 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:54.841 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:54.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:54.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:54.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:54.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:54.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:55.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:55.121 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:55.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:55.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:55.223 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:55.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:55.262 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:55.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:55.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:55.398 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:55.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:55.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:55.537 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:55.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:55.638 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:55.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:55.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:55.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:55.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:55.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:55.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:55.898 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:55.935 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:55.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:56.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:56.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:56.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:56.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:56.234 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:56.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:56.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:56.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:56.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:56.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:56.483 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:56.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:46:56.512 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:56.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:46:56.614 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:56.641 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:56.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:56.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:56.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:56 smithi027 ceph-mon[93921]: pgmap v1770: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 42 MiB/s wr, 1.03k op/s 2024-08-29T15:46:56.646 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:56 smithi027 ceph-mon[98970]: pgmap v1770: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 42 MiB/s wr, 1.03k op/s 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29420 9.0 e430/420 9:0e19c1e7:::benchmark_data_smithi027_341149_object29419:head v 430'1735, mlcod=430'1732)", 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.491631+0000", 2024-08-29T15:46:56.689 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 407.19465506900002, 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35499817, 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491631+0000", 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491631+0000", 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491632+0000", 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2600000000000002e-07 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491652+0000", 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9599999999999999e-05 2024-08-29T15:46:56.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491662+0000", 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0621e-05 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491669+0000", 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6159999999999998e-06 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491744+0000", 2024-08-29T15:46:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5499000000000003e-05 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491764+0000", 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9318999999999999e-05 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846582+0000", 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35481835099999998 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846630+0000", 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7438000000000002e-05 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29447 9.2 e430/420 9:4ae6b342:::benchmark_data_smithi027_341149_object29446:head v 430'1859, mlcod=430'1857)", 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.501916+0000", 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 407.18437060700001, 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34462358999999998, 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.694 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501916+0000", 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501916+0000", 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501917+0000", 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.156e-06 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501945+0000", 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8394e-05 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843686+0000", 2024-08-29T15:46:56.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34174009900000002 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843712+0000", 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6857999999999999e-05 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844135+0000", 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042273499999999998 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844159+0000", 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3467000000000001e-05 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846502+0000", 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023433460000000001 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846539+0000", 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7534999999999999e-05 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29456 9.6 e430/420 9:6f12b949:::benchmark_data_smithi027_341149_object29455:head v 430'1808, mlcod=430'1806)", 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.505599+0000", 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 407.18068743999999, 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34067662999999998, 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.699 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505599+0000", 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505599+0000", 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505600+0000", 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4589999999999999e-06 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505650+0000", 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9872e-05 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843666+0000", 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33801598399999999 2024-08-29T15:46:56.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843688+0000", 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2158e-05 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843781+0000", 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2003000000000007e-05 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843809+0000", 2024-08-29T15:46:56.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8126000000000001e-05 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846234+0000", 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024256030000000001 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846276+0000", 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1424999999999997e-05 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.703 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29465 9.0 e430/420 9:098160e2:::benchmark_data_smithi027_341149_object29464:head v 430'1736, mlcod=430'1733)", 2024-08-29T15:46:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.509005+0000", 2024-08-29T15:46:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 407.17728106499999, 2024-08-29T15:46:56.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33833135600000003, 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509005+0000", 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509005+0000", 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509007+0000", 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.297e-06 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509033+0000", 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6035000000000001e-05 2024-08-29T15:46:56.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843695+0000", 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.334662401 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843705+0000", 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0069e-05 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844810+0000", 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011047920000000001 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844839+0000", 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9156999999999999e-05 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.847284+0000", 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024448899999999999 2024-08-29T15:46:56.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.847337+0000", 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2714999999999999e-05 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:82239 9.6 e432/420 9:6c4ee926:::benchmark_data_smithi027_341149_object82238:head v 432'5131, mlcod=432'5127)", 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:44.318942+0000", 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 372.367344059, 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.063525874999999996, 2024-08-29T15:46:56.709 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.318942+0000", 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.318942+0000", 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.318943+0000", 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.02e-06 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.318997+0000", 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4024999999999999e-05 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.319009+0000", 2024-08-29T15:46:56.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1559999999999999e-05 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.319016+0000", 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0609999999999998e-06 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.319574+0000", 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00055755099999999999 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.319592+0000", 2024-08-29T15:46:56.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8045000000000001e-05 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.382415+0000", 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.062823622999999995 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.382468+0000", 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2989999999999999e-05 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:82247 9.2 e432/420 9:4dfff4ef:::benchmark_data_smithi027_341149_object82246:head v 432'5119, mlcod=432'5115)", 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:44.321633+0000", 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 372.36465335100002, 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.064235445000000002, 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.321633+0000", 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.321633+0000", 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.321634+0000", 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8599999999999997e-07 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.323334+0000", 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001699905 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.323345+0000", 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1318000000000001e-05 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.323353+0000", 2024-08-29T15:46:56.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6290000000000001e-06 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.323633+0000", 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028028500000000002 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.323652+0000", 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9154999999999999e-05 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.385814+0000", 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.062161345 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:44.385869+0000", 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4922e-05 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:101185 9.7 e444/440 9:ea4b3e9a:::benchmark_data_smithi027_341149_object101184:head v 444'6261, mlcod=0'0)", 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:41:26.204702+0000", 2024-08-29T15:46:56.718 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 330.48158429799997, 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.061937180000000001, 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.204702+0000", 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.204702+0000", 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.204703+0000", 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1200000000000001e-07 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.204783+0000", 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9586999999999995e-05 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.204803+0000", 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.999e-05 2024-08-29T15:46:56.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.204812+0000", 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8610000000000004e-06 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.212131+0000", 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073192789999999997 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.212306+0000", 2024-08-29T15:46:56.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017546900000000001 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.266577+0000", 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.054270574000000002 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:26.266639+0000", 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2508000000000005e-05 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:105823 9.7 e444/440 9:e91f935c:::benchmark_data_smithi027_341149_object3393:head v 444'6543, mlcod=0'0)", 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:41:34.578325+0000", 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 322.10796172699997, 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.071164542999999997, 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.578325+0000", 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.578325+0000", 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.578326+0000", 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.184e-06 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.578334+0000", 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3499999999999997e-06 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.578352+0000", 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8213e-05 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.578362+0000", 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4259999999999992e-06 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.604068+0000", 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.025706242000000001 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.604134+0000", 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5551999999999999e-05 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.649435+0000", 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.045301526000000002 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:41:34.649489+0000", 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4049999999999999e-05 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:120748 9.7 e447/446 9:e4e2746a:::benchmark_data_smithi027_341149_object18318:head v 447'7473, mlcod=429'1121)", 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:02.914734+0000", 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 293.77155267699999, 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134365084, 2024-08-29T15:46:56.727 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914734+0000", 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914734+0000", 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914735+0000", 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.28e-06 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914745+0000", 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0117e-05 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914762+0000", 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6680999999999998e-05 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914771+0000", 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1519999999999998e-06 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914859+0000", 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7588000000000002e-05 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.914884+0000", 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5751000000000001e-05 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:03.049055+0000", 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13417014199999999 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:03.049099+0000", 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4372999999999997e-05 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(9.7 447/446 [PushOp(9:eca0674a:::benchmark_data_smithi027_341149_object18339:head, version: 429'1126, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:eca0674a:::benchmark_data_smithi027_341149_object18339:head@429'1126, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:02.923120+0000", 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 293.76316621900003, 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.12598695700000001, 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.732 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.923120+0000", 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.923120+0000", 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.923135+0000", 2024-08-29T15:46:56.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4600000000000001e-05 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.923187+0000", 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2073000000000001e-05 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.923209+0000", 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1691e-05 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.923228+0000", 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9538000000000002e-05 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.930010+0000", 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067819949999999999 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:02.930024+0000", 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3383e-05 2024-08-29T15:46:56.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:03.049107+0000", 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.119083677 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:127191 9.2 e448/440 9:4ff02e25:::benchmark_data_smithi027_341149_object24761:head v 448'7938, mlcod=0'0)", 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:15.227527+0000", 2024-08-29T15:46:56.736 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 281.458759628, 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091033281999999993, 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227527+0000", 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227527+0000", 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227528+0000", 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139e-06 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227544+0000", 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5766000000000001e-05 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227557+0000", 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2921999999999999e-05 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227564+0000", 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.469e-06 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227648+0000", 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3768000000000002e-05 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.227671+0000", 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2880000000000001e-05 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.318448+0000", 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.090777192000000007 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.318560+0000", 2024-08-29T15:46:56.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011214600000000001 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:127196 9.2 e448/440 9:46bddc86:::benchmark_data_smithi027_341149_object24766:head v 448'7939, mlcod=0'0)", 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:15.235770+0000", 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 281.45051684200001, 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.083006413000000001, 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.741 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.235770+0000", 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.235770+0000", 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.235771+0000", 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.387e-06 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.235780+0000", 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8929999999999994e-06 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.300055+0000", 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.064274960000000006 2024-08-29T15:46:56.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.300065+0000", 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0195e-05 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.303924+0000", 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003859127 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.303950+0000", 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5996e-05 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.318566+0000", 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014616183 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.318776+0000", 2024-08-29T15:46:56.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020967199999999999 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:127155 9.7 e448/446 9:ebbb6b96:::benchmark_data_smithi027_341149_object24725:head v 448'7866, mlcod=429'1516)", 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:15.246796+0000", 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 281.439490175, 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.084711104999999995, 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.246796+0000", 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.246796+0000", 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.246798+0000", 2024-08-29T15:46:56.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.279e-06 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.246808+0000", 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0028e-05 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.299986+0000", 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053178112 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.300001+0000", 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4992e-05 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.300066+0000", 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5312000000000001e-05 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.300096+0000", 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9696000000000002e-05 2024-08-29T15:46:56.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.331344+0000", 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.031248022 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:15.331507+0000", 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000163664 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.750 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:166954 9.7 e460/446 9:e4db641f:::benchmark_data_smithi027_341149_object64524:head v 460'10329, mlcod=431'3979)", 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:43:30.007616+0000", 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 206.67867086000001, 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.048420605999999998, 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.007616+0000", 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.007616+0000", 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.007617+0000", 2024-08-29T15:46:56.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.48e-06 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.007626+0000", 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2399999999999996e-06 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.007646+0000", 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9621000000000001e-05 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.007655+0000", 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9879999999999993e-06 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.009643+0000", 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019885160000000001 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.009673+0000", 2024-08-29T15:46:56.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9699000000000001e-05 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.055768+0000", 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.046094531000000001 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:43:30.056036+0000", 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026853100000000002 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.755 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51080 10.9 e486/480 10:9f1d1fe6:::benchmark_data_smithi027_374610_object51079:head v 486'3226, mlcod=486'3224)", 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.742142+0000", 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 73.944144072, 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31543603799999997, 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742142+0000", 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742142+0000", 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742143+0000", 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8200000000000008e-07 2024-08-29T15:46:56.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742180+0000", 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6165999999999999e-05 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742196+0000", 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6762999999999999e-05 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742209+0000", 2024-08-29T15:46:56.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2422999999999999e-05 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742270+0000", 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1316999999999996e-05 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742288+0000", 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8278999999999999e-05 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057489+0000", 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31520077899999999 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057578+0000", 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9328999999999998e-05 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.760 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51084 10.e e486/480 10:78604a72:::benchmark_data_smithi027_374610_object51083:head v 486'3245, mlcod=486'3243)", 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.742236+0000", 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 73.944049978999999, 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31535221400000002, 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742236+0000", 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742236+0000", 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742237+0000", 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0600000000000001e-07 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742257+0000", 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0392999999999999e-05 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:56.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742265+0000", 2024-08-29T15:46:56.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4410000000000004e-06 2024-08-29T15:46:56.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742268+0000", 2024-08-29T15:46:56.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3749999999999999e-06 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742336+0000", 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7867999999999995e-05 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053962+0000", 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.311625861 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057430+0000", 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003468526 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057589+0000", 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015844399999999999 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.766 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51089 10.5 e486/480 10:a181d637:::benchmark_data_smithi027_374610_object51088:head v 486'3265, mlcod=486'3263)", 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.742479+0000", 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 73.943807269000004, 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31610776899999998, 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742479+0000", 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742479+0000", 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742480+0000", 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6000000000000003e-07 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742500+0000", 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.02e-05 2024-08-29T15:46:56.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053967+0000", 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31146699700000002 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053986+0000", 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8902e-05 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054117+0000", 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013076799999999999 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054142+0000", 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5327e-05 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.058540+0000", 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043983599999999996 2024-08-29T15:46:56.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.058587+0000", 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6554999999999997e-05 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51088 10.e e486/480 10:7d47e90e:::benchmark_data_smithi027_374610_object51087:head v 486'3246, mlcod=486'3243)", 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 73.942060807000004, 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31355423500000001, 2024-08-29T15:46:56.771 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:46:56.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7000000000000004e-07 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744246+0000", 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9559e-05 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053974+0000", 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30972837800000003 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053986+0000", 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1372999999999999e-05 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.055839+0000", 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001853146 2024-08-29T15:46:56.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.055861+0000", 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.181e-05 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057753+0000", 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018923029999999999 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057780+0000", 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6896000000000001e-05 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51095 10.a e486/480 10:596831c7:::benchmark_data_smithi027_374610_object51094:head v 486'3255, mlcod=486'3253)", 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.746000+0000", 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 73.940286850999996, 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31383123400000001, 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.776 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746000+0000", 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746000+0000", 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746001+0000", 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.083e-06 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746064+0000", 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2955999999999998e-05 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053983+0000", 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30791897299999998 2024-08-29T15:46:56.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053994+0000", 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1012000000000001e-05 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054517+0000", 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00052327500000000002 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054536+0000", 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8861000000000001e-05 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.059775+0000", 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052396079999999998 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.059831+0000", 2024-08-29T15:46:56.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5466000000000001e-05 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:91801 10.a e489/480 10:5480a82d:::benchmark_data_smithi027_374610_object91800:head v 489'5799, mlcod=489'5797)", 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:14.139149+0000", 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 42.547137282000001, 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.091287824000000004, 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:46:56.782 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.139149+0000", 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.139149+0000", 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.139151+0000", 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.607e-06 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.139203+0000", 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1994999999999999e-05 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.139223+0000", 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9735999999999999e-05 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.139234+0000", 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1236999999999999e-05 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.215198+0000", 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.075964508 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.215223+0000", 2024-08-29T15:46:56.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4731000000000001e-05 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.230346+0000", 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015122607 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:14.230437+0000", 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1403000000000006e-05 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.786 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:46:56.787 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:46:56.787 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:46:56.788 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:56 smithi171 ceph-mon[90031]: pgmap v1770: 119 pgs: 119 active+clean; 7.5 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 42 MiB/s wr, 1.03k op/s 2024-08-29T15:46:56.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:56.788 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:56.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:56.889 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:56.919 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:56.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:57.021 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:57.046 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:57.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:57.148 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:57.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:57.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:57.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:57.319 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:57.351 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.0 to 1 (10000) 2024-08-29T15:46:57.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:57.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:57.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:57.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:46:57.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1435611026' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:46:57.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1435611026' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:46:57.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:57.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:46:57.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:46:57.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:46:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1435611026' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:46:57.798 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:57.819 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:57.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:57.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:57.943 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:57.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:57.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:58.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:58.067 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:58.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:58.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:58.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:58.292 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:58.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:58.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:58.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:58.440 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:58.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:58.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:58.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:46:58.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:58.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:46:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:58 smithi171 ceph-mon[90031]: pgmap v1771: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 1.03k op/s 2024-08-29T15:46:58.799 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1435611026' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:46:58.799 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:58 smithi171 ceph-mon[90031]: osdmap e495: 8 total, 8 up, 8 in 2024-08-29T15:46:58.799 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:58 smithi027 ceph-mon[93921]: pgmap v1771: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 1.03k op/s 2024-08-29T15:46:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1435611026' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:46:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:58 smithi027 ceph-mon[93921]: osdmap e495: 8 total, 8 up, 8 in 2024-08-29T15:46:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:58 smithi027 ceph-mon[98970]: pgmap v1771: 119 pgs: 119 active+clean; 7.3 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 1.03k op/s 2024-08-29T15:46:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1435611026' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:46:58.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:58 smithi027 ceph-mon[98970]: osdmap e495: 8 total, 8 up, 8 in 2024-08-29T15:46:58.820 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:58.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:58.949 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:58.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:59.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:59.072 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:59.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:59.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:46:59.199 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:59.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:46:59.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:46:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:46:59] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-08-29T15:46:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.325 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:59.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:59.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:46:59.460 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:46:59.508 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:46:59.508 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:46:59.539 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:46:59.539 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:46:59.562 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:59.586 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:46:59.586 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:46:59.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.588 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:59.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:59.627 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:46:59.627 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:46:59.627 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:59 smithi027 ceph-mon[98970]: osdmap e496: 8 total, 8 up, 8 in 2024-08-29T15:46:59.627 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:59.627 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:46:59 smithi027 ceph-mon[98970]: osdmap e497: 8 total, 8 up, 8 in 2024-08-29T15:46:59.628 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:59 smithi027 ceph-mon[93921]: osdmap e496: 8 total, 8 up, 8 in 2024-08-29T15:46:59.628 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:59.628 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:46:59 smithi027 ceph-mon[93921]: osdmap e497: 8 total, 8 up, 8 in 2024-08-29T15:46:59.666 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:46:59.666 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:46:59.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:46:59.705 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:46:59.705 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:46:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.729 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:46:59.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:46:59.735 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:46:59.735 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:46:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:59 smithi171 ceph-mon[90031]: osdmap e496: 8 total, 8 up, 8 in 2024-08-29T15:46:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:46:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:46:59 smithi171 ceph-mon[90031]: osdmap e497: 8 total, 8 up, 8 in 2024-08-29T15:46:59.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:46:59.771 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:46:59.830 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:46:59.861 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:59.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:46:59.962 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:46:59.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:46:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:46:59.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:00.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:00.131 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:00.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:00.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:00.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:00.274 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:00.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:00.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:00.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-29T15:47:00.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:00.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:00.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:00.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:00.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:00.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[98970]: pgmap v1774: 119 pgs: 1 remapped+peering, 4 peering, 114 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 1.7 MiB/s wr, 901 op/s 2024-08-29T15:47:00.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[98970]: 10.d starting backfill to osd.0 from (0'0,0'0] MAX to 494'8411 2024-08-29T15:47:00.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[98970]: 10.4 starting backfill to osd.0 from (0'0,0'0] MAX to 494'8353 2024-08-29T15:47:00.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[98970]: 10.a starting backfill to osd.0 from (0'0,0'0] MAX to 494'8627 2024-08-29T15:47:00.572 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/585692881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:00.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[93921]: pgmap v1774: 119 pgs: 1 remapped+peering, 4 peering, 114 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 1.7 MiB/s wr, 901 op/s 2024-08-29T15:47:00.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[93921]: 10.d starting backfill to osd.0 from (0'0,0'0] MAX to 494'8411 2024-08-29T15:47:00.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[93921]: 10.4 starting backfill to osd.0 from (0'0,0'0] MAX to 494'8353 2024-08-29T15:47:00.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[93921]: 10.a starting backfill to osd.0 from (0'0,0'0] MAX to 494'8627 2024-08-29T15:47:00.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:00 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/585692881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:00.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:00.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:00.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:00.731 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:00.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:00.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:00 smithi171 ceph-mon[90031]: pgmap v1774: 119 pgs: 1 remapped+peering, 4 peering, 114 active+clean; 7.2 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 1.7 MiB/s wr, 901 op/s 2024-08-29T15:47:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:00 smithi171 ceph-mon[90031]: 10.d starting backfill to osd.0 from (0'0,0'0] MAX to 494'8411 2024-08-29T15:47:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:00 smithi171 ceph-mon[90031]: 10.4 starting backfill to osd.0 from (0'0,0'0] MAX to 494'8353 2024-08-29T15:47:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:00 smithi171 ceph-mon[90031]: 10.a starting backfill to osd.0 from (0'0,0'0] MAX to 494'8627 2024-08-29T15:47:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:00 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/585692881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:00.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:00.874 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:00.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:00.976 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:01.014 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:01.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:01.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:01.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 1 1 2024-08-29T15:47:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:01.192 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:01.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:01.208 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:01.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:01.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:01.343 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:01.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:01.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:01.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:01.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:01.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:01.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:01.628 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:01.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:01.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:01.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:01 smithi027 ceph-mon[93921]: osdmap e498: 8 total, 8 up, 8 in 2024-08-29T15:47:01.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:01 smithi027 ceph-mon[93921]: osdmap e499: 8 total, 8 up, 8 in 2024-08-29T15:47:01.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:01 smithi027 ceph-mon[98970]: osdmap e498: 8 total, 8 up, 8 in 2024-08-29T15:47:01.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:01 smithi027 ceph-mon[98970]: osdmap e499: 8 total, 8 up, 8 in 2024-08-29T15:47:01.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:01.736 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:01 smithi171 ceph-mon[90031]: osdmap e498: 8 total, 8 up, 8 in 2024-08-29T15:47:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:01 smithi171 ceph-mon[90031]: osdmap e499: 8 total, 8 up, 8 in 2024-08-29T15:47:01.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:01.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:01.905 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:01.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:01.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:01.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:02.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:02.079 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:02.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:02.179 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:02.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:02.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:02.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:02.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:02.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:02.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:02.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:02.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:02.567 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:02.592 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:02.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:02.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:02.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:02.742 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:02.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:02 smithi171 ceph-mon[90031]: pgmap v1777: 119 pgs: 1 remapped+peering, 7 peering, 111 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 1024 KiB/s, 2 objects/s recovering 2024-08-29T15:47:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:02 smithi027 ceph-mon[98970]: pgmap v1777: 119 pgs: 1 remapped+peering, 7 peering, 111 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 1024 KiB/s, 2 objects/s recovering 2024-08-29T15:47:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:02 smithi027 ceph-mon[93921]: pgmap v1777: 119 pgs: 1 remapped+peering, 7 peering, 111 active+clean; 7.1 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 1024 KiB/s, 2 objects/s recovering 2024-08-29T15:47:02.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:02.877 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:02.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:02.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:02.979 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:03.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:03.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:03.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:03.175 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:03.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:03.277 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:03.306 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:03.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:03.408 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:03.436 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:03.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:03.538 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:03.571 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:03.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:03.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:03.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:03.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:47:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:03.723 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:03.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:03.824 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:03.876 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:03.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:03.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:03.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:04.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:04.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:04.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:04.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:04.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:04.155 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:04.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:04.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:04.298 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:04.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:04.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:04.452 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:04.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:04.553 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:04.582 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:04.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:04.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:04 smithi027 ceph-mon[98970]: pgmap v1779: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 3 peering, 113 active+clean; 7.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 14055/346146 objects misplaced (4.060%); 2.6 MiB/s, 30 objects/s recovering 2024-08-29T15:47:04.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:04 smithi027 ceph-mon[93921]: pgmap v1779: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 3 peering, 113 active+clean; 7.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 14055/346146 objects misplaced (4.060%); 2.6 MiB/s, 30 objects/s recovering 2024-08-29T15:47:04.710 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:04.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:04.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:04.716 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:04 smithi171 ceph-mon[90031]: pgmap v1779: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 remapped+peering, 3 peering, 113 active+clean; 7.0 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 14055/346146 objects misplaced (4.060%); 2.6 MiB/s, 30 objects/s recovering 2024-08-29T15:47:04.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:04.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:04.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:04.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:05.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:05.115 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:05.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:05.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:05.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:05.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:05.344 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:47:05.344 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:47:05.361 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:05.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:05.434 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:47:05.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:05.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:05.557 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:05.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1740760651' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:47:05.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:47:05.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1740760651' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:47:05.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:47:05.586 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:05.688 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:05.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:05.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:05.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1740760651' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:47:05.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:47:05.814 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:05.844 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:05.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:05.983 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:05.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:06.084 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:06.129 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:06.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:06.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-29T15:47:06.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:06.284 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:06.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:06.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:06.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:06.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:06.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[93921]: pgmap v1780: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 3 peering, 113 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 21248/343689 objects misplaced (6.182%); 2.1 MiB/s, 24 objects/s recovering 2024-08-29T15:47:06.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:47:06.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:47:06.438 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[93921]: osdmap e500: 8 total, 8 up, 8 in 2024-08-29T15:47:06.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[98970]: pgmap v1780: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 3 peering, 113 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 21248/343689 objects misplaced (6.182%); 2.1 MiB/s, 24 objects/s recovering 2024-08-29T15:47:06.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:47:06.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:47:06.439 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:06 smithi027 ceph-mon[98970]: osdmap e500: 8 total, 8 up, 8 in 2024-08-29T15:47:06.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:06.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:06.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:06.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:06.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:06.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:06.687 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:06.715 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:06.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:06 smithi171 ceph-mon[90031]: pgmap v1780: 119 pgs: 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 3 peering, 113 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 21248/343689 objects misplaced (6.182%); 2.1 MiB/s, 24 objects/s recovering 2024-08-29T15:47:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:06 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:47:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:06 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:47:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:06 smithi171 ceph-mon[90031]: osdmap e500: 8 total, 8 up, 8 in 2024-08-29T15:47:06.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:06.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:06.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:06.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:06.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:07.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:07.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:07.105 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:07.136 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:07.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:07.235 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:07.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:07.293 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:07.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:07.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:07.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:07.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:07.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:07.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:07.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:07.567 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:07.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:07.668 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:07.701 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:07.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:07.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:07.848 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:07.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:07.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:08.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:08.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:08.108 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:08.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:08.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:08.273 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:08.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:08.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:08.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.415 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:08.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:08.517 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:08.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:08.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:08.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:08.698 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:08.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:08.723 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:08 smithi027 ceph-mon[93921]: pgmap v1782: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 20279/333693 objects misplaced (6.077%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:47:08.724 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:08 smithi027 ceph-mon[98970]: pgmap v1782: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 20279/333693 objects misplaced (6.077%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:47:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:08 smithi171 ceph-mon[90031]: pgmap v1782: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 20279/333693 objects misplaced (6.077%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:47:08.800 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:08.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:08.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:08.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:08.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:09.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:09.099 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:09.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:09.201 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:09.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:09.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:09.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:09.313 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:47:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:47:09] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-08-29T15:47:09.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:09.363 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:09.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:09.465 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:09.490 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:09.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:09.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:09.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:09.629 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:09.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:09.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:09.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:09.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:09.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:09.863 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:09.896 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:09.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:09.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:10.039 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:10.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:10.141 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:10.168 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:10.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:10.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:10.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:10.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:10.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:10.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:10.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:10.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:10.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:10.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:10.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:47:10.591 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:10.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:10.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:10.741 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:10.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:10.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:10.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:10 smithi027 ceph-mon[98970]: pgmap v1783: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 19959/330939 objects misplaced (6.031%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:47:10.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:10 smithi027 ceph-mon[93921]: pgmap v1783: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 19959/330939 objects misplaced (6.031%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:47:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:10 smithi171 ceph-mon[90031]: pgmap v1783: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.7 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 19959/330939 objects misplaced (6.031%); 4.0 MiB/s, 64 objects/s recovering 2024-08-29T15:47:10.843 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:10.870 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:10.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:10.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:10.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:11.028 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:11.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.182 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:11.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:11.283 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:11.313 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:11.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:11.415 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:11.445 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:11.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:11.466 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.1 to 1 (10000) 2024-08-29T15:47:11.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:11.577 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:47:11.578 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:47:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:11.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:11.621 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:47:11.621 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:47:11.621 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/228720943' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:47:11.621 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:47:11.622 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/228720943' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:47:11.622 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:47:11.663 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:47:11.663 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:47:11.694 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:11.701 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:47:11.701 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:47:11.729 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:11.738 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:47:11.739 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:47:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/228720943' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:47:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:47:11.771 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:47:11.772 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:47:11.809 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:47:11.810 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:47:11.831 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:11.848 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:47:11.848 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:47:11.860 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:11.963 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:11.996 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:11.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:11.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:12.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:12.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:12.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:12.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:12.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:12.292 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:12.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:12.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:12.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:12.447 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:12.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:12.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:12.549 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:12.575 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:12.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:12.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:12.677 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:12.702 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:12.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:12.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[98970]: pgmap v1784: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 19959/325122 objects misplaced (6.139%); 3.3 MiB/s, 52 objects/s recovering 2024-08-29T15:47:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:47:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[98970]: osdmap e501: 8 total, 8 up, 8 in 2024-08-29T15:47:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3537145781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:12.729 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[98970]: osdmap e502: 8 total, 8 up, 8 in 2024-08-29T15:47:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[93921]: pgmap v1784: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 19959/325122 objects misplaced (6.139%); 3.3 MiB/s, 52 objects/s recovering 2024-08-29T15:47:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:47:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[93921]: osdmap e501: 8 total, 8 up, 8 in 2024-08-29T15:47:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3537145781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:12.730 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:12 smithi027 ceph-mon[93921]: osdmap e502: 8 total, 8 up, 8 in 2024-08-29T15:47:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:12 smithi171 ceph-mon[90031]: pgmap v1784: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 19959/325122 objects misplaced (6.139%); 3.3 MiB/s, 52 objects/s recovering 2024-08-29T15:47:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:12 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:47:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:12 smithi171 ceph-mon[90031]: osdmap e501: 8 total, 8 up, 8 in 2024-08-29T15:47:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3537145781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:12 smithi171 ceph-mon[90031]: osdmap e502: 8 total, 8 up, 8 in 2024-08-29T15:47:12.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:12.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:12.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:12.891 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:12.951 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:12.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:12.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:12.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:13.056 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:47:13.056 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:47:13.056 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:47:13.056 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:47:13.056 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:47:13.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:13.108 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:13.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:13.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:13.255 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:13.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:13.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:13.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:13.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:13.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:13.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:13.531 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:13.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:13.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:13.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:13.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:13.781 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:13.806 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:13 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:47:13.806 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:13 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:47:13.816 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:13.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:13.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:13.918 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:13.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:13.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:13.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:14.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:13 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:47:14.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:14.086 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:14.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:14.189 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:14.219 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:14.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:14.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:14.368 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:14.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:14.469 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:14.510 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:14.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:14.612 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:14.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:14.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:14.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:14.786 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:14.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:14 smithi027 ceph-mon[93921]: pgmap v1787: 119 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+remapped+backfill_wait, 112 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 19008/317016 objects misplaced (5.996%); 6.0 MiB/s, 95 objects/s recovering 2024-08-29T15:47:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:14.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:14 smithi027 ceph-mon[98970]: pgmap v1787: 119 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+remapped+backfill_wait, 112 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 19008/317016 objects misplaced (5.996%); 6.0 MiB/s, 95 objects/s recovering 2024-08-29T15:47:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:14.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:14.886 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:14.922 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:14.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:14.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:14 smithi171 ceph-mon[90031]: pgmap v1787: 119 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+remapped+backfill_wait, 112 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 19008/317016 objects misplaced (5.996%); 6.0 MiB/s, 95 objects/s recovering 2024-08-29T15:47:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:15.024 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:15.049 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:15.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:15.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:15.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:15.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:15.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:15.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:15.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:15.331 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:15.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:15.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:15.469 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:15.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:15.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 2 1 2024-08-29T15:47:15.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:15.659 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:15.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:15.761 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:15.791 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:15.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:15.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:47:15.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:15.933 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:15.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:16.033 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:16.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:16.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:16.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:16.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:16.199 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:16.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:16.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:16.331 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:16.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:16.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:16.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:16.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:16.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:16.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:16.625 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:16.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:16.726 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:16.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:16.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:16.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:16.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:16 smithi027 ceph-mon[93921]: pgmap v1788: 119 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+remapped+backfill_wait, 112 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 18680/314265 objects misplaced (5.944%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:47:16.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:16 smithi027 ceph-mon[98970]: pgmap v1788: 119 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+remapped+backfill_wait, 112 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 18680/314265 objects misplaced (5.944%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:47:16.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:16.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:16.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:16.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:17.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-29T15:47:17.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:17.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:16 smithi171 ceph-mon[90031]: pgmap v1788: 119 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+remapped+backfill_wait, 112 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 18680/314265 objects misplaced (5.944%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:47:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:17.089 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:17.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:17.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:17.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:17.255 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:17.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:17.356 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:17.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:17.389 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:17.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:17.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:17.530 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:17.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:17.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:17.690 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:17.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:17.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:17.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:17.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:17.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:17.939 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:17.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:17.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:17.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:18.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:17 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:47:18.018 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:17 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:47:18.019 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:17 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:47:18.045 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:18.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:18.086 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:18.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:18.172 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:18.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:18.273 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:18.302 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:18.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:18.398 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:18.404 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:18.429 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:18.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:18.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:18.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:18.574 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:18.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:18.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:18.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:18.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:18.807 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:18 smithi027 ceph-mon[98970]: pgmap v1789: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 17715/304104 objects misplaced (5.825%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:47:18.808 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:18 smithi027 ceph-mon[93921]: pgmap v1789: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 17715/304104 objects misplaced (5.825%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:47:18.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:18.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:18.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:18.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:18.998 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:18.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:18.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:19.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:18 smithi171 ceph-mon[90031]: pgmap v1789: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s; 17715/304104 objects misplaced (5.825%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:47:19.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:47:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:47:19] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-08-29T15:47:19.099 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:19.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:19.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:19.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:19.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:19.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:19.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:19.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:19.410 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:19.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:19.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:19.512 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:19.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:19.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:19.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:19.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:19.681 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:19.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:19.782 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:19.810 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:19.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:19.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:19.956 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:19.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:19.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:20.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:20.101 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:20.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:20.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:20.240 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:20.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:20.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:20.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:20.362 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:20.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:20.464 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:20.497 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:20.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:20.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:20.636 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:20.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:20.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:20.770 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:20.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:20.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:20.869 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:20 smithi027 ceph-mon[98970]: pgmap v1790: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 17422/301455 objects misplaced (5.779%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:47:20.870 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:20 smithi027 ceph-mon[93921]: pgmap v1790: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 17422/301455 objects misplaced (5.779%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:47:20.872 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:20.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:20.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:21.002 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:21.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:20 smithi171 ceph-mon[90031]: pgmap v1790: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 17422/301455 objects misplaced (5.779%); 6.0 MiB/s, 96 objects/s recovering 2024-08-29T15:47:21.036 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:21.138 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:21.159 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:21.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:21.299 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:21.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:21.401 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:21.428 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:21.530 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:21.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:21.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:21.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:21.700 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:21.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:21.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.839 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:21.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:21.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:21.982 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:21.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:22.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:22.112 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:22.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:22.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:22.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:22.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:22.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:22.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:22.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:22.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:22.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:22.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:22.519 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:22.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:22.620 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:22.655 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:22.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:22.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:22.794 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:22.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:22.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:22.938 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:22.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:23.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:22 smithi171 ceph-mon[90031]: pgmap v1791: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 17422/296055 objects misplaced (5.885%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:47:23.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:23.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:22 smithi027 ceph-mon[98970]: pgmap v1791: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 17422/296055 objects misplaced (5.885%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:47:23.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:22 smithi027 ceph-mon[93921]: pgmap v1791: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 6.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 17422/296055 objects misplaced (5.885%); 5.0 MiB/s, 79 objects/s recovering 2024-08-29T15:47:23.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:23.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:23.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:23.218 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:23.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:23.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:23.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:23.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:23.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:23.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:23.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:23.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:23.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:23.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:23.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:23.628 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:23.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:23.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:23.766 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:23.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:23.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:23.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:23.911 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:23.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:24.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:24.051 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:24.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:24.153 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:24.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:24.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:24.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:24.324 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:24.425 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:24.465 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:24.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:24.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:24.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:24.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:24.721 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:24.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:24.865 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:24.967 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:24.993 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:24.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:24.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:25.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:24 smithi171 ceph-mon[90031]: pgmap v1792: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 16499/287862 objects misplaced (5.732%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:47:25.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:24 smithi027 ceph-mon[98970]: pgmap v1792: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 16499/287862 objects misplaced (5.732%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:47:25.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:24 smithi027 ceph-mon[93921]: pgmap v1792: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 16499/287862 objects misplaced (5.732%); 4.8 MiB/s, 76 objects/s recovering 2024-08-29T15:47:25.095 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:25.127 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:25.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:25.229 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:25.258 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:25.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:25.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:25.390 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:25.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:25.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:25.534 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:25.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:25.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:25.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:25.679 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:25.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:25.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:25.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:25.820 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:25.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:25.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:25.922 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:25.953 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:25.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:26.054 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:26.086 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:26.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:26.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:26.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:26.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:26.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:26.362 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:26.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:26.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:26.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:26.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:26.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:26.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:26.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:26.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:26.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:26.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:26.772 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:26.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:26.875 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:26.903 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:26.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:27.005 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:27.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:26 smithi171 ceph-mon[90031]: pgmap v1793: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 16193/285036 objects misplaced (5.681%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:47:27.037 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:27.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:27.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:26 smithi027 ceph-mon[98970]: pgmap v1793: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 16193/285036 objects misplaced (5.681%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:47:27.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:26 smithi027 ceph-mon[93921]: pgmap v1793: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 16193/285036 objects misplaced (5.681%); 4.3 MiB/s, 68 objects/s recovering 2024-08-29T15:47:27.140 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:27.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:27.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:27.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:27.294 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:27.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:27.437 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:27.539 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:27.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:27.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:27.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:27.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:27.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:27.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:27.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:27.959 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:27.984 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:27.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:28.086 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:28.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:28.114 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:28.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:28.215 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:28.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:28.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:28.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:28.388 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:28.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:28.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:28.490 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:28.516 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:28.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:28.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:28.663 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:28.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:28.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:28.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:28.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:28.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:28.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:28.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:28.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:28.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:29.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:28 smithi027 ceph-mon[98970]: pgmap v1794: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 15617/277587 objects misplaced (5.626%); 6.3 MiB/s, 100 objects/s recovering 2024-08-29T15:47:29.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:28 smithi027 ceph-mon[93921]: pgmap v1794: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 15617/277587 objects misplaced (5.626%); 6.3 MiB/s, 100 objects/s recovering 2024-08-29T15:47:29.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:28 smithi171 ceph-mon[90031]: pgmap v1794: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 15617/277587 objects misplaced (5.626%); 6.3 MiB/s, 100 objects/s recovering 2024-08-29T15:47:29.048 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:29.077 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:29.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:29.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:29.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:29.221 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:29.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:29.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:29.286 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:47:29] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.51.0" 2024-08-29T15:47:29.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:29.356 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:29.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:29.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:29.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:29.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:29.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:29.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:29.636 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:29.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:29.738 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:29.757 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:47:29.764 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:29.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:29.866 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:29.891 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:29.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:29.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:29.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4041405941' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:47:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:29 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:47:30.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:29 smithi171 ceph-mon[90031]: pgmap v1795: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 15087/272238 objects misplaced (5.542%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:47:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:30.034 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:30.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:30.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4041405941' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[98970]: pgmap v1795: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 15087/272238 objects misplaced (5.542%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4041405941' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:47:30.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:47:30.071 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:29 smithi027 ceph-mon[93921]: pgmap v1795: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 15087/272238 objects misplaced (5.542%); 4.1 MiB/s, 65 objects/s recovering 2024-08-29T15:47:30.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:30.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:30.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:30.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:30.308 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:30.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:30.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:30.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:30.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:30.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:30.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:30.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:30.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:30.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:30.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:30.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:30.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:30.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:30.753 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.2 to 1 (10000) 2024-08-29T15:47:30.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:30.856 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:47:30.856 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:47:30.858 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:30.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:30.894 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:47:30.895 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:47:30.944 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:47:30.944 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:47:30.945 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:47:30.945 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:47:30.945 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:47:30.945 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[98970]: osdmap e503: 8 total, 8 up, 8 in 2024-08-29T15:47:30.945 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3146831040' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:47:30.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:47:30.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:47:30.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:47:30.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[93921]: osdmap e503: 8 total, 8 up, 8 in 2024-08-29T15:47:30.946 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3146831040' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:47:30.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:30.985 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:47:30.986 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:47:31.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.001 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:31.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:30 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:47:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:30 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:47:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:47:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:30 smithi171 ceph-mon[90031]: osdmap e503: 8 total, 8 up, 8 in 2024-08-29T15:47:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3146831040' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:47:31.018 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:47:31.018 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:47:31.054 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:47:31.054 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:47:31.089 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:47:31.089 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:47:31.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:31.128 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:47:31.128 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:47:31.141 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:31.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:31.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:31.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:31.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:31.411 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:31.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:31.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:31.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 3 1 2024-08-29T15:47:31.553 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:31.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:31.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:31.731 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:31.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:31.833 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:31.859 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:31.961 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:31.989 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:31.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:31.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3146831040' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:47:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:31 smithi171 ceph-mon[90031]: osdmap e504: 8 total, 8 up, 8 in 2024-08-29T15:47:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1411832638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:31 smithi171 ceph-mon[90031]: pgmap v1798: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 15087/266952 objects misplaced (5.652%); 3.0 MiB/s, 47 objects/s recovering 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3146831040' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[98970]: osdmap e504: 8 total, 8 up, 8 in 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1411832638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[98970]: pgmap v1798: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 15087/266952 objects misplaced (5.652%); 3.0 MiB/s, 47 objects/s recovering 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3146831040' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[93921]: osdmap e504: 8 total, 8 up, 8 in 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1411832638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:32.070 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:31 smithi027 ceph-mon[93921]: pgmap v1798: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 15087/266952 objects misplaced (5.652%); 3.0 MiB/s, 47 objects/s recovering 2024-08-29T15:47:32.091 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:32.117 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:32.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:32.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:32.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:32.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:32.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:32.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:32.394 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:32.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:32.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:32.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:32.528 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:32.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:32.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:32.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:32.672 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:32.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:32.775 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:32.805 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:32.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:32.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:32.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:32.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:32.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:32.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:33.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:33.094 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:33.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:33.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:33.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:33.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:33.341 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:33.364 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:47:33.364 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:47:33.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:33.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:33.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:33.372 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:33.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:33.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:33.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:33.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:33.653 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:33.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:33.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:33.798 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:33.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:33.900 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:33.925 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:33.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:34.027 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:34.054 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:34.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:34.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:34.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:34.191 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:34.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:34.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:34.293 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:34.334 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:34.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:34.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:34.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:34.463 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:34 smithi027 ceph-mon[98970]: pgmap v1799: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 14274/259464 objects misplaced (5.501%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:47:34.463 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:34 smithi027 ceph-mon[93921]: pgmap v1799: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 14274/259464 objects misplaced (5.501%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:47:34.476 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:34.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:34.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:34.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:34 smithi171 ceph-mon[90031]: pgmap v1799: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 14274/259464 objects misplaced (5.501%); 5.8 MiB/s, 92 objects/s recovering 2024-08-29T15:47:34.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:47:34.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:34.670 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:34.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:34.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:34.813 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:34.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:34.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:34.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:34.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:34.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:35.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:47:35.057 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:35.079 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:35.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:35.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:35.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:35.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:35.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:35.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:35.362 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:35.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:35.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:35.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:35.507 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:35.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:35.608 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:35.644 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:35.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:35.745 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:35.767 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:35.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:35.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:35.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:35.892 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:35.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:35.993 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:36.014 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:36.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:36.116 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:36.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:36.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:36.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:36.278 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:36.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:36.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:36.516 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:36 smithi171 ceph-mon[90031]: pgmap v1800: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 13933/256653 objects misplaced (5.429%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:47:36.537 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:36.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:36 smithi027 ceph-mon[93921]: pgmap v1800: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 13933/256653 objects misplaced (5.429%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:47:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:36 smithi027 ceph-mon[98970]: pgmap v1800: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 13933/256653 objects misplaced (5.429%); 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:47:36.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:36.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:36.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:36.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:36.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:36.911 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:36.932 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:36.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:36.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:37.034 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:37.055 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:37.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:37.157 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:37.185 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:37.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:37.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:37.287 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:37.312 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:37.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:37.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:37.443 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:37.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:37.545 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:37.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:37.595 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:37.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:37.696 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:37.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-29T15:47:37.720 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:37.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:37.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:37.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:37.878 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:37.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:37.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:37.980 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:38.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:38.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:38.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:38.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:38.144 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:38.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:38.246 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:38.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:38.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:38.278 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:38.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:38.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:38.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:38.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:38.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:38 smithi171 ceph-mon[90031]: pgmap v1801: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 13212/249912 objects misplaced (5.287%); 7.2 MiB/s, 114 objects/s recovering 2024-08-29T15:47:38.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:38 smithi027 ceph-mon[98970]: pgmap v1801: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 13212/249912 objects misplaced (5.287%); 7.2 MiB/s, 114 objects/s recovering 2024-08-29T15:47:38.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:38 smithi027 ceph-mon[93921]: pgmap v1801: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 13212/249912 objects misplaced (5.287%); 7.2 MiB/s, 114 objects/s recovering 2024-08-29T15:47:38.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:38.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:38.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:38.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:38.710 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:38.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:38.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:38.871 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:38.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:38.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:39.035 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:39.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:39.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:39.140 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:47:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:47:39] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.51.0" 2024-08-29T15:47:39.180 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:39.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:39.282 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:39.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:39.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:39.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:39.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:39.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:39.466 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:39.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:39.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:39.590 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:39.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:39.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:39.726 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:39.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:39.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:39.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:39.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:39.973 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:40.005 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:40.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:40.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:40.250 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:40.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:40.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:40.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:40.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:40.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:40 smithi171 ceph-mon[90031]: pgmap v1802: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 12670/244803 objects misplaced (5.176%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:47:40.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:40.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:40.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:40 smithi027 ceph-mon[98970]: pgmap v1802: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 12670/244803 objects misplaced (5.176%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:47:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:40 smithi027 ceph-mon[93921]: pgmap v1802: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 5.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 973 op/s; 12670/244803 objects misplaced (5.176%); 6.1 MiB/s, 97 objects/s recovering 2024-08-29T15:47:40.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:40.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.705 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:40.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:40.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:40.849 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:40.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:40.996 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:40.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:40.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:41.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:41.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:41.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:41.244 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:41.278 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:41.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:41.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:41.416 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:41.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:41.517 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:41.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:41.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:41.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:41.650 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:41.675 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:41.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:41.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:41.777 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:41.791 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:41.802 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:41.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:41.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:41.930 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:41.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:41.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:42.031 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:42.056 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:42.158 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:42.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.184 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:42.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:42.229 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.3 to 1 (10000) 2024-08-29T15:47:42.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:42.321 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:42.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:42.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:42.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:42 smithi171 ceph-mon[90031]: pgmap v1803: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 12670/240003 objects misplaced (5.279%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:47:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/769959468' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:47:42.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:42 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:47:42.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:42.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:42 smithi027 ceph-mon[98970]: pgmap v1803: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 12670/240003 objects misplaced (5.279%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:47:42.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/769959468' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:47:42.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:42 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:47:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:42 smithi027 ceph-mon[93921]: pgmap v1803: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 864 op/s; 12670/240003 objects misplaced (5.279%); 5.5 MiB/s, 88 objects/s recovering 2024-08-29T15:47:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/769959468' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:47:42.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:42 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:47:42.590 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:42.691 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:42.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:42.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:42.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.840 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:42.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:42.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 4 1 2024-08-29T15:47:42.941 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:42.964 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:42.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:43.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:43.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:43.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:43.142 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:47:43.142 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:47:43.163 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:47:43.163 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:47:43.181 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:47:43.181 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:47:43.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:43.202 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:47:43.202 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:47:43.219 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:43.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:43.221 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:47:43.221 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:47:43.241 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:47:43.241 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:47:43.268 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:47:43.269 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:47:43.294 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:47:43.294 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:47:43.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:43.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:43.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:43.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:43.453 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:43.477 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:43.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:47:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:43 smithi171 ceph-mon[90031]: osdmap e505: 8 total, 8 up, 8 in 2024-08-29T15:47:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:43 smithi171 ceph-mon[90031]: osdmap e506: 8 total, 8 up, 8 in 2024-08-29T15:47:43.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:43 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2705069496' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:43.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:47:43.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[93921]: osdmap e505: 8 total, 8 up, 8 in 2024-08-29T15:47:43.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[93921]: osdmap e506: 8 total, 8 up, 8 in 2024-08-29T15:47:43.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2705069496' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:43.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:47:43.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[98970]: osdmap e505: 8 total, 8 up, 8 in 2024-08-29T15:47:43.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[98970]: osdmap e506: 8 total, 8 up, 8 in 2024-08-29T15:47:43.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:43 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2705069496' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:47:43.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:43.619 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:43.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:43.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:43.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:43.765 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:43.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:43.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:43.897 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:43.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:43.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:43.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:44.030 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:44.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:44.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:44.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:44.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:44.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:44.260 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:47:44.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:44.322 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:44.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:44.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:44.424 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:44.453 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:44.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:44.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:44.532 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:44.555 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[93921]: pgmap v1806: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 11731/233916 objects misplaced (5.015%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[93921]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 504'10921 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[93921]: osdmap e507: 8 total, 8 up, 8 in 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2110446703' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[98970]: pgmap v1806: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 11731/233916 objects misplaced (5.015%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:47:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[98970]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 504'10921 2024-08-29T15:47:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[98970]: osdmap e507: 8 total, 8 up, 8 in 2024-08-29T15:47:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2110446703' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:47:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:44.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:44.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:44.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:44.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:44.722 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:44.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:44.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:44 smithi171 ceph-mon[90031]: pgmap v1806: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 4.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 11731/233916 objects misplaced (5.015%); 8.3 MiB/s, 132 objects/s recovering 2024-08-29T15:47:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:44 smithi171 ceph-mon[90031]: 10.3 starting backfill to osd.3 from (0'0,0'0] MAX to 504'10921 2024-08-29T15:47:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:44 smithi171 ceph-mon[90031]: osdmap e507: 8 total, 8 up, 8 in 2024-08-29T15:47:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2110446703' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:47:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:44.824 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:44.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:44.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:44.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:45.013 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:45.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:45.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:45.276 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.307 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:45.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:45.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:45.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:45.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:45.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:45 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:47:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2110446703' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:47:45.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:45 smithi027 ceph-mon[98970]: osdmap e508: 8 total, 8 up, 8 in 2024-08-29T15:47:45.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:45 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:47:45.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2110446703' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:47:45.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:45 smithi027 ceph-mon[93921]: osdmap e508: 8 total, 8 up, 8 in 2024-08-29T15:47:45.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.581 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:45.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:45.684 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.711 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:45.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:45 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:47:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2110446703' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:47:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:45 smithi171 ceph-mon[90031]: osdmap e508: 8 total, 8 up, 8 in 2024-08-29T15:47:45.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:45.847 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:45.949 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:45.981 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:45.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:46.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:46.125 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:46.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:46.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:46.269 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:46.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:46.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:46.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:46.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:46.474 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:46 smithi027 ceph-mon[93921]: pgmap v1809: 119 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 16209/230355 objects misplaced (7.037%); 8.4 MiB/s, 134 objects/s recovering 2024-08-29T15:47:46.474 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:46 smithi027 ceph-mon[93921]: osdmap e509: 8 total, 8 up, 8 in 2024-08-29T15:47:46.475 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:46 smithi027 ceph-mon[98970]: pgmap v1809: 119 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 16209/230355 objects misplaced (7.037%); 8.4 MiB/s, 134 objects/s recovering 2024-08-29T15:47:46.475 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:46 smithi027 ceph-mon[98970]: osdmap e509: 8 total, 8 up, 8 in 2024-08-29T15:47:46.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:46.539 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:46.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:46.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:46.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:46.685 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:46.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:46.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:46 smithi171 ceph-mon[90031]: pgmap v1809: 119 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 4.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 16209/230355 objects misplaced (7.037%); 8.4 MiB/s, 134 objects/s recovering 2024-08-29T15:47:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:46 smithi171 ceph-mon[90031]: osdmap e509: 8 total, 8 up, 8 in 2024-08-29T15:47:46.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:46.828 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:46.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:46.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:46.972 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:46.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:47.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:47.103 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:47.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:47.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:47:47.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:47.234 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:47.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:47.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:47.375 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:47.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:47.477 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:47.515 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:47.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:47 smithi027 ceph-mon[98970]: osdmap e510: 8 total, 8 up, 8 in 2024-08-29T15:47:47.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:47 smithi027 ceph-mon[93921]: osdmap e510: 8 total, 8 up, 8 in 2024-08-29T15:47:47.617 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:47.645 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:47.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:47.747 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:47 smithi171 ceph-mon[90031]: osdmap e510: 8 total, 8 up, 8 in 2024-08-29T15:47:47.775 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:47.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:47.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:47:47.878 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:47.916 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:47.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:47.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:48.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:48.045 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:48.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:48.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:48.180 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:48.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:48.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:48.338 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:48.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:48.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:48.477 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:48.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:48.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:48 smithi027 ceph-mon[98970]: pgmap v1812: 119 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 15364/224286 objects misplaced (6.850%); 13 MiB/s, 203 objects/s recovering 2024-08-29T15:47:48.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:48 smithi027 ceph-mon[93921]: pgmap v1812: 119 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 15364/224286 objects misplaced (6.850%); 13 MiB/s, 203 objects/s recovering 2024-08-29T15:47:48.578 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:48.611 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:48.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:48.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:48.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:48.754 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:48 smithi171 ceph-mon[90031]: pgmap v1812: 119 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 114 active+clean; 4.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 802 op/s; 15364/224286 objects misplaced (6.850%); 13 MiB/s, 203 objects/s recovering 2024-08-29T15:47:48.783 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:48.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:48.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:48.931 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:48.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:49.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:49.083 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:49.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:49.184 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:49.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:49.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:47:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:47:49] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-29T15:47:49.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:49.357 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:49.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:49.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:49.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:49.488 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:49.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:49.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:49.589 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:49.616 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:49.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:49.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:49.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:49.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:49.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:49.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:49.862 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:49.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:49.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:49.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:49.999 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:50.023 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:50.125 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:50.155 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:50.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:50.257 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:50.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:50.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.422 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:50.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:50.523 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:50.557 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:50.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:50.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:50 smithi171 ceph-mon[90031]: pgmap v1813: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 13873/219396 objects misplaced (6.323%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:47:50.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:50.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:50 smithi027 ceph-mon[93921]: pgmap v1813: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 13873/219396 objects misplaced (6.323%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:47:50.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:50 smithi027 ceph-mon[98970]: pgmap v1813: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 13873/219396 objects misplaced (6.323%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:47:50.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:50.948 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:50.980 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:50.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:51.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:51.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:51.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:51.219 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:51.252 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:51.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:51.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:51.358 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:47:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:51.406 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:51.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:51.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:51.555 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:51.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:51.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:51.697 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:51.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:51.799 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:51.841 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:51.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:51.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:51.943 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:51.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:51.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:51.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:52.072 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:52.094 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:52.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:52.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:52.196 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:52.228 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:52.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:52.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:52.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:52.357 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:52.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:52.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:52.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:52.487 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:52.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:52.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:52.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:52.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:52.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:52 smithi027 ceph-mon[93921]: pgmap v1814: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 13873/213978 objects misplaced (6.483%); 13 MiB/s, 210 objects/s recovering 2024-08-29T15:47:52.664 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:52 smithi027 ceph-mon[98970]: pgmap v1814: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 13873/213978 objects misplaced (6.483%); 13 MiB/s, 210 objects/s recovering 2024-08-29T15:47:52.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:52.763 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:52.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:52.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:52 smithi171 ceph-mon[90031]: pgmap v1814: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 13873/213978 objects misplaced (6.483%); 13 MiB/s, 210 objects/s recovering 2024-08-29T15:47:52.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:52.903 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:52.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:53.004 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:53.035 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:53.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:53.137 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:53.170 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:53.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:53.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:53.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:53.305 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:53.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:53.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:53.458 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:53.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:53.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:53.592 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:53.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:53.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:53.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:53.733 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:53.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:47:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/100640994' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:47:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:53 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:47:53.801 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.4 to 1 (10000) 2024-08-29T15:47:53.801 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:47:53.801 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/100640994' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:47:53.802 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:53 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:47:53.802 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:47:53.802 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/100640994' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:47:53.802 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:53 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:47:53.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:53.866 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:53.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:53.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:53.968 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:54.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:54.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:54.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:54.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:54.277 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:54.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:54.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.433 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:54.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:54.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:54 smithi171 ceph-mon[90031]: pgmap v1815: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 13176/207924 objects misplaced (6.337%); 14 MiB/s, 230 objects/s recovering 2024-08-29T15:47:54.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:54 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:47:54.518 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:54 smithi171 ceph-mon[90031]: osdmap e511: 8 total, 8 up, 8 in 2024-08-29T15:47:54.534 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:54.574 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:54.676 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:54.706 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:54.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:54 smithi027 ceph-mon[98970]: pgmap v1815: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 13176/207924 objects misplaced (6.337%); 14 MiB/s, 230 objects/s recovering 2024-08-29T15:47:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:54 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:47:54.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:54 smithi027 ceph-mon[98970]: osdmap e511: 8 total, 8 up, 8 in 2024-08-29T15:47:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:54 smithi027 ceph-mon[93921]: pgmap v1815: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 13176/207924 objects misplaced (6.337%); 14 MiB/s, 230 objects/s recovering 2024-08-29T15:47:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:54 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:47:54.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:54 smithi027 ceph-mon[93921]: osdmap e511: 8 total, 8 up, 8 in 2024-08-29T15:47:54.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:54.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:54.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:54.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:55.084 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:55.118 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:55.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:55.219 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:55.259 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:55.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:55.360 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:55.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:55.395 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:55.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:55.495 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:55.526 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:55.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:55.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:55.628 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:55.662 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:55.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:55.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:55.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 7 1 2024-08-29T15:47:55.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:55.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:55.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:55.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:55.894 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:55.921 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:55.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:56.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:56.088 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:56.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:56.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:56.241 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:56.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:56.344 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:56.371 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:56.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:56.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:56.498 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:56.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:56.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:56.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:56.639 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:56.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:56.740 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:56.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:56 smithi171 ceph-mon[90031]: pgmap v1817: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 12026/203901 objects misplaced (5.898%); 18 MiB/s, 285 objects/s recovering 2024-08-29T15:47:56.778 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:56.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:56.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:47:56.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:56 smithi027 ceph-mon[98970]: pgmap v1817: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 12026/203901 objects misplaced (5.898%); 18 MiB/s, 285 objects/s recovering 2024-08-29T15:47:56.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:56 smithi027 ceph-mon[93921]: pgmap v1817: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 12026/203901 objects misplaced (5.898%); 18 MiB/s, 285 objects/s recovering 2024-08-29T15:47:56.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:56.923 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:56.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:56.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:57.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:57.069 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:57.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:57.171 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:57.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:57.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:57.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:57.303 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:57.348 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:57.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:57.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:57.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:57.491 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:57.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:57.593 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:47:57.620 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:57.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:47:57.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:57.761 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:57.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:57.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:57.909 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:57.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:58.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:58.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:58.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:58.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:58.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:58.193 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:58.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:58.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:58.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:58.350 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:58.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:58.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:47:58.493 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:58.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:58.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:47:58.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:58.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:58.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:58.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:58 smithi171 ceph-mon[90031]: pgmap v1818: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11617/197424 objects misplaced (5.884%); 13 MiB/s, 214 objects/s recovering 2024-08-29T15:47:58.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:58.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:58 smithi027 ceph-mon[98970]: pgmap v1818: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11617/197424 objects misplaced (5.884%); 13 MiB/s, 214 objects/s recovering 2024-08-29T15:47:58.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:58 smithi027 ceph-mon[93921]: pgmap v1818: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11617/197424 objects misplaced (5.884%); 13 MiB/s, 214 objects/s recovering 2024-08-29T15:47:58.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:47:58.911 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:58.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:58.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:47:59.013 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:47:59.041 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:59.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:59.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:47:59.143 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:47:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:59.178 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:59.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:47:59.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:59.293 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:47:59] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-29T15:47:59.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:59.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:59.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:59.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:59.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:59.461 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:59.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:59.562 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:47:59.614 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:59.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:47:59.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:47:59.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:59.759 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:47:59.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:47:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[98970]: pgmap v1819: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 10111/192237 objects misplaced (5.260%); 14 MiB/s, 220 objects/s recovering 2024-08-29T15:47:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:59.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:59.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:47:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[93921]: pgmap v1819: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 10111/192237 objects misplaced (5.260%); 14 MiB/s, 220 objects/s recovering 2024-08-29T15:47:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:59.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:47:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:47:59.860 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:190900 10.b 10:d3671fad:::benchmark_data_smithi027_374610_object64770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.863804+0000", 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.011383182, 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011415751, 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.878 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190900 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.863804+0000", 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.863804+0000", 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.863805+0000", 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0780000000000001e-06 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.863809+0000", 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6150000000000001e-06 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.863818+0000", 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4259999999999992e-06 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.863832+0000", 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3658999999999999e-05 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864218+0000", 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038588700000000001 2024-08-29T15:47:59.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864337+0000", 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119264 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864447+0000", 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010990800000000001 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:190903 10.b 10:de6ad2f6:::benchmark_data_smithi027_374610_object64773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.864200+0000", 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.01098762, 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01108926, 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190903 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.883 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864200+0000", 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864200+0000", 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864201+0000", 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.06e-06 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864205+0000", 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8559999999999998e-06 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864220+0000", 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4381e-05 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864228+0000", 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3250000000000008e-06 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870990+0000", 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006761375 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.871309+0000", 2024-08-29T15:47:59.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031938799999999999 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.871703+0000", 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000394256 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:190904 10.b 10.acad2bdb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e512)", 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.867891+0000", 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0072959649999999997, 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074558410000000004, 2024-08-29T15:47:59.887 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190904 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867891+0000", 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867891+0000", 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867892+0000", 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.206e-06 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867896+0000", 2024-08-29T15:47:59.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0740000000000001e-06 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867911+0000", 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5739e-05 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867923+0000", 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1691e-05 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:190905 10.2 10:458187c0:::benchmark_data_smithi027_374610_object64775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.867944+0000", 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0072428179999999998, 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074538219999999997, 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190905 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.891 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867944+0000", 2024-08-29T15:47:59.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867944+0000", 2024-08-29T15:47:59.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867945+0000", 2024-08-29T15:47:59.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1999999999999998e-07 2024-08-29T15:47:59.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867949+0000", 2024-08-29T15:47:59.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9670000000000003e-06 2024-08-29T15:47:59.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867959+0000", 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4429999999999992e-06 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.867966+0000", 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0790000000000003e-06 2024-08-29T15:47:59.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.868787+0000", 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00082165899999999997 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.869118+0000", 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000330754 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.869540+0000", 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042209700000000002 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.47036.0:190900 10.b e512/480)", 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.870898+0000", 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0042891259999999999, 2024-08-29T15:47:59.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045613119999999997, 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870898+0000", 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870898+0000", 2024-08-29T15:47:59.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870899+0000", 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0640000000000001e-06 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870904+0000", 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2820000000000004e-06 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870930+0000", 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5931e-05 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870939+0000", 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8019999999999998e-06 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.47036.0:190900 10.b e512/480)", 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.873103+0000", 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0020837809999999998, 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023934310000000001, 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.873103+0000", 2024-08-29T15:47:59.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.873103+0000", 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.873104+0000", 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2200000000000003e-07 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.873109+0000", 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2799999999999997e-06 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.873124+0000", 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5085e-05 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.873131+0000", 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0840000000000003e-06 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.903 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.47036.0:190905 10.2 e512/480)", 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.875433+0000", 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.999754, 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010283100000000001, 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.875433+0000", 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.875433+0000", 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.875434+0000", 2024-08-29T15:47:59.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9300000000000006e-07 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.875439+0000", 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2700000000000004e-06 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.875455+0000", 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5812e-05 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.875462+0000", 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2699999999999999e-06 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:190881 10.b 10:d00927fe:::benchmark_data_smithi027_374610_object64751:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:59.842885+0000", 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.032302399000000002, 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.032695947000000003, 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:47:59.907 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 190881 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.842885+0000", 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.842885+0000", 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.842886+0000", 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4810000000000001e-06 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.842891+0000", 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.566e-06 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.842908+0000", 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7535999999999999e-05 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.842917+0000", 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6600000000000001e-06 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.857362+0000", 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014444914 2024-08-29T15:47:59.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.857536+0000", 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017428300000000001 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.857678+0000", 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141323 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:47:59.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.864188+0000", 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0065103619999999996 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:59.870824+0000", 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066360200000000003 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 8 2024-08-29T15:47:59.912 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:47:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:47:59.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:47:59.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:00.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:59 smithi171 ceph-mon[90031]: pgmap v1819: 119 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 115 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 10111/192237 objects misplaced (5.260%); 14 MiB/s, 220 objects/s recovering 2024-08-29T15:48:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:47:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:00.047 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:00.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:00.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.185 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:00.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:00.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:00.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.310 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:00.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:00.411 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:00.434 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:00.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:00.564 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:00.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:00.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:00.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:00.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:00.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:00.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:00.951 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:00.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:01.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:00 smithi171 ceph-mon[90031]: osdmap e512: 8 total, 8 up, 8 in 2024-08-29T15:48:01.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:01.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:00 smithi027 ceph-mon[98970]: osdmap e512: 8 total, 8 up, 8 in 2024-08-29T15:48:01.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:00 smithi027 ceph-mon[93921]: osdmap e512: 8 total, 8 up, 8 in 2024-08-29T15:48:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:01.097 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:01.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:01.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:01.226 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:01.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:01.328 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:01.352 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:01.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:01.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:01.480 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:01.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:01.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:01.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:01.623 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:01.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:01.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:01.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:01.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:01.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:01.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:01.895 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:01.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:01.996 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:02.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:02.031 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:02.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:01 smithi027 ceph-mon[98970]: osdmap e513: 8 total, 8 up, 8 in 2024-08-29T15:48:02.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:01 smithi027 ceph-mon[98970]: pgmap v1822: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 9863/187167 objects misplaced (5.270%); 17 MiB/s, 264 objects/s recovering 2024-08-29T15:48:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:01 smithi027 ceph-mon[93921]: osdmap e513: 8 total, 8 up, 8 in 2024-08-29T15:48:02.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:01 smithi027 ceph-mon[93921]: pgmap v1822: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 9863/187167 objects misplaced (5.270%); 17 MiB/s, 264 objects/s recovering 2024-08-29T15:48:02.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:02.168 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:02.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:01 smithi171 ceph-mon[90031]: osdmap e513: 8 total, 8 up, 8 in 2024-08-29T15:48:02.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:01 smithi171 ceph-mon[90031]: pgmap v1822: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 9863/187167 objects misplaced (5.270%); 17 MiB/s, 264 objects/s recovering 2024-08-29T15:48:02.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:02.312 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:02.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:02.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:02.414 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:02.437 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:02.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:02.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:02.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:02.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:02.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:02.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:02.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:02.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:02.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:02.867 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:02.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:02.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:02.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.012 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:03.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:03.113 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:03.138 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:03.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:03.241 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:03.266 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:03.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:03.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:03.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:03.513 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:03.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:03.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:03.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:03.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:03.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:03.845 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:03.947 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:03.981 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:03.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:04.083 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:04.119 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:04.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:04.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:04.231 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:48:04.260 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:04.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:04.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:04.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:04.409 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:04.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:04.510 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:04 smithi171 ceph-mon[90031]: pgmap v1823: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 9657/182958 objects misplaced (5.278%); 11 MiB/s, 172 objects/s recovering 2024-08-29T15:48:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:04 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1923913360' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:04 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:04.540 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:04.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:04.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:04.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:04 smithi027 ceph-mon[93921]: pgmap v1823: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 9657/182958 objects misplaced (5.278%); 11 MiB/s, 172 objects/s recovering 2024-08-29T15:48:04.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:04 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1923913360' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:04.544 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:04 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:04.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:04 smithi027 ceph-mon[98970]: pgmap v1823: 119 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 116 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 9657/182958 objects misplaced (5.278%); 11 MiB/s, 172 objects/s recovering 2024-08-29T15:48:04.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:04 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1923913360' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:04.544 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:04 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:04.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-29T15:48:04.642 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:04.669 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:04.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:04.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:04.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:04.796 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:04.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:04.898 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:04.928 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:04.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:04.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:05.030 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:05.059 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:05.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:05.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:05.065 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:05.157 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:48:05.157 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:48:05.161 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:05.190 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:05.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:05.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:48:05.198 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:48:05.245 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:48:05.245 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:48:05.282 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:48:05.282 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:48:05.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:05.316 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:48:05.316 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:48:05.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:05.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:05.355 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:48:05.355 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:48:05.391 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:48:05.391 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:48:05.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:05.426 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:48:05.426 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:48:05.454 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:48:05.455 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:48:05.455 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[93921]: osdmap e514: 8 total, 8 up, 8 in 2024-08-29T15:48:05.455 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2087687324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:05.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:48:05.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:48:05.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[98970]: osdmap e514: 8 total, 8 up, 8 in 2024-08-29T15:48:05.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:05 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2087687324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:05.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:05.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:05 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:05 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:05 smithi171 ceph-mon[90031]: osdmap e514: 8 total, 8 up, 8 in 2024-08-29T15:48:05.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:05 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2087687324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:05.609 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:05.639 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:05.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:05.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:05.741 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:05.771 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:05.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:05.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:05.872 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:05.911 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:05.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:06.013 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:06.040 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:06.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:06.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:06.142 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:06.177 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:06.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:06.278 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:06.320 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:06.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:06.421 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:06.457 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:06.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:06.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:06 smithi171 ceph-mon[90031]: pgmap v1825: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7756/179385 objects misplaced (4.324%); 19 MiB/s, 311 objects/s recovering 2024-08-29T15:48:06.559 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:06.562 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:06.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:06 smithi027 ceph-mon[93921]: pgmap v1825: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7756/179385 objects misplaced (4.324%); 19 MiB/s, 311 objects/s recovering 2024-08-29T15:48:06.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:06 smithi027 ceph-mon[98970]: pgmap v1825: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7756/179385 objects misplaced (4.324%); 19 MiB/s, 311 objects/s recovering 2024-08-29T15:48:06.583 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:06.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:06.685 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:06.719 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:06.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:06.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:06.862 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:06.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:06.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:07.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:07.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:07.143 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:07.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:07.288 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:07.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:07.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:07.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:07.535 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:07.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:07.665 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:07.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:07.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:07.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:07.846 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:07.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:07.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:07.992 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:07.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:08.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:08.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:08.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:08.238 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:08.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:08.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:08.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:08.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:48:08.380 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:08.398 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:08 smithi027 ceph-mon[93921]: pgmap v1826: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 7756/172566 objects misplaced (4.495%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:48:08.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:08 smithi027 ceph-mon[98970]: pgmap v1826: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 7756/172566 objects misplaced (4.495%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:48:08.411 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:08.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:08.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:08.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:08 smithi171 ceph-mon[90031]: pgmap v1826: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 7756/172566 objects misplaced (4.495%); 16 MiB/s, 253 objects/s recovering 2024-08-29T15:48:08.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:08.565 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:08.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:08.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:08.705 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:08.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:08.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:08.807 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:08.837 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:08.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:08.938 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:08.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:08.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:09.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:48:09.069 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:09.107 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:09.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:09.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:09.209 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:09.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:09.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:09.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:09.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:48:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:48:09] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-29T15:48:09.342 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:09.382 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:09.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:09.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:09.527 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:09.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:09.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:09.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:09.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:09.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:09.807 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:09.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:09.908 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:09.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:09.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:10.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:10.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:10.087 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:10.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:10.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:10.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:10.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:10.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:10.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:10.352 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:10.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:10.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:10.490 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:10.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:10 smithi171 ceph-mon[90031]: pgmap v1827: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 6210/167094 objects misplaced (3.716%); 21 MiB/s, 337 objects/s recovering 2024-08-29T15:48:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:10 smithi027 ceph-mon[93921]: pgmap v1827: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 6210/167094 objects misplaced (3.716%); 21 MiB/s, 337 objects/s recovering 2024-08-29T15:48:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:10 smithi027 ceph-mon[98970]: pgmap v1827: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 6210/167094 objects misplaced (3.716%); 21 MiB/s, 337 objects/s recovering 2024-08-29T15:48:10.591 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:10.617 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:10.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:10.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:10.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:10.758 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:10.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:10.859 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:10.886 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:10.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:10.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:11.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:11.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:11.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.159 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:11.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:11.260 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:11.291 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:11.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:11.393 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:11.414 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:11.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:11.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:11.541 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:11.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:11.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:11 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/541541919' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:48:11.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:11 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/541541919' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:48:11.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:11.668 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:11.681 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.7 to 1 (10000) 2024-08-29T15:48:11.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:11 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/541541919' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:48:11.770 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:11.792 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:11.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:11.918 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:11.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:12.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:12.043 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:12.145 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:12.167 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:12.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:12.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:12.296 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:12.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:12.398 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:12.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:12.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[98970]: pgmap v1828: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6210/160812 objects misplaced (3.862%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/541541919' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[98970]: osdmap e515: 8 total, 8 up, 8 in 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[98970]: osdmap e516: 8 total, 8 up, 8 in 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[93921]: pgmap v1828: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6210/160812 objects misplaced (3.862%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/541541919' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[93921]: osdmap e515: 8 total, 8 up, 8 in 2024-08-29T15:48:12.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:12 smithi027 ceph-mon[93921]: osdmap e516: 8 total, 8 up, 8 in 2024-08-29T15:48:12.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:12.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:12.675 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:12.706 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:12.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:12 smithi171 ceph-mon[90031]: pgmap v1828: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 6210/160812 objects misplaced (3.862%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/541541919' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:48:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:12 smithi171 ceph-mon[90031]: osdmap e515: 8 total, 8 up, 8 in 2024-08-29T15:48:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:12 smithi171 ceph-mon[90031]: osdmap e516: 8 total, 8 up, 8 in 2024-08-29T15:48:12.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:12.830 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:12.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:12.931 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:12.965 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:12.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:12.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:13.066 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:13.103 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:13.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:13.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:13.243 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:13.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:13.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:13.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:13.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:13.502 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:13.531 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:13.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:13.633 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:13.660 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:13.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:13.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:13.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:13.811 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:13.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:13.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 5 1 2024-08-29T15:48:13.912 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:13.938 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:13.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:14.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:14.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:14.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:14.183 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:14.210 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:14.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:14.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:14.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:14.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:14.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:14.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:14.479 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:14.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:14.497 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:14.508 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:14.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:14 smithi027 ceph-mon[93921]: pgmap v1831: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 2 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 5971/156273 objects misplaced (3.821%); 7.5 MiB/s, 119 objects/s recovering 2024-08-29T15:48:14.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:14 smithi027 ceph-mon[93921]: osdmap e517: 8 total, 8 up, 8 in 2024-08-29T15:48:14.574 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:14.574 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:14 smithi027 ceph-mon[98970]: pgmap v1831: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 2 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 5971/156273 objects misplaced (3.821%); 7.5 MiB/s, 119 objects/s recovering 2024-08-29T15:48:14.575 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:14 smithi027 ceph-mon[98970]: osdmap e517: 8 total, 8 up, 8 in 2024-08-29T15:48:14.575 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:14.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:14.642 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:14.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:14.744 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:14 smithi171 ceph-mon[90031]: pgmap v1831: 119 pgs: 1 active+remapped+backfill_wait, 2 peering, 2 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 5971/156273 objects misplaced (3.821%); 7.5 MiB/s, 119 objects/s recovering 2024-08-29T15:48:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:14 smithi171 ceph-mon[90031]: osdmap e517: 8 total, 8 up, 8 in 2024-08-29T15:48:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:14.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:14.772 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:14.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:14.873 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:14.900 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:14.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:15.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:15.038 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:15.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:15.183 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:15.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:15.327 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:15.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:15.429 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:15.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:15.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:15.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:15.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:15 smithi027 ceph-mon[93921]: osdmap e518: 8 total, 8 up, 8 in 2024-08-29T15:48:15.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:15 smithi027 ceph-mon[93921]: osdmap e519: 8 total, 8 up, 8 in 2024-08-29T15:48:15.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:15 smithi027 ceph-mon[98970]: osdmap e518: 8 total, 8 up, 8 in 2024-08-29T15:48:15.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:15 smithi027 ceph-mon[98970]: osdmap e519: 8 total, 8 up, 8 in 2024-08-29T15:48:15.580 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:15.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:15.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:15.712 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:15.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:15 smithi171 ceph-mon[90031]: osdmap e518: 8 total, 8 up, 8 in 2024-08-29T15:48:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:15 smithi171 ceph-mon[90031]: osdmap e519: 8 total, 8 up, 8 in 2024-08-29T15:48:15.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:15.856 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:15.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:15.987 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:15.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:15.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:16.089 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:16.110 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:16.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:16.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:16.212 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:16.233 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:16.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:16.335 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:16.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:16.366 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:16.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:16.467 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:16.493 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:16.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:16.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:16 smithi027 ceph-mon[93921]: pgmap v1834: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 114 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 4795/152757 objects misplaced (3.139%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:16.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:16 smithi027 ceph-mon[98970]: pgmap v1834: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 114 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 4795/152757 objects misplaced (3.139%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:16.595 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:16.628 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:16.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:16.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:16.730 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:16.759 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:16.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:16 smithi171 ceph-mon[90031]: pgmap v1834: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 114 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 4795/152757 objects misplaced (3.139%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:16.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:16.898 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:16.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:16.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:17.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:17.027 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:17.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:17.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:17.164 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:17.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:17.266 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:17.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:17.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:17.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:17.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:17.421 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:17.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:17.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:17.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:17.571 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:17.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:17.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:17.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:17.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:17.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:17.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:17.870 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:17.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:17.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:17.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:18.015 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:18.117 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:18.141 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:18.212 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:48:18.213 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:48:18.243 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:18.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:18.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:18.398 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:18.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:18.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:18.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:18 smithi027 ceph-mon[93921]: pgmap v1836: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 114 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4795/146985 objects misplaced (3.262%); 13 MiB/s, 211 objects/s recovering 2024-08-29T15:48:18.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:18 smithi027 ceph-mon[98970]: pgmap v1836: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 114 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4795/146985 objects misplaced (3.262%); 13 MiB/s, 211 objects/s recovering 2024-08-29T15:48:18.645 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.677 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:18.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:18 smithi171 ceph-mon[90031]: pgmap v1836: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 114 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4795/146985 objects misplaced (3.262%); 13 MiB/s, 211 objects/s recovering 2024-08-29T15:48:18.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:18.822 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:18.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:18.967 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:18.968 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:18.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:19.016 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:48:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:48:19] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-08-29T15:48:19.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:19.113 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:19.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:19.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:19.251 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:19.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:19.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:19.394 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:19.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:19.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:19.496 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:19.524 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:19.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:19.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:19.662 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:19.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:19.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:19.810 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:19.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:19.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:19.952 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:19.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:19.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:20.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:20.092 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:20.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:20.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:20.229 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:20.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:20.331 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:20.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:20.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:20.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:20.513 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:20.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:20.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:20.561 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:20 smithi027 ceph-mon[98970]: pgmap v1837: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 peering, 116 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3943/142047 objects misplaced (2.776%); 16 MiB/s, 254 objects/s recovering 2024-08-29T15:48:20.562 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:20 smithi027 ceph-mon[93921]: pgmap v1837: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 peering, 116 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3943/142047 objects misplaced (2.776%); 16 MiB/s, 254 objects/s recovering 2024-08-29T15:48:20.615 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:20.645 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:20.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:20.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:20 smithi171 ceph-mon[90031]: pgmap v1837: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 1 peering, 116 active+clean; 2.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 3943/142047 objects misplaced (2.776%); 16 MiB/s, 254 objects/s recovering 2024-08-29T15:48:20.785 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:20.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:20.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:20.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:20.927 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:20.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:21.029 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:21.061 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:21.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:21.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-29T15:48:21.162 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:21.189 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:21.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:21.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:21.291 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:21.315 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:21.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:21.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:21.420 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:48:21.420 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:48:21.452 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:48:21.452 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:48:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:21.471 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:21.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:21.488 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:48:21.488 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:48:21.530 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:48:21.530 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:48:21.562 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:48:21.562 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:48:21.572 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:21.598 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:48:21.598 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:48:21.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:21.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:21.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:21.633 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:48:21.633 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:48:21.664 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:48:21.664 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:48:21.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:21.748 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:21.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:21.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:21.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:21.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:21.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:21.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:21.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:22.012 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:22.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:22.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:22.166 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:22.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:22.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:22.268 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:22.300 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:22.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:22.402 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:22.432 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:22.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:22.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:22.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:22.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:22 smithi027 ceph-mon[93921]: pgmap v1838: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 3943/136239 objects misplaced (2.894%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:48:22.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3619387761' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:22.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:22 smithi027 ceph-mon[98970]: pgmap v1838: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 3943/136239 objects misplaced (2.894%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:48:22.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3619387761' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:22.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:22.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:22.681 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:22.707 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:22.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:22.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:22 smithi171 ceph-mon[90031]: pgmap v1838: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 3943/136239 objects misplaced (2.894%); 12 MiB/s, 193 objects/s recovering 2024-08-29T15:48:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3619387761' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:22.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:22.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:22.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:22.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:23.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:23.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:23.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:23.138 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:23.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:23.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:23.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:23.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:23.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:23.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:23.434 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:23.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:23.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:23.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:23.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:23.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:23.726 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:23.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:23.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:23.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:23.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:23.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:23.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:24.008 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:24.110 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:24.137 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:24.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:24.239 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:24.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:24.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:24.395 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:24 smithi027 ceph-mon[93921]: pgmap v1839: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 3652/130644 objects misplaced (2.795%); 3.5 MiB/s, 56 objects/s recovering 2024-08-29T15:48:24.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:24 smithi027 ceph-mon[98970]: pgmap v1839: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 3652/130644 objects misplaced (2.795%); 3.5 MiB/s, 56 objects/s recovering 2024-08-29T15:48:24.412 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.430 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:24.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:24.531 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:24.560 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:24.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:24.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:24.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:48:24.717 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:24 smithi171 ceph-mon[90031]: pgmap v1839: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 3652/130644 objects misplaced (2.795%); 3.5 MiB/s, 56 objects/s recovering 2024-08-29T15:48:24.819 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:24.857 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:24.959 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:24.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:24.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:25.094 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:25.125 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:25.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:25.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:25.227 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:25.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:25.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:25.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:25.397 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:25.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:25.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:25.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:25.540 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:25.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:25.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:25.681 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:25.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:25.782 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:25.821 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:25.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:25.923 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:25.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:25.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:26.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:26.084 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:26.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:26.186 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:26.213 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:26.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:26.315 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:26.335 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:26.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:26.437 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:26.466 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:26.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:26.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:26.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:26.608 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:26.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:26.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:26.753 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:26.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:26 smithi171 ceph-mon[90031]: pgmap v1840: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 3232/126018 objects misplaced (2.565%); 5.2 MiB/s, 82 objects/s recovering 2024-08-29T15:48:26.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:26 smithi027 ceph-mon[93921]: pgmap v1840: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 3232/126018 objects misplaced (2.565%); 5.2 MiB/s, 82 objects/s recovering 2024-08-29T15:48:26.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:26 smithi027 ceph-mon[98970]: pgmap v1840: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 890 op/s; 3232/126018 objects misplaced (2.565%); 5.2 MiB/s, 82 objects/s recovering 2024-08-29T15:48:26.855 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:26.887 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:26.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:26.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.020 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:27.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:27.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:27.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.157 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:27.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:27.258 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:27.288 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:27.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:27.426 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:27.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:27.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.573 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:27.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:27.674 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.709 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:27.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:27.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:27.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:27.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:27.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:27.988 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:27.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:27.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:28.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:28.112 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:28.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:28.214 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:28.252 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:28.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:28.354 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:28.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:28.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:28.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:28.535 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:28.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:28.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:28.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:28.691 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:28.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:28.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:28 smithi027 ceph-mon[93921]: pgmap v1841: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 3232/118905 objects misplaced (2.718%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:48:28.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:28 smithi027 ceph-mon[98970]: pgmap v1841: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 3232/118905 objects misplaced (2.718%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:48:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:28 smithi171 ceph-mon[90031]: pgmap v1841: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 3232/118905 objects misplaced (2.718%); 4.4 MiB/s, 69 objects/s recovering 2024-08-29T15:48:28.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:28.838 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:28.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:28.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:28.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:28.971 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:28.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:29.073 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:29.101 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:29.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:29.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:29.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:29.230 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:29.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:48:29] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.51.0" 2024-08-29T15:48:29.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:29.337 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.5 to 1 (10000) 2024-08-29T15:48:29.390 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:29.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:29.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:29.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:29.521 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:29.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:29.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:29.647 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:29.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:29.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:29.739 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2281664721' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:48:29.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:29.740 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:29.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:29.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2281664721' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:48:29.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:29.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:29.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2281664721' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:48:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:29.773 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:29.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:29.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:29.875 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:29.898 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:29.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:30.000 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:30.024 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:30.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:30.150 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:30.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:30.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:30.277 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:30.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:30.342 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:48:30.379 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:30.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:30.507 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:30.532 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:30.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:30.658 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:30.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:30 smithi171 ceph-mon[90031]: pgmap v1842: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 2535/113670 objects misplaced (2.230%); 6.0 MiB/s, 95 objects/s recovering 2024-08-29T15:48:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2281664721' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:48:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:30 smithi171 ceph-mon[90031]: osdmap e520: 8 total, 8 up, 8 in 2024-08-29T15:48:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1245026094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:48:30.784 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:30.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[93921]: pgmap v1842: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 2535/113670 objects misplaced (2.230%); 6.0 MiB/s, 95 objects/s recovering 2024-08-29T15:48:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2281664721' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:48:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[93921]: osdmap e520: 8 total, 8 up, 8 in 2024-08-29T15:48:30.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1245026094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:48:30.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[98970]: pgmap v1842: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 2535/113670 objects misplaced (2.230%); 6.0 MiB/s, 95 objects/s recovering 2024-08-29T15:48:30.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2281664721' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:48:30.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[98970]: osdmap e520: 8 total, 8 up, 8 in 2024-08-29T15:48:30.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1245026094' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:48:30.887 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:30.954 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:30.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:31.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:31.078 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:31.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:31.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:31.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 6 1 2024-08-29T15:48:31.181 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:31.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:31.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:31.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:31.309 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:31.334 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:31.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:31.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:31.341 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:48:31.341 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1245026094' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:48:31.341 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[93921]: osdmap e521: 8 total, 8 up, 8 in 2024-08-29T15:48:31.341 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:48:31.342 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:48:31.342 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:31.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:31.464 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:31.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:31.566 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:31.599 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:31.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:31.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:31.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:31.748 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:31.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:31 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:48:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:31 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1245026094' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:48:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:31 smithi171 ceph-mon[90031]: osdmap e521: 8 total, 8 up, 8 in 2024-08-29T15:48:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:48:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:48:31.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:31.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:48:31.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1245026094' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:48:31.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[98970]: osdmap e521: 8 total, 8 up, 8 in 2024-08-29T15:48:31.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:48:31.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:48:31.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:48:31.850 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:31.881 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:31.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:31.983 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:32.016 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:32.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:32.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:32.157 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:32.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:32.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:32.259 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:32.291 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:32.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:32.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:32.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:32.439 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:32.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:32.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:32.580 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:32.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:32.682 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:32.723 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:32.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:32.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:32 smithi171 ceph-mon[90031]: pgmap v1845: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 117 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2338/109833 objects misplaced (2.129%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:48:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:32 smithi171 ceph-mon[90031]: osdmap e522: 8 total, 8 up, 8 in 2024-08-29T15:48:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:32 smithi171 ceph-mon[90031]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 519'13487 2024-08-29T15:48:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:32 smithi171 ceph-mon[90031]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 519'13547 2024-08-29T15:48:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:32 smithi171 ceph-mon[90031]: osdmap e523: 8 total, 8 up, 8 in 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[93921]: pgmap v1845: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 117 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2338/109833 objects misplaced (2.129%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[93921]: osdmap e522: 8 total, 8 up, 8 in 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[93921]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 519'13487 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[93921]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 519'13547 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[93921]: osdmap e523: 8 total, 8 up, 8 in 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[98970]: pgmap v1845: 119 pgs: 1 peering, 1 active+remapped+backfill_wait, 117 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 2338/109833 objects misplaced (2.129%); 6.7 MiB/s, 107 objects/s recovering 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[98970]: osdmap e522: 8 total, 8 up, 8 in 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[98970]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 519'13487 2024-08-29T15:48:32.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[98970]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 519'13547 2024-08-29T15:48:32.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:32 smithi027 ceph-mon[98970]: osdmap e523: 8 total, 8 up, 8 in 2024-08-29T15:48:32.817 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:32.825 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:32.854 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:32.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:32.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:33.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:33.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:33.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:33.151 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:33.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:33.252 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:33.279 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:33.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:33.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:33.411 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:33.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:33.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:33.554 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:33.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:33.555 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:33.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:33.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:33.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:33.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:33.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:33.876 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:33.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:33.978 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:34.008 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:34.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:34.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:34.110 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:34.139 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:34.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:34.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:34.241 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:34.270 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:34.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:34.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:34.372 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:34.397 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:34.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:34.499 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:34.535 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:34.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:34 smithi027 ceph-mon[98970]: pgmap v1848: 119 pgs: 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 114 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 645 op/s; 5508/106002 objects misplaced (5.196%); 19 MiB/s, 306 objects/s recovering 2024-08-29T15:48:34.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:34 smithi027 ceph-mon[98970]: osdmap e524: 8 total, 8 up, 8 in 2024-08-29T15:48:34.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:34 smithi027 ceph-mon[93921]: pgmap v1848: 119 pgs: 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 114 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 645 op/s; 5508/106002 objects misplaced (5.196%); 19 MiB/s, 306 objects/s recovering 2024-08-29T15:48:34.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:34 smithi027 ceph-mon[93921]: osdmap e524: 8 total, 8 up, 8 in 2024-08-29T15:48:34.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:34.678 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:34.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:34.749 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:48:34.749 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:48:34.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:48:34.749 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:48:34.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.749 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145346 9.4 9:2da78b2a:::benchmark_data_smithi027_341149_object42916:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:48:34.749 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.463379+0000", 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 343.90986374599998, 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.221362903, 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145346 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463379+0000", 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463379+0000", 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463380+0000", 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1000000000000001e-06 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463385+0000", 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.374e-06 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463401+0000", 2024-08-29T15:48:34.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6308e-05 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.463408+0000", 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2810000000000003e-06 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.465915+0000", 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025071009999999999 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.466023+0000", 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107811 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.466133+0000", 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010950000000000001 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470663+0000", 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045307819999999997 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.474864+0000", 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042004750000000004 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.684648+0000", 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.209783885 2024-08-29T15:48:34.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.684700+0000", 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2476999999999999e-05 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.684742+0000", 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1808999999999999e-05 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.756 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145347 9.4 9:2e71130c:::benchmark_data_smithi027_341149_object42917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 343.90895347999998, 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.227039728, 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:48:34.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145347 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464290+0000", 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0399999999999995e-07 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464294+0000", 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8010000000000001e-06 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464308+0000", 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3567000000000001e-05 2024-08-29T15:48:34.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.464314+0000", 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9040000000000003e-06 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470690+0000", 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063753079999999997 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470816+0000", 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126632 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.470946+0000", 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012938000000000001 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.478372+0000", 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074261450000000003 2024-08-29T15:48:34.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.478408+0000", 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6189999999999997e-05 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691246+0000", 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21283770399999999 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691290+0000", 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3763000000000001e-05 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691329+0000", 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9629999999999998e-05 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145349 9.4 9:2a2db507:::benchmark_data_smithi027_341149_object42919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.471338+0000", 2024-08-29T15:48:34.763 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 343.90190469700002, 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.220100609, 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145349 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471338+0000", 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471338+0000", 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471340+0000", 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3039999999999999e-06 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471344+0000", 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1570000000000002e-06 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471355+0000", 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1581e-05 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.471367+0000", 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1378e-05 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.474937+0000", 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035707260000000002 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.475093+0000", 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000155501 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.475213+0000", 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120015 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.478582+0000", 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033687000000000001 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.479052+0000", 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00047079199999999999 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691366+0000", 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21231396999999999 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691406+0000", 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9988999999999999e-05 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691439+0000", 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2496000000000002e-05 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46778.0:145357 9.4 9:2f78fc4c:::benchmark_data_smithi027_341149_object42927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:42:50.483919+0000", 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 343.88932353899997, 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20763130699999999, 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.770 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46778", 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 145357 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483919+0000", 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483919+0000", 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483921+0000", 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4899999999999999e-06 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483926+0000", 2024-08-29T15:48:34.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2859999999999999e-06 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483942+0000", 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5673999999999999e-05 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.483949+0000", 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4139999999999997e-06 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.484005+0000", 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6084e-05 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.484127+0000", 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121888 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,4", 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.484246+0000", 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118497 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.487752+0000", 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035064229999999998 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.487856+0000", 2024-08-29T15:48:34.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103711 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691479+0000", 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.203623532 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691519+0000", 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9487999999999999e-05 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:42:50.691551+0000", 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1819999999999997e-05 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:94237 10.b 10:df119e6d:::benchmark_data_smithi027_374610_object94236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:16.434269+0000", 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 137.93897368899999, 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25353919200000002, 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.777 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 94237 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434269+0000", 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434269+0000", 2024-08-29T15:48:34.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434271+0000", 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.364e-06 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434294+0000", 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3828e-05 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434307+0000", 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2809000000000001e-05 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434374+0000", 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6229999999999994e-05 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434625+0000", 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025134400000000002 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434925+0000", 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00030008299999999998 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.435063+0000", 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013777700000000001 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.442684+0000", 2024-08-29T15:48:34.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076214990000000003 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.442713+0000", 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8827e-05 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.687692+0000", 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24497892199999999 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.687757+0000", 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5121000000000002e-05 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.687808+0000", 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1388000000000003e-05 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:94238 10.b 10:dcbcc0cc:::benchmark_data_smithi027_374610_object94237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:16.434542+0000", 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 137.93870097600001, 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26106675400000001, 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 94238 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.784 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434542+0000", 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434542+0000", 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434543+0000", 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1399999999999996e-07 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434563+0000", 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0766000000000001e-05 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434574+0000", 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0485e-05 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.434581+0000", 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6490000000000002e-06 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.439490+0000", 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049091990000000004 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.439703+0000", 2024-08-29T15:48:34.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021335899999999999 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.439804+0000", 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101233 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.442719+0000", 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002915033 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.442817+0000", 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7269999999999995e-05 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.695447+0000", 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25262989200000002 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.695547+0000", 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100305 2024-08-29T15:48:34.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:16.695609+0000", 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1848999999999997e-05 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135104 10.4 10:2f4bdcdb:::benchmark_data_smithi027_374610_object8974:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.361849+0000", 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 96.011394355999997, 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.147816495, 2024-08-29T15:48:34.790 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135104 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361849+0000", 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361849+0000", 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361850+0000", 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6920000000000001e-06 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361857+0000", 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4590000000000003e-06 2024-08-29T15:48:34.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361877+0000", 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0409999999999999e-05 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361889+0000", 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1837e-05 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361944+0000", 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5327000000000001e-05 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361954+0000", 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5119999999999997e-06 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.486342+0000", 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.124388143 2024-08-29T15:48:34.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.489339+0000", 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029968920000000001 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3,4", 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.489496+0000", 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015671399999999999 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.493498+0000", 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040020740000000004 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.493563+0000", 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5371999999999997e-05 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.497523+0000", 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039603930000000004 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509513+0000", 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011989154 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509566+0000", 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3514999999999998e-05 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509665+0000", 2024-08-29T15:48:34.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9000999999999996e-05 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135126 10.4 10:266106a9:::benchmark_data_smithi027_374610_object8996:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.361910+0000", 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 96.011332566999997, 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.148140757, 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.798 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135126 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361910+0000", 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361910+0000", 2024-08-29T15:48:34.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361911+0000", 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9100000000000004e-07 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361914+0000", 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.18e-06 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361923+0000", 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2599999999999994e-06 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361930+0000", 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5350000000000001e-06 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361982+0000", 2024-08-29T15:48:34.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.236e-05 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361988+0000", 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9229999999999999e-06 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.493595+0000", 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.13160718 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.493763+0000", 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016714 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3,4", 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.493918+0000", 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015512600000000001 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.497587+0000", 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036688810000000001 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.497600+0000", 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3811999999999999e-05 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509456+0000", 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011855464 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509881+0000", 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042534099999999998 2024-08-29T15:48:34.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509939+0000", 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7985999999999997e-05 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.510051+0000", 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111978 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135131 10.4 10:285646e9:::benchmark_data_smithi027_374610_object9001:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.361940+0000", 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 96.011303139999995, 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1503333440000001, 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.806 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135131 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361940+0000", 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361940+0000", 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361941+0000", 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6300000000000005e-07 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361943+0000", 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6199999999999999e-06 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361950+0000", 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2080000000000003e-06 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.361957+0000", 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4230000000000002e-06 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362036+0000", 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8864999999999998e-05 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362046+0000", 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0291e-05 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.497624+0000", 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1355776609999999 2024-08-29T15:48:34.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.497789+0000", 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016517100000000001 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3,4", 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.497926+0000", 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013692199999999999 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509367+0000", 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011441674000000001 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509387+0000", 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9338999999999999e-05 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509801+0000", 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041412900000000001 2024-08-29T15:48:34.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.512090+0000", 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022892799999999999 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.512148+0000", 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8193999999999999e-05 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.512273+0000", 2024-08-29T15:48:34.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012490400000000001 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135142 10.4 10:2fd264db:::benchmark_data_smithi027_374610_object9012:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.362007+0000", 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 96.011236030000006, 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1650876669999999, 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.814 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135142 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362007+0000", 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362007+0000", 2024-08-29T15:48:34.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362008+0000", 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5300000000000003e-07 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362012+0000", 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9099999999999998e-06 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362020+0000", 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3000000000000002e-06 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362026+0000", 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3779999999999998e-06 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362080+0000", 2024-08-29T15:48:34.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4101000000000003e-05 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362088+0000", 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7989999999999994e-06 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.500991+0000", 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.138903099 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.501165+0000", 2024-08-29T15:48:34.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017332 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3,4", 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.501305+0000", 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014088100000000001 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509708+0000", 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084027219999999996 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.516084+0000", 2024-08-29T15:48:34.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063760609999999997 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.516170+0000", 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5342999999999999e-05 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.526925+0000", 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010755684 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.526983+0000", 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7318000000000002e-05 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.527095+0000", 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011199799999999999 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135146 10.4 10:25ef9a04:::benchmark_data_smithi027_374610_object9016:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:48:34.821 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.362039+0000", 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 96.011204097999993, 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.167538951, 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135146 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362039+0000", 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362039+0000", 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362039+0000", 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7000000000000005e-07 2024-08-29T15:48:34.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362042+0000", 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7760000000000002e-06 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362049+0000", 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5400000000000001e-06 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362054+0000", 2024-08-29T15:48:34.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1340000000000003e-06 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362139+0000", 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4702000000000003e-05 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.362145+0000", 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7109999999999998e-06 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.504889+0000", 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1427432 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.505043+0000", 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000154308 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3,4", 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.505169+0000", 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012573900000000001 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509730+0000", 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004561926 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.516257+0000", 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006526947 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.516328+0000", 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0318e-05 2024-08-29T15:48:34.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.529412+0000", 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013084383999999999 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.529466+0000", 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4129000000000003e-05 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.529578+0000", 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111567 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:187898 10.2 10:491c9432:::benchmark_data_smithi027_374610_object61768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:56.405668+0000", 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 37.967574683000002, 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18350196399999999, 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.829 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187898 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.405668+0000", 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.405668+0000", 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.405669+0000", 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3000000000000001e-07 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.405671+0000", 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5380000000000001e-06 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.405679+0000", 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2679999999999997e-06 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.405684+0000", 2024-08-29T15:48:34.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7490000000000004e-06 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.408328+0000", 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026440510000000001 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.408642+0000", 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00031334100000000002 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.409211+0000", 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00056951599999999995 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.414996+0000", 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057845420000000002 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.581650+0000", 2024-08-29T15:48:34.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.166653881 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.589085+0000", 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074351419999999996 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.589129+0000", 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3869999999999998e-05 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.589170+0000", 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1535999999999997e-05 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:187901 10.2 10:475076b9:::benchmark_data_smithi027_374610_object61771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:47:56.409808+0000", 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 37.963435388999997, 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17948581599999999, 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.836 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187901 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.409808+0000", 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.409808+0000", 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.409809+0000", 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.327e-06 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.409814+0000", 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7129999999999996e-06 2024-08-29T15:48:34.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.409832+0000", 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7943000000000001e-05 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.409839+0000", 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6510000000000001e-06 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.415026+0000", 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051868620000000004 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.415158+0000", 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013193100000000001 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.416105+0000", 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00094702299999999996 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.424732+0000", 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0086269759999999997 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.581686+0000", 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.156953659 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.589220+0000", 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075340570000000003 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.589259+0000", 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9254000000000001e-05 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:47:56.589293+0000", 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4419999999999999e-05 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.842 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:212980 10.b 10:d048279f:::benchmark_data_smithi027_374610_object86850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:25.241303+0000", 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 9.1319396659999992, 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.069925898, 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 212980 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241303+0000", 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241303+0000", 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241304+0000", 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.009e-06 2024-08-29T15:48:34.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241308+0000", 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5209999999999998e-06 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241316+0000", 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3410000000000003e-06 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241321+0000", 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6929999999999998e-06 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241361+0000", 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0595e-05 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241453+0000", 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1813999999999999e-05 2024-08-29T15:48:34.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.241513+0000", 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0090000000000002e-05 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.244133+0000", 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026193259999999999 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.245048+0000", 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00091486099999999999 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.311131+0000", 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.06608348 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.311184+0000", 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2834999999999998e-05 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:25.311229+0000", 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5333000000000001e-05 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:214233 10.b 10:d5b42c85:::benchmark_data_smithi027_374610_object88103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:26.575845+0000", 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 7.7973980190000001, 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.073124736999999995, 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.849 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 214233 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575845+0000", 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575845+0000", 2024-08-29T15:48:34.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575847+0000", 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.052e-06 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575852+0000", 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4779999999999999e-06 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575866+0000", 2024-08-29T15:48:34.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3321e-05 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575873+0000", 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2280000000000001e-06 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.577533+0000", 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0016598540000000001 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.577650+0000", 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000116618 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.577747+0000", 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7233999999999998e-05 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.587013+0000", 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092666929999999995 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.591023+0000", 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040094730000000004 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.648902+0000", 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.057878588000000002 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.648940+0000", 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8247999999999997e-05 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.648970+0000", 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.995e-05 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.855 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:214236 10.b 10:d145b420:::benchmark_data_smithi027_374610_object88106:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:26.575918+0000", 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 7.7973247429999999, 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.073140207999999998, 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 214236 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575918+0000", 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575918+0000", 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575919+0000", 2024-08-29T15:48:34.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4400000000000001e-07 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575920+0000", 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9e-06 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575926+0000", 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5060000000000003e-06 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.575931+0000", 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3489999999999999e-06 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.581951+0000", 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060191619999999998 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.582082+0000", 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013178199999999999 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.582189+0000", 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106233 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.587049+0000", 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004860456 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.591030+0000", 2024-08-29T15:48:34.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039811300000000003 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.649008+0000", 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.057977898 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.649037+0000", 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8934000000000001e-05 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.649058+0000", 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1514000000000001e-05 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:214238 10.b 10:dd6043f0:::benchmark_data_smithi027_374610_object88108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:26.576926+0000", 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 7.7963172370000002, 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.072207334999999997, 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.862 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 214238 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.576926+0000", 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.576926+0000", 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.576926+0000", 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0399999999999995e-07 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.576929+0000", 2024-08-29T15:48:34.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6800000000000002e-06 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.576937+0000", 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8960000000000003e-06 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.576942+0000", 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.322e-06 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.587076+0000", 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010133689 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.587215+0000", 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138517 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.587323+0000", 2024-08-29T15:48:34.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108366 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.591035+0000", 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003712473 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.597193+0000", 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061578759999999996 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.649086+0000", 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.051892344 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.649113+0000", 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7359e-05 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:26.649133+0000", 2024-08-29T15:48:34.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0109e-05 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:214893 10.b 10:d65cc563:::benchmark_data_smithi027_374610_object88763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:27.300253+0000", 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 7.0729901259999997, 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10799712, 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.869 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 214893 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300253+0000", 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300253+0000", 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300254+0000", 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.187e-06 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300258+0000", 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4379999999999997e-06 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300270+0000", 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1875999999999999e-05 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300277+0000", 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9929999999999998e-06 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300333+0000", 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5436000000000003e-05 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300473+0000", 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014045099999999999 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.300581+0000", 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108126 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.396393+0000", 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.095811360999999998 2024-08-29T15:48:34.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.396776+0000", 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038294199999999999 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.407806+0000", 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011030546 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.408202+0000", 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039575500000000002 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.408250+0000", 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8009000000000001e-05 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:214895 10.b 10:d907723b:::benchmark_data_smithi027_374610_object88765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:27.302672+0000", 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 7.0705711610000002, 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.105864135, 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 214895 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.302672+0000", 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.302672+0000", 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.302673+0000", 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9100000000000003e-07 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.302676+0000", 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4869999999999998e-06 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.302685+0000", 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4260000000000003e-06 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.302690+0000", 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.198e-06 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.303601+0000", 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000910826 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.303688+0000", 2024-08-29T15:48:34.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7717000000000001e-05 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,6", 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.303753+0000", 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4950000000000007e-05 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.396665+0000", 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.092911471999999995 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.396804+0000", 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013930299999999999 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.408444+0000", 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011639854 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.408494+0000", 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9761e-05 2024-08-29T15:48:34.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.408536+0000", 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2349999999999999e-05 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.47036.0:214893 10.b e519/480 ondisk, result = 0)", 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:27.306457+0000", 2024-08-29T15:48:34.882 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 7.0667856860000002, 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.089981438999999996, 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.306457+0000", 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.306457+0000", 2024-08-29T15:48:34.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.306459+0000", 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1999999999999999e-06 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.306466+0000", 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2169999999999997e-06 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.392365+0000", 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.085899182000000004 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.392378+0000", 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3443e-05 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.396338+0000", 2024-08-29T15:48:34.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039596739999999998 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.396366+0000", 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7610000000000002e-05 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:27.396439+0000", 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3113000000000003e-05 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.886 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.887 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.887 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:48:34.887 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:48:34.887 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:48:34.887 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:34.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:48:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:34 smithi171 ceph-mon[90031]: pgmap v1848: 119 pgs: 2 active+remapped+backfilling, 1 active+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 114 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 645 op/s; 5508/106002 objects misplaced (5.196%); 19 MiB/s, 306 objects/s recovering 2024-08-29T15:48:34.889 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:34 smithi171 ceph-mon[90031]: osdmap e524: 8 total, 8 up, 8 in 2024-08-29T15:48:34.936 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:34.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:35.038 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:35.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:35.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:35.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:35.220 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:35.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:35.321 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:35.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:35.364 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:35.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:35.465 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:35.505 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:35.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:35.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:35.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:35.644 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:35.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:35.746 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:35.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:35.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:35.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:35.879 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:35.905 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:35.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:35.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:36.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:36.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.049 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:36.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:36.149 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:36.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.176 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:36.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:36.277 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:36.303 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:36.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:36.405 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:36.433 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:36.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:36.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:36 smithi027 ceph-mon[93921]: pgmap v1850: 119 pgs: 2 active+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 115 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 7.7 KiB/s rd, 23 KiB/s wr, 592 op/s; 5508/104463 objects misplaced (5.273%); 16 MiB/s, 254 objects/s recovering 2024-08-29T15:48:36.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:36 smithi027 ceph-mon[98970]: pgmap v1850: 119 pgs: 2 active+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 115 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 7.7 KiB/s rd, 23 KiB/s wr, 592 op/s; 5508/104463 objects misplaced (5.273%); 16 MiB/s, 254 objects/s recovering 2024-08-29T15:48:36.578 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:36.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:36.680 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:36.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:36.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:36 smithi171 ceph-mon[90031]: pgmap v1850: 119 pgs: 2 active+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 115 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 7.7 KiB/s rd, 23 KiB/s wr, 592 op/s; 5508/104463 objects misplaced (5.273%); 16 MiB/s, 254 objects/s recovering 2024-08-29T15:48:36.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:36.850 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:36.952 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:36.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:36.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:37.081 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:37.106 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:37.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:37.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:37.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:37.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:37.248 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:37.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:37.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:37.378 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:37.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:37.479 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:37.509 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:37.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:37.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:37.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:37.648 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:37.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:37.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:37.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:37.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:37.785 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:37.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:37.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-29T15:48:37.886 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:37.913 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:37.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:38.015 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:38.042 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:38.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:38.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:38.144 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:38.172 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:38.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:38.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:38.274 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:38.318 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:38.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:38.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:38.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:38.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:38.496 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:38 smithi027 ceph-mon[93921]: pgmap v1851: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 6.2 KiB/s rd, 18 KiB/s wr, 752 op/s; 4584/96357 objects misplaced (4.757%); 18 MiB/s, 289 objects/s recovering 2024-08-29T15:48:38.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:38 smithi027 ceph-mon[98970]: pgmap v1851: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 6.2 KiB/s rd, 18 KiB/s wr, 752 op/s; 4584/96357 objects misplaced (4.757%); 18 MiB/s, 289 objects/s recovering 2024-08-29T15:48:38.569 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:38 smithi171 ceph-mon[90031]: pgmap v1851: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 2.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 6.2 KiB/s rd, 18 KiB/s wr, 752 op/s; 4584/96357 objects misplaced (4.757%); 18 MiB/s, 289 objects/s recovering 2024-08-29T15:48:38.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:38.605 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:38.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:38.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:38.706 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:38.745 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:38.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:38.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:38.880 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:38.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:38.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:38.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:39.021 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:39.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:39.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:39.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:39.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:39.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:39.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:39.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:39.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:39.319 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:48:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:48:39] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.51.0" 2024-08-29T15:48:39.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:39.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:39.468 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:39.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:39.569 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:39.595 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:39.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:39.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:39.697 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:39.727 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:39.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:39.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:39.829 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:39.867 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:39.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:39.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:39.969 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:40.002 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:40.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:40.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:40.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:40.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:40.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.318 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:40.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:40.419 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:40.455 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:40.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:40.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:40 smithi027 ceph-mon[93921]: pgmap v1852: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 755 op/s; 3818/92202 objects misplaced (4.141%); 16 MiB/s, 263 objects/s recovering 2024-08-29T15:48:40.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:40 smithi027 ceph-mon[98970]: pgmap v1852: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 755 op/s; 3818/92202 objects misplaced (4.141%); 16 MiB/s, 263 objects/s recovering 2024-08-29T15:48:40.586 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:40.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:40.687 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:40.726 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:40 smithi171 ceph-mon[90031]: pgmap v1852: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 755 op/s; 3818/92202 objects misplaced (4.141%); 16 MiB/s, 263 objects/s recovering 2024-08-29T15:48:40.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:40.853 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:40.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:40.955 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:40.979 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:40.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:41.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:41.142 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:41.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:41.161 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:41.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:41.285 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:41.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:41.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:41.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:41.441 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:41.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:41.542 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:41.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:41.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:41.611 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:48:41.611 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:48:41.621 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:41.632 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:48:41.632 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:48:41.675 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:48:41.676 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:48:41.678 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:41.704 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:41.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:41.713 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:48:41.713 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:48:41.753 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:48:41.753 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:48:41.790 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:48:41.790 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:48:41.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:41.837 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:48:41.838 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:48:41.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:41.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:41.879 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:48:41.879 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:48:41.962 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:41.991 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:41.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:42.093 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:42.120 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:42.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:42.221 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:42.243 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:42.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:42.319 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.6 to 1 (10000) 2024-08-29T15:48:42.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:42.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:42.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:42.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:42.538 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:42.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[93921]: pgmap v1853: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 3818/86370 objects misplaced (4.421%); 14 MiB/s, 225 objects/s recovering 2024-08-29T15:48:42.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3906388574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:42.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1373219561' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:48:42.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:48:42.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[98970]: pgmap v1853: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 3818/86370 objects misplaced (4.421%); 14 MiB/s, 225 objects/s recovering 2024-08-29T15:48:42.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3906388574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:42.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1373219561' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:48:42.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:42 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:48:42.639 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:42.676 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:42.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:42 smithi171 ceph-mon[90031]: pgmap v1853: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 3818/86370 objects misplaced (4.421%); 14 MiB/s, 225 objects/s recovering 2024-08-29T15:48:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3906388574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:42 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1373219561' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:48:42.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:42 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:48:42.778 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:42.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:42.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:42.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:42.954 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:42.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:43.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:43.096 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:43.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:43.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:43.198 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:43.222 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:43.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:43.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:43.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:43.367 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:43.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:43.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:43.469 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:43.505 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:43.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:43.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:43 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:48:43.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:43 smithi027 ceph-mon[98970]: osdmap e525: 8 total, 8 up, 8 in 2024-08-29T15:48:43.513 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:43 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:48:43.513 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:43 smithi027 ceph-mon[93921]: osdmap e525: 8 total, 8 up, 8 in 2024-08-29T15:48:43.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:43.650 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:43.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:43.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:43.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:43 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:48:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:43 smithi171 ceph-mon[90031]: osdmap e525: 8 total, 8 up, 8 in 2024-08-29T15:48:43.800 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:43.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:43.902 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:43.933 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:43.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:43.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:44.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:44.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:44.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:44.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:44.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:44.212 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:44.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:44.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:44.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:44.357 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:44.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:44.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:44.459 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:44.485 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:44.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:44.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:44 smithi027 ceph-mon[98970]: pgmap v1855: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 2286/80439 objects misplaced (2.842%); 13 MiB/s, 210 objects/s recovering 2024-08-29T15:48:44.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:44 smithi027 ceph-mon[93921]: pgmap v1855: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 2286/80439 objects misplaced (2.842%); 13 MiB/s, 210 objects/s recovering 2024-08-29T15:48:44.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:44.587 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:44.622 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:44.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:44.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:44.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:44.766 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:44 smithi171 ceph-mon[90031]: pgmap v1855: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 2286/80439 objects misplaced (2.842%); 13 MiB/s, 210 objects/s recovering 2024-08-29T15:48:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:48:44.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:44.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:44.869 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:44.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:44.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:44.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:45.002 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:45.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:45.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:45.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.174 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:45.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:45.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:48:45.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:45.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:45.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:45.421 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:45.448 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:45.550 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:45.573 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:45.675 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:45.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:45.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:45.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:45.926 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:45.947 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:45.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:46.049 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:46.072 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:46.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:46.197 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:46.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:46.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:46.321 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:46.423 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:46.454 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:46.475 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:46 smithi027 ceph-mon[98970]: pgmap v1856: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 2286/77898 objects misplaced (2.935%); 13 MiB/s, 207 objects/s recovering 2024-08-29T15:48:46.475 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:46 smithi027 ceph-mon[98970]: osdmap e526: 8 total, 8 up, 8 in 2024-08-29T15:48:46.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:46 smithi027 ceph-mon[93921]: pgmap v1856: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 2286/77898 objects misplaced (2.935%); 13 MiB/s, 207 objects/s recovering 2024-08-29T15:48:46.476 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:46 smithi027 ceph-mon[93921]: osdmap e526: 8 total, 8 up, 8 in 2024-08-29T15:48:46.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:46.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.593 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:46.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:46.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:46.718 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:46 smithi171 ceph-mon[90031]: pgmap v1856: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 2286/77898 objects misplaced (2.935%); 13 MiB/s, 207 objects/s recovering 2024-08-29T15:48:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:46 smithi171 ceph-mon[90031]: osdmap e526: 8 total, 8 up, 8 in 2024-08-29T15:48:46.820 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:46.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:46.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:46.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:46.969 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:46.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:47.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:47.075 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-29T15:48:47.076 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:48:47.092 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:47.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:47.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:47.218 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:47.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:47.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:47.320 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:47.341 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:47.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:47.443 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:47.469 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:47.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:47.489 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:47 smithi027 ceph-mon[98970]: osdmap e527: 8 total, 8 up, 8 in 2024-08-29T15:48:47.490 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:47 smithi027 ceph-mon[93921]: osdmap e527: 8 total, 8 up, 8 in 2024-08-29T15:48:47.571 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:47.603 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:47.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:47.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:47.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:47.750 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:47.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:47.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:47 smithi171 ceph-mon[90031]: osdmap e527: 8 total, 8 up, 8 in 2024-08-29T15:48:47.844 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:48:47.844 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:48:47.852 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:47.889 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:47.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:47.991 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:48.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.024 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:48.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:48.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:48.163 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:48.265 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:48.292 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:48.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:48.394 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:48.426 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:48.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.562 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:48.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:48.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:48 smithi027 ceph-mon[93921]: pgmap v1859: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 2286/70902 objects misplaced (3.224%); 11 MiB/s, 174 objects/s recovering 2024-08-29T15:48:48.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:48 smithi027 ceph-mon[93921]: osdmap e528: 8 total, 8 up, 8 in 2024-08-29T15:48:48.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:48 smithi027 ceph-mon[98970]: pgmap v1859: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 2286/70902 objects misplaced (3.224%); 11 MiB/s, 174 objects/s recovering 2024-08-29T15:48:48.653 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:48 smithi027 ceph-mon[98970]: osdmap e528: 8 total, 8 up, 8 in 2024-08-29T15:48:48.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-29T15:48:48.662 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:48.694 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:48.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:48 smithi171 ceph-mon[90031]: pgmap v1859: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 859 op/s; 2286/70902 objects misplaced (3.224%); 11 MiB/s, 174 objects/s recovering 2024-08-29T15:48:48.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:48 smithi171 ceph-mon[90031]: osdmap e528: 8 total, 8 up, 8 in 2024-08-29T15:48:48.796 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:48.825 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:48.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:48.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:48.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:48.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:49.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:49.149 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:49.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:49.163 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:49.164 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:49.193 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:48:49.252 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:49.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:49.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:49.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:48:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:48:49] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-08-29T15:48:49.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:49.450 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:49.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:49.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:49.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:49.588 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:49.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:49.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:49.690 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:49.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:49.713 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:49.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:49.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:49.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:49.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:49.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:49.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:49.961 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:49.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:50.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:50.085 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:50.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:50.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:50.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:50.209 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:50.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:50.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:50.332 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:50.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:50.388 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:48:50.434 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:50.464 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:50.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:50.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[93921]: pgmap v1861: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 507/67452 objects misplaced (0.752%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[93921]: osdmap e529: 8 total, 8 up, 8 in 2024-08-29T15:48:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3600975645' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:50.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:50.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[98970]: pgmap v1861: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 507/67452 objects misplaced (0.752%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:50.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[98970]: osdmap e529: 8 total, 8 up, 8 in 2024-08-29T15:48:50.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3600975645' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:50.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:50 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:50.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:50.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:50.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:50.742 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:50.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:50 smithi171 ceph-mon[90031]: pgmap v1861: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 507/67452 objects misplaced (0.752%); 16 MiB/s, 258 objects/s recovering 2024-08-29T15:48:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:50 smithi171 ceph-mon[90031]: osdmap e529: 8 total, 8 up, 8 in 2024-08-29T15:48:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3600975645' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:50 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:48:50.845 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:50.874 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:50.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:50.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:50.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:51.017 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:51.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:51.119 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:51.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:51.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:51.254 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:51.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:51.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:51.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:51.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.420 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:51.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:51.521 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:51.555 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:51.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:51.686 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:51.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:51 smithi171 ceph-mon[90031]: 1.8 scrub starts 2024-08-29T15:48:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:51 smithi171 ceph-mon[90031]: 1.8 scrub ok 2024-08-29T15:48:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:51 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:48:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:51 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:48:51.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:51 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:48:51.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:51 smithi171 ceph-mon[90031]: osdmap e530: 8 total, 8 up, 8 in 2024-08-29T15:48:51.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:51.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[98970]: 1.8 scrub starts 2024-08-29T15:48:51.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[98970]: 1.8 scrub ok 2024-08-29T15:48:51.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:48:51.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:48:51.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:48:51.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[98970]: osdmap e530: 8 total, 8 up, 8 in 2024-08-29T15:48:51.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[93921]: 1.8 scrub starts 2024-08-29T15:48:51.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[93921]: 1.8 scrub ok 2024-08-29T15:48:51.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:48:51.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:48:51.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:48:51.820 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:51 smithi027 ceph-mon[93921]: osdmap e530: 8 total, 8 up, 8 in 2024-08-29T15:48:51.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.829 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:51.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:51.930 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:51.970 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:51.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:51.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:52.072 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:52.114 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:52.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:52.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:52.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:52.254 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:52.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:52.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:52.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:52.408 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:52.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:52.509 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:52.537 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:52.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:52.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:52.674 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:52.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:52.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:52 smithi171 ceph-mon[90031]: pgmap v1864: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 211/62412 objects misplaced (0.338%); 24 MiB/s, 378 objects/s recovering 2024-08-29T15:48:52.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:52.804 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:52.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:52.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:52 smithi027 ceph-mon[98970]: pgmap v1864: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 211/62412 objects misplaced (0.338%); 24 MiB/s, 378 objects/s recovering 2024-08-29T15:48:52.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:52 smithi027 ceph-mon[93921]: pgmap v1864: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 211/62412 objects misplaced (0.338%); 24 MiB/s, 378 objects/s recovering 2024-08-29T15:48:52.907 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:52.931 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:52.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:52.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:53.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:53.076 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:53.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:53.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:53.215 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:53.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:53.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:53.317 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:53.353 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:53.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:53.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:53.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:53.486 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:53.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:53.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:53.615 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:53.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:53.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:53.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:53.754 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:53.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:53.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:53.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:53.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:53.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:53.992 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:54.026 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:54.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:54.128 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:54.165 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:54.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:54.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:54.304 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:54.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:54.406 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:54.449 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:54.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:54.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:54 smithi027 ceph-mon[93921]: pgmap v1865: 119 pgs: 119 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 21 MiB/s, 338 objects/s recovering 2024-08-29T15:48:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:54 smithi027 ceph-mon[98970]: pgmap v1865: 119 pgs: 119 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 21 MiB/s, 338 objects/s recovering 2024-08-29T15:48:54.584 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:54.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:54.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:54.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:54.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:54.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:54 smithi171 ceph-mon[90031]: pgmap v1865: 119 pgs: 119 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 21 MiB/s, 338 objects/s recovering 2024-08-29T15:48:54.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:54.869 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:54.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:54.971 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:55.001 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:55.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:55.145 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:55.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:55.247 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:55.274 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:55.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:55.412 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:55.514 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:55.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.552 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:55.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:55.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:55.707 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:55.809 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:55.842 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:55.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:55.980 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:55.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:55.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:56.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:56.116 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:56.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:56.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:56.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:56.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:56.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:56.392 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:56.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:56.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:56.532 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:56.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:56.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:56.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:56.669 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:56.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:56.671 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:56 smithi027 ceph-mon[98970]: pgmap v1866: 119 pgs: 119 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7.1 MiB/s, 113 objects/s recovering 2024-08-29T15:48:56.671 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:56 smithi027 ceph-mon[93921]: pgmap v1866: 119 pgs: 119 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7.1 MiB/s, 113 objects/s recovering 2024-08-29T15:48:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:56 smithi171 ceph-mon[90031]: pgmap v1866: 119 pgs: 119 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7.1 MiB/s, 113 objects/s recovering 2024-08-29T15:48:56.771 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:56.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:56.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:56.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:56.845 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:48:56.846 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:48:56.880 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:48:56.880 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:48:56.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:56.931 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:48:56.931 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:48:56.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:56.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:56.967 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:48:56.967 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:48:57.002 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:48:57.003 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:48:57.041 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:48:57.041 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:48:57.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:57.080 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:48:57.080 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:48:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:57.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:57.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:57.116 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:48:57.117 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:48:57.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:57.222 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:57.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:57.324 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:57.359 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:57.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:57.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:57.498 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:57.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:57.600 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:57.629 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:57.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:57.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:57.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set noscrub 2024-08-29T15:48:57.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3047439296' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:57.706 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3047439296' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:57.731 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:48:57.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:57.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:48:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3047439296' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:48:57.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:57.914 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:57.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:57.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:58.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:58.080 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:58.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:58.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:58.228 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:58.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:58.330 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:58.355 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:58.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:58.457 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:48:58.486 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:58.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:58.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:48:58.588 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:58.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:58.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:58.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:58.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:58 smithi027 ceph-mon[98970]: pgmap v1867: 119 pgs: 119 active+clean; 943 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 3.7 MiB/s, 60 objects/s recovering 2024-08-29T15:48:58.699 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:58 smithi027 ceph-mon[93921]: pgmap v1867: 119 pgs: 119 active+clean; 943 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 3.7 MiB/s, 60 objects/s recovering 2024-08-29T15:48:58.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:48:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:58 smithi171 ceph-mon[90031]: pgmap v1867: 119 pgs: 119 active+clean; 943 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 3.7 MiB/s, 60 objects/s recovering 2024-08-29T15:48:58.793 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:58.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:48:58.895 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:48:58.930 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:58.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:48:58.982 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:48:58.982 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:48:58.982 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:48:58.983 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:48:58.983 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:48:59.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:59.068 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:59.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:59.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:59.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:59.210 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:59.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:59.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:59.312 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:48:59.320 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:48:59] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-29T15:48:59.343 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:48:59.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:59.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:48:59.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:59.519 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:59.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:59.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:59.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:48:59.632 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:48:59.634 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:48:59.659 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:59.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:59.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:48:59.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:48:59.801 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:59.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:48:59.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:48:59.941 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:48:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:48:59.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:48:59.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-29T15:49:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:59 smithi171 ceph-mon[90031]: pgmap v1868: 119 pgs: 119 active+clean; 838 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 3.1 MiB/s, 49 objects/s recovering 2024-08-29T15:49:00.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:48:59 smithi171 ceph-mon[90031]: from='client.37332 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:49:00.028 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:00.029 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:00.029 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[93921]: pgmap v1868: 119 pgs: 119 active+clean; 838 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 3.1 MiB/s, 49 objects/s recovering 2024-08-29T15:49:00.029 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[93921]: from='client.37332 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:49:00.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:00.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:00.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[98970]: pgmap v1868: 119 pgs: 119 active+clean; 838 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 3.1 MiB/s, 49 objects/s recovering 2024-08-29T15:49:00.029 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:48:59 smithi027 ceph-mon[98970]: from='client.37332 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:49:00.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:00.105 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:00.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:00.202 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:00.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:00.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:00.279 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:00.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:00.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:00.417 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:00.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:00.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:00.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:49:00.518 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:00.549 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:00.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:00.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:00.675 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1868,"stamp":"2024-08-29T15:48:59.178902+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":878254077,"num_objects":13357,"num_object_clones":0,"num_object_copies":40071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13357,"num_whiteouts":0,"num_read":5988,"num_read_kb":13001,"num_write":239835,"num_write_kb":8084754,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17753,"num_bytes_recovered":1166417551,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111257,"ondisk_log_size":111257,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9840064,"kb_used_data":2996928,"kb_used_omap":43,"kb_used_meta":6842900,"kb_avail":740121152,"statfs":{"total":767960285184,"available":757884059648,"internally_reserved":0,"allocated":3068854272,"data_stored":3042155702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":7007130092},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-598173907,"num_objects":-9127,"num_object_clones":0,"num_object_copies":-27381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-507,"num_objects_unfound":0,"num_objects_dirty":-9127,"num_whiteouts":0,"num_read":-17,"num_read_kb":-37,"num_write":9118,"num_write_kb":-109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":482,"num_bytes_recovered":31376504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.797879"},"pg_stats":[{"pgid":"1.11","version":"90'149","reported_seq":949,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429559+0000","last_change":"2024-08-29T15:47:01.417257+0000","last_active":"2024-08-29T15:48:50.429559+0000","last_peered":"2024-08-29T15:48:50.429559+0000","last_clean":"2024-08-29T15:48:50.429559+0000","last_became_active":"2024-08-29T15:47:01.411204+0000","last_became_peered":"2024-08-29T15:47:01.411204+0000","last_unstale":"2024-08-29T15:48:50.429559+0000","last_undegraded":"2024-08-29T15:48:50.429559+0000","last_fullsized":"2024-08-29T15:48:50.429559+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:36.699540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:36.699540+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:08.236678+0000","scrub_duration":0.0014941869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":983,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385817+0000","last_change":"2024-08-29T15:47:12.481794+0000","last_active":"2024-08-29T15:48:50.385817+0000","last_peered":"2024-08-29T15:48:50.385817+0000","last_clean":"2024-08-29T15:48:50.385817+0000","last_became_active":"2024-08-29T15:47:12.480977+0000","last_became_peered":"2024-08-29T15:47:12.480977+0000","last_unstale":"2024-08-29T15:48:50.385817+0000","last_undegraded":"2024-08-29T15:48:50.385817+0000","last_fullsized":"2024-08-29T15:48:50.385817+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:29.573851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":273,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.386010+0000","last_change":"2024-08-29T15:46:36.697686+0000","last_active":"2024-08-29T15:48:49.386010+0000","last_peered":"2024-08-29T15:48:49.386010+0000","last_clean":"2024-08-29T15:48:49.386010+0000","last_became_active":"2024-08-29T15:46:36.697367+0000","last_became_peered":"2024-08-29T15:46:36.697367+0000","last_unstale":"2024-08-29T15:48:49.386010+0000","last_undegraded":"2024-08-29T15:48:49.386010+0000","last_fullsized":"2024-08-29T15:48:49.386010+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:40.001922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":1024,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383103+0000","last_change":"2024-08-29T15:44:59.245746+0000","last_active":"2024-08-29T15:48:49.383103+0000","last_peered":"2024-08-29T15:48:49.383103+0000","last_clean":"2024-08-29T15:48:49.383103+0000","last_became_active":"2024-08-29T15:44:59.245400+0000","last_became_peered":"2024-08-29T15:44:59.245400+0000","last_unstale":"2024-08-29T15:48:49.383103+0000","last_undegraded":"2024-08-29T15:48:49.383103+0000","last_fullsized":"2024-08-29T15:48:49.383103+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.968809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.968809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:37.657803+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383451+0000","last_change":"2024-08-29T15:39:53.258205+0000","last_active":"2024-08-29T15:48:49.383451+0000","last_peered":"2024-08-29T15:48:49.383451+0000","last_clean":"2024-08-29T15:48:49.383451+0000","last_became_active":"2024-08-29T15:39:53.257784+0000","last_became_peered":"2024-08-29T15:39:53.257784+0000","last_unstale":"2024-08-29T15:48:49.383451+0000","last_undegraded":"2024-08-29T15:48:49.383451+0000","last_fullsized":"2024-08-29T15:48:49.383451+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:30:42.270982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430157+0000","last_change":"2024-08-29T15:46:58.374246+0000","last_active":"2024-08-29T15:48:50.430157+0000","last_peered":"2024-08-29T15:48:50.430157+0000","last_clean":"2024-08-29T15:48:50.430157+0000","last_became_active":"2024-08-29T15:46:58.373708+0000","last_became_peered":"2024-08-29T15:46:58.373708+0000","last_unstale":"2024-08-29T15:48:50.430157+0000","last_undegraded":"2024-08-29T15:48:50.430157+0000","last_fullsized":"2024-08-29T15:48:50.430157+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:27.833391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":1095,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385691+0000","last_change":"2024-08-29T15:47:46.309647+0000","last_active":"2024-08-29T15:48:50.385691+0000","last_peered":"2024-08-29T15:48:50.385691+0000","last_clean":"2024-08-29T15:48:50.385691+0000","last_became_active":"2024-08-29T15:47:46.309290+0000","last_became_peered":"2024-08-29T15:47:46.309290+0000","last_unstale":"2024-08-29T15:48:50.385691+0000","last_undegraded":"2024-08-29T15:48:50.385691+0000","last_fullsized":"2024-08-29T15:48:50.385691+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.615446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.615446+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:58.511333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":87,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429635+0000","last_change":"2024-08-29T15:48:12.267269+0000","last_active":"2024-08-29T15:48:50.429635+0000","last_peered":"2024-08-29T15:48:50.429635+0000","last_clean":"2024-08-29T15:48:50.429635+0000","last_became_active":"2024-08-29T15:48:12.266893+0000","last_became_peered":"2024-08-29T15:48:12.266893+0000","last_unstale":"2024-08-29T15:48:50.429635+0000","last_undegraded":"2024-08-29T15:48:50.429635+0000","last_fullsized":"2024-08-29T15:48:50.429635+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:07:31.630164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":318,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389357+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:48:49.389357+0000","last_peered":"2024-08-29T15:48:49.389357+0000","last_clean":"2024-08-29T15:48:49.389357+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:48:49.389357+0000","last_undegraded":"2024-08-29T15:48:49.389357+0000","last_fullsized":"2024-08-29T15:48:49.389357+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":1044,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383067+0000","last_change":"2024-08-29T15:44:59.241957+0000","last_active":"2024-08-29T15:48:49.383067+0000","last_peered":"2024-08-29T15:48:49.383067+0000","last_clean":"2024-08-29T15:48:49.383067+0000","last_became_active":"2024-08-29T15:44:59.241187+0000","last_became_peered":"2024-08-29T15:44:59.241187+0000","last_unstale":"2024-08-29T15:48:49.383067+0000","last_undegraded":"2024-08-29T15:48:49.383067+0000","last_fullsized":"2024-08-29T15:48:49.383067+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.831081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.831081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:04:04.531194+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387332+0000","last_change":"2024-08-29T15:48:12.268685+0000","last_active":"2024-08-29T15:48:50.387332+0000","last_peered":"2024-08-29T15:48:50.387332+0000","last_clean":"2024-08-29T15:48:50.387332+0000","last_became_active":"2024-08-29T15:48:12.268311+0000","last_became_peered":"2024-08-29T15:48:12.268311+0000","last_unstale":"2024-08-29T15:48:50.387332+0000","last_undegraded":"2024-08-29T15:48:50.387332+0000","last_fullsized":"2024-08-29T15:48:50.387332+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:30:18.347971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":107,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429993+0000","last_change":"2024-08-29T15:48:30.358799+0000","last_active":"2024-08-29T15:48:50.429993+0000","last_peered":"2024-08-29T15:48:50.429993+0000","last_clean":"2024-08-29T15:48:50.429993+0000","last_became_active":"2024-08-29T15:48:30.353158+0000","last_became_peered":"2024-08-29T15:48:30.353158+0000","last_unstale":"2024-08-29T15:48:50.429993+0000","last_undegraded":"2024-08-29T15:48:50.429993+0000","last_fullsized":"2024-08-29T15:48:50.429993+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:17:11.641951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.6","version":"530'15074","reported_seq":15134,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:58.690011+0000","last_change":"2024-08-29T15:45:06.877844+0000","last_active":"2024-08-29T15:48:58.690011+0000","last_peered":"2024-08-29T15:48:58.690011+0000","last_clean":"2024-08-29T15:48:58.690011+0000","last_became_active":"2024-08-29T15:45:06.877523+0000","last_became_peered":"2024-08-29T15:45:06.877523+0000","last_unstale":"2024-08-29T15:48:58.690011+0000","last_undegraded":"2024-08-29T15:48:58.690011+0000","last_fullsized":"2024-08-29T15:48:58.690011+0000","mapping_epoch":480,"log_start":"493'7527","ondisk_log_start":"493'7527","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:37:30.523325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44826624,"num_objects":684,"num_object_clones":0,"num_object_copies":2052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15074,"num_write_kb":504256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":980,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389827+0000","last_change":"2024-08-29T15:48:15.289183+0000","last_active":"2024-08-29T15:48:49.389827+0000","last_peered":"2024-08-29T15:48:49.389827+0000","last_clean":"2024-08-29T15:48:49.389827+0000","last_became_active":"2024-08-29T15:48:15.288693+0000","last_became_peered":"2024-08-29T15:48:15.288693+0000","last_unstale":"2024-08-29T15:48:49.389827+0000","last_undegraded":"2024-08-29T15:48:49.389827+0000","last_fullsized":"2024-08-29T15:48:49.389827+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.799642+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.799642+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:07.837651+0000","scrub_duration":0.0012023820000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":374,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389955+0000","last_change":"2024-08-29T15:46:58.374075+0000","last_active":"2024-08-29T15:48:49.389955+0000","last_peered":"2024-08-29T15:48:49.389955+0000","last_clean":"2024-08-29T15:48:49.389955+0000","last_became_active":"2024-08-29T15:46:58.369089+0000","last_became_peered":"2024-08-29T15:46:58.369089+0000","last_unstale":"2024-08-29T15:48:49.389955+0000","last_undegraded":"2024-08-29T15:48:49.389955+0000","last_fullsized":"2024-08-29T15:48:49.389955+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:40:45.123913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":120,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383284+0000","last_change":"2024-08-29T15:39:53.258473+0000","last_active":"2024-08-29T15:48:49.383284+0000","last_peered":"2024-08-29T15:48:49.383284+0000","last_clean":"2024-08-29T15:48:49.383284+0000","last_became_active":"2024-08-29T15:39:53.258055+0000","last_became_peered":"2024-08-29T15:39:53.258055+0000","last_unstale":"2024-08-29T15:48:49.383284+0000","last_undegraded":"2024-08-29T15:48:49.383284+0000","last_fullsized":"2024-08-29T15:48:49.383284+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:38:54.441868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"530'14994","reported_seq":17829,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:57.318189+0000","last_change":"2024-08-29T15:48:50.421937+0000","last_active":"2024-08-29T15:48:57.318189+0000","last_peered":"2024-08-29T15:48:57.318189+0000","last_clean":"2024-08-29T15:48:57.318189+0000","last_became_active":"2024-08-29T15:48:50.388985+0000","last_became_peered":"2024-08-29T15:48:50.388985+0000","last_unstale":"2024-08-29T15:48:57.318189+0000","last_undegraded":"2024-08-29T15:48:57.318189+0000","last_fullsized":"2024-08-29T15:48:57.318189+0000","mapping_epoch":529,"log_start":"499'8546","ondisk_log_start":"499'8546","created":480,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6448,"log_dups_size":0,"ondisk_log_size":6448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:55.673298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49807360,"num_objects":760,"num_object_clones":0,"num_object_copies":2280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14994,"num_write_kb":504128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1351,"num_bytes_recovered":88539136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'149","reported_seq":1139,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385666+0000","last_change":"2024-08-29T15:44:59.240657+0000","last_active":"2024-08-29T15:48:50.385666+0000","last_peered":"2024-08-29T15:48:50.385666+0000","last_clean":"2024-08-29T15:48:50.385666+0000","last_became_active":"2024-08-29T15:44:59.240261+0000","last_became_peered":"2024-08-29T15:44:59.240261+0000","last_unstale":"2024-08-29T15:48:50.385666+0000","last_undegraded":"2024-08-29T15:48:50.385666+0000","last_fullsized":"2024-08-29T15:48:50.385666+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.834998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.834998+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:47:15.459581+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":322,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.390056+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:48:49.390056+0000","last_peered":"2024-08-29T15:48:49.390056+0000","last_clean":"2024-08-29T15:48:49.390056+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:48:49.390056+0000","last_undegraded":"2024-08-29T15:48:49.390056+0000","last_fullsized":"2024-08-29T15:48:49.390056+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":59,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386227+0000","last_change":"2024-08-29T15:47:43.263617+0000","last_active":"2024-08-29T15:48:50.386227+0000","last_peered":"2024-08-29T15:48:50.386227+0000","last_clean":"2024-08-29T15:48:50.386227+0000","last_became_active":"2024-08-29T15:47:43.263296+0000","last_became_peered":"2024-08-29T15:47:43.263296+0000","last_unstale":"2024-08-29T15:48:50.386227+0000","last_undegraded":"2024-08-29T15:48:50.386227+0000","last_fullsized":"2024-08-29T15:48:50.386227+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:46:43.176620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"530'14666","reported_seq":25126,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:56.021885+0000","last_change":"2024-08-29T15:48:00.824043+0000","last_active":"2024-08-29T15:48:56.021885+0000","last_peered":"2024-08-29T15:48:56.021885+0000","last_clean":"2024-08-29T15:48:56.021885+0000","last_became_active":"2024-08-29T15:48:00.770733+0000","last_became_peered":"2024-08-29T15:48:00.770733+0000","last_unstale":"2024-08-29T15:48:56.021885+0000","last_undegraded":"2024-08-29T15:48:56.021885+0000","last_fullsized":"2024-08-29T15:48:56.021885+0000","mapping_epoch":512,"log_start":"497'8392","ondisk_log_start":"497'8392","created":480,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6274,"log_dups_size":0,"ondisk_log_size":6274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:03:45.715507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55967744,"num_objects":854,"num_object_clones":0,"num_object_copies":2562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14666,"num_write_kb":496640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5171,"num_bytes_recovered":338886656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":970,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.384053+0000","last_change":"2024-08-29T15:44:59.247796+0000","last_active":"2024-08-29T15:48:49.384053+0000","last_peered":"2024-08-29T15:48:49.384053+0000","last_clean":"2024-08-29T15:48:49.384053+0000","last_became_active":"2024-08-29T15:44:59.246494+0000","last_became_peered":"2024-08-29T15:44:59.246494+0000","last_unstale":"2024-08-29T15:48:49.384053+0000","last_undegraded":"2024-08-29T15:48:49.384053+0000","last_fullsized":"2024-08-29T15:48:49.384053+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:21.844245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:21.844245+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:26.154513+0000","scrub_duration":0.0016201309999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":124,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385486+0000","last_change":"2024-08-29T15:39:53.262167+0000","last_active":"2024-08-29T15:48:50.385486+0000","last_peered":"2024-08-29T15:48:50.385486+0000","last_clean":"2024-08-29T15:48:50.385486+0000","last_became_active":"2024-08-29T15:39:53.258406+0000","last_became_peered":"2024-08-29T15:39:53.258406+0000","last_unstale":"2024-08-29T15:48:50.385486+0000","last_undegraded":"2024-08-29T15:48:50.385486+0000","last_fullsized":"2024-08-29T15:48:50.385486+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:05:19.731559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":556,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387476+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:48:50.387476+0000","last_peered":"2024-08-29T15:48:50.387476+0000","last_clean":"2024-08-29T15:48:50.387476+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:48:50.387476+0000","last_undegraded":"2024-08-29T15:48:50.387476+0000","last_fullsized":"2024-08-29T15:48:50.387476+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"530'14891","reported_seq":15003,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:55.980542+0000","last_change":"2024-08-29T15:45:07.147672+0000","last_active":"2024-08-29T15:48:55.980542+0000","last_peered":"2024-08-29T15:48:55.980542+0000","last_clean":"2024-08-29T15:48:55.980542+0000","last_became_active":"2024-08-29T15:45:07.147363+0000","last_became_peered":"2024-08-29T15:45:07.147363+0000","last_unstale":"2024-08-29T15:48:55.980542+0000","last_undegraded":"2024-08-29T15:48:55.980542+0000","last_fullsized":"2024-08-29T15:48:55.980542+0000","mapping_epoch":480,"log_start":"499'8584","ondisk_log_start":"499'8584","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:03.126615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56164352,"num_objects":857,"num_object_clones":0,"num_object_copies":2571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14891,"num_write_kb":503936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":831,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.390018+0000","last_change":"2024-08-29T15:45:25.871113+0000","last_active":"2024-08-29T15:48:49.390018+0000","last_peered":"2024-08-29T15:48:49.390018+0000","last_clean":"2024-08-29T15:48:49.390018+0000","last_became_active":"2024-08-29T15:44:59.246190+0000","last_became_peered":"2024-08-29T15:44:59.246190+0000","last_unstale":"2024-08-29T15:48:49.390018+0000","last_undegraded":"2024-08-29T15:48:49.390018+0000","last_fullsized":"2024-08-29T15:48:49.390018+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.870329+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.870329+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:42.080608+0000","scrub_duration":0.0042968349999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":426,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383304+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:48:49.383304+0000","last_peered":"2024-08-29T15:48:49.383304+0000","last_clean":"2024-08-29T15:48:49.383304+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:48:49.383304+0000","last_undegraded":"2024-08-29T15:48:49.383304+0000","last_fullsized":"2024-08-29T15:48:49.383304+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":935,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430304+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:48:50.430304+0000","last_peered":"2024-08-29T15:48:50.430304+0000","last_clean":"2024-08-29T15:48:50.430304+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:48:50.430304+0000","last_undegraded":"2024-08-29T15:48:50.430304+0000","last_fullsized":"2024-08-29T15:48:50.430304+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"530'14898","reported_seq":15010,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:56.816868+0000","last_change":"2024-08-29T15:45:06.877939+0000","last_active":"2024-08-29T15:48:56.816868+0000","last_peered":"2024-08-29T15:48:56.816868+0000","last_clean":"2024-08-29T15:48:56.816868+0000","last_became_active":"2024-08-29T15:45:06.877593+0000","last_became_peered":"2024-08-29T15:45:06.877593+0000","last_unstale":"2024-08-29T15:48:56.816868+0000","last_undegraded":"2024-08-29T15:48:56.816868+0000","last_fullsized":"2024-08-29T15:48:56.816868+0000","mapping_epoch":480,"log_start":"494'8224","ondisk_log_start":"494'8224","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:32.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54919168,"num_objects":838,"num_object_clones":0,"num_object_copies":2514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14898,"num_write_kb":503552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":1078,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386126+0000","last_change":"2024-08-29T15:44:30.836092+0000","last_active":"2024-08-29T15:48:50.386126+0000","last_peered":"2024-08-29T15:48:50.386126+0000","last_clean":"2024-08-29T15:48:50.386126+0000","last_became_active":"2024-08-29T15:44:22.624657+0000","last_became_peered":"2024-08-29T15:44:22.624657+0000","last_unstale":"2024-08-29T15:48:50.386126+0000","last_undegraded":"2024-08-29T15:48:50.386126+0000","last_fullsized":"2024-08-29T15:48:50.386126+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:30.836022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:30.836022+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:36:10.537009+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":973,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386606+0000","last_change":"2024-08-29T15:46:58.377694+0000","last_active":"2024-08-29T15:48:50.386606+0000","last_peered":"2024-08-29T15:48:50.386606+0000","last_clean":"2024-08-29T15:48:50.386606+0000","last_became_active":"2024-08-29T15:46:58.372090+0000","last_became_peered":"2024-08-29T15:46:58.372090+0000","last_unstale":"2024-08-29T15:48:50.386606+0000","last_undegraded":"2024-08-29T15:48:50.386606+0000","last_fullsized":"2024-08-29T15:48:50.386606+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:19:50.948014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":310,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386606+0000","last_change":"2024-08-29T15:46:36.598273+0000","last_active":"2024-08-29T15:48:50.386606+0000","last_peered":"2024-08-29T15:48:50.386606+0000","last_clean":"2024-08-29T15:48:50.386606+0000","last_became_active":"2024-08-29T15:46:36.591062+0000","last_became_peered":"2024-08-29T15:46:36.591062+0000","last_unstale":"2024-08-29T15:48:50.386606+0000","last_undegraded":"2024-08-29T15:48:50.386606+0000","last_fullsized":"2024-08-29T15:48:50.386606+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:32:51.573470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"530'15001","reported_seq":23047,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:58.730982+0000","last_change":"2024-08-29T15:48:12.294106+0000","last_active":"2024-08-29T15:48:58.730982+0000","last_peered":"2024-08-29T15:48:58.730982+0000","last_clean":"2024-08-29T15:48:58.730982+0000","last_became_active":"2024-08-29T15:48:12.262915+0000","last_became_peered":"2024-08-29T15:48:12.262915+0000","last_unstale":"2024-08-29T15:48:58.730982+0000","last_undegraded":"2024-08-29T15:48:58.730982+0000","last_fullsized":"2024-08-29T15:48:58.730982+0000","mapping_epoch":515,"log_start":"493'7484","ondisk_log_start":"493'7484","created":480,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:54:51.872371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44892160,"num_objects":685,"num_object_clones":0,"num_object_copies":2055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15001,"num_write_kb":501952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3980,"num_bytes_recovered":260833280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":740,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.307906+0000","last_change":"2024-08-29T15:48:50.307906+0000","last_active":"2024-08-29T15:48:50.307906+0000","last_peered":"2024-08-29T15:48:50.307906+0000","last_clean":"2024-08-29T15:48:50.307906+0000","last_became_active":"2024-08-29T15:48:33.284378+0000","last_became_peered":"2024-08-29T15:48:33.284378+0000","last_unstale":"2024-08-29T15:48:50.307906+0000","last_undegraded":"2024-08-29T15:48:50.307906+0000","last_fullsized":"2024-08-29T15:48:50.307906+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:48:50.307797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:48:50.307797+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:36.775569+0000","scrub_duration":0.0020697210000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":323,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389385+0000","last_change":"2024-08-29T15:47:12.469624+0000","last_active":"2024-08-29T15:48:49.389385+0000","last_peered":"2024-08-29T15:48:49.389385+0000","last_clean":"2024-08-29T15:48:49.389385+0000","last_became_active":"2024-08-29T15:47:12.469286+0000","last_became_peered":"2024-08-29T15:47:12.469286+0000","last_unstale":"2024-08-29T15:48:49.389385+0000","last_undegraded":"2024-08-29T15:48:49.389385+0000","last_fullsized":"2024-08-29T15:48:49.389385+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:09:13.811380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":316,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386335+0000","last_change":"2024-08-29T15:46:36.703991+0000","last_active":"2024-08-29T15:48:50.386335+0000","last_peered":"2024-08-29T15:48:50.386335+0000","last_clean":"2024-08-29T15:48:50.386335+0000","last_became_active":"2024-08-29T15:46:36.703653+0000","last_became_peered":"2024-08-29T15:46:36.703653+0000","last_unstale":"2024-08-29T15:48:50.386335+0000","last_undegraded":"2024-08-29T15:48:50.386335+0000","last_fullsized":"2024-08-29T15:48:50.386335+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:10:01.192871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"530'15002","reported_seq":15113,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:56.840529+0000","last_change":"2024-08-29T15:45:06.875151+0000","last_active":"2024-08-29T15:48:56.840529+0000","last_peered":"2024-08-29T15:48:56.840529+0000","last_clean":"2024-08-29T15:48:56.840529+0000","last_became_active":"2024-08-29T15:45:06.874862+0000","last_became_peered":"2024-08-29T15:45:06.874862+0000","last_unstale":"2024-08-29T15:48:56.840529+0000","last_undegraded":"2024-08-29T15:48:56.840529+0000","last_fullsized":"2024-08-29T15:48:56.840529+0000","mapping_epoch":480,"log_start":"494'8252","ondisk_log_start":"494'8252","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6750,"log_dups_size":0,"ondisk_log_size":6750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:54:54.225355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56754176,"num_objects":866,"num_object_clones":0,"num_object_copies":2598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15002,"num_write_kb":507776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":1100,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385539+0000","last_change":"2024-08-29T15:44:59.243847+0000","last_active":"2024-08-29T15:48:50.385539+0000","last_peered":"2024-08-29T15:48:50.385539+0000","last_clean":"2024-08-29T15:48:50.385539+0000","last_became_active":"2024-08-29T15:44:59.243399+0000","last_became_peered":"2024-08-29T15:44:59.243399+0000","last_unstale":"2024-08-29T15:48:50.385539+0000","last_undegraded":"2024-08-29T15:48:50.385539+0000","last_fullsized":"2024-08-29T15:48:50.385539+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.906140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.906140+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:51:09.311132+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":319,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389274+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:48:49.389274+0000","last_peered":"2024-08-29T15:48:49.389274+0000","last_clean":"2024-08-29T15:48:49.389274+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:48:49.389274+0000","last_undegraded":"2024-08-29T15:48:49.389274+0000","last_fullsized":"2024-08-29T15:48:49.389274+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":560,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.390090+0000","last_change":"2024-08-29T15:47:12.467689+0000","last_active":"2024-08-29T15:48:49.390090+0000","last_peered":"2024-08-29T15:48:49.390090+0000","last_clean":"2024-08-29T15:48:49.390090+0000","last_became_active":"2024-08-29T15:47:12.467400+0000","last_became_peered":"2024-08-29T15:47:12.467400+0000","last_unstale":"2024-08-29T15:48:49.390090+0000","last_undegraded":"2024-08-29T15:48:49.390090+0000","last_fullsized":"2024-08-29T15:48:49.390090+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:45:54.623271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"530'15312","reported_seq":15418,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:55.980943+0000","last_change":"2024-08-29T15:46:36.601176+0000","last_active":"2024-08-29T15:48:55.980943+0000","last_peered":"2024-08-29T15:48:55.980943+0000","last_clean":"2024-08-29T15:48:55.980943+0000","last_became_active":"2024-08-29T15:46:36.589279+0000","last_became_peered":"2024-08-29T15:46:36.589279+0000","last_unstale":"2024-08-29T15:48:55.980943+0000","last_undegraded":"2024-08-29T15:48:55.980943+0000","last_fullsized":"2024-08-29T15:48:55.980943+0000","mapping_epoch":492,"log_start":"499'9004","ondisk_log_start":"499'9004","created":480,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6308,"log_dups_size":0,"ondisk_log_size":6308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:18:23.326237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60686336,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15312,"num_write_kb":519616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":911,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383202+0000","last_change":"2024-08-29T15:44:53.533605+0000","last_active":"2024-08-29T15:48:49.383202+0000","last_peered":"2024-08-29T15:48:49.383202+0000","last_clean":"2024-08-29T15:48:49.383202+0000","last_became_active":"2024-08-29T15:44:22.611492+0000","last_became_peered":"2024-08-29T15:44:22.611492+0000","last_unstale":"2024-08-29T15:48:49.383202+0000","last_undegraded":"2024-08-29T15:48:49.383202+0000","last_fullsized":"2024-08-29T15:48:49.383202+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:53.533488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:53.533488+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:10:06.648003+0000","scrub_duration":0.0059276559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":59,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385748+0000","last_change":"2024-08-29T15:47:43.258937+0000","last_active":"2024-08-29T15:48:50.385748+0000","last_peered":"2024-08-29T15:48:50.385748+0000","last_clean":"2024-08-29T15:48:50.385748+0000","last_became_active":"2024-08-29T15:47:43.258572+0000","last_became_peered":"2024-08-29T15:47:43.258572+0000","last_unstale":"2024-08-29T15:48:50.385748+0000","last_undegraded":"2024-08-29T15:48:50.385748+0000","last_fullsized":"2024-08-29T15:48:50.385748+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:58.220504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":925,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386192+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:48:50.386192+0000","last_peered":"2024-08-29T15:48:50.386192+0000","last_clean":"2024-08-29T15:48:50.386192+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:48:50.386192+0000","last_undegraded":"2024-08-29T15:48:50.386192+0000","last_fullsized":"2024-08-29T15:48:50.386192+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"530'14774","reported_seq":14886,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:56.764096+0000","last_change":"2024-08-29T15:45:06.878140+0000","last_active":"2024-08-29T15:48:56.764096+0000","last_peered":"2024-08-29T15:48:56.764096+0000","last_clean":"2024-08-29T15:48:56.764096+0000","last_became_active":"2024-08-29T15:45:06.877878+0000","last_became_peered":"2024-08-29T15:45:06.877878+0000","last_unstale":"2024-08-29T15:48:56.764096+0000","last_undegraded":"2024-08-29T15:48:56.764096+0000","last_fullsized":"2024-08-29T15:48:56.764096+0000","mapping_epoch":480,"log_start":"493'8038","ondisk_log_start":"493'8038","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:33:28.011162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54919168,"num_objects":838,"num_object_clones":0,"num_object_copies":2514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14774,"num_write_kb":499584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":1078,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430103+0000","last_change":"2024-08-29T15:47:01.394949+0000","last_active":"2024-08-29T15:48:50.430103+0000","last_peered":"2024-08-29T15:48:50.430103+0000","last_clean":"2024-08-29T15:48:50.430103+0000","last_became_active":"2024-08-29T15:47:01.394614+0000","last_became_peered":"2024-08-29T15:47:01.394614+0000","last_unstale":"2024-08-29T15:48:50.430103+0000","last_undegraded":"2024-08-29T15:48:50.430103+0000","last_fullsized":"2024-08-29T15:48:50.430103+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.008533+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.008533+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:06:09.577560+0000","scrub_duration":111,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":1012,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385845+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:48:50.385845+0000","last_peered":"2024-08-29T15:48:50.385845+0000","last_clean":"2024-08-29T15:48:50.385845+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:48:50.385845+0000","last_undegraded":"2024-08-29T15:48:50.385845+0000","last_fullsized":"2024-08-29T15:48:50.385845+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":339,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385481+0000","last_change":"2024-08-29T15:37:26.313495+0000","last_active":"2024-08-29T15:48:50.385481+0000","last_peered":"2024-08-29T15:48:50.385481+0000","last_clean":"2024-08-29T15:48:50.385481+0000","last_became_active":"2024-08-29T15:37:26.313188+0000","last_became_peered":"2024-08-29T15:37:26.313188+0000","last_unstale":"2024-08-29T15:48:50.385481+0000","last_undegraded":"2024-08-29T15:48:50.385481+0000","last_fullsized":"2024-08-29T15:48:50.385481+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:39.580490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":285,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389640+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:48:49.389640+0000","last_peered":"2024-08-29T15:48:49.389640+0000","last_clean":"2024-08-29T15:48:49.389640+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:48:49.389640+0000","last_undegraded":"2024-08-29T15:48:49.389640+0000","last_fullsized":"2024-08-29T15:48:49.389640+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"530'14766","reported_seq":14878,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:55.950980+0000","last_change":"2024-08-29T15:45:07.146463+0000","last_active":"2024-08-29T15:48:55.950980+0000","last_peered":"2024-08-29T15:48:55.950980+0000","last_clean":"2024-08-29T15:48:55.950980+0000","last_became_active":"2024-08-29T15:45:07.146148+0000","last_became_peered":"2024-08-29T15:45:07.146148+0000","last_unstale":"2024-08-29T15:48:55.950980+0000","last_undegraded":"2024-08-29T15:48:55.950980+0000","last_fullsized":"2024-08-29T15:48:55.950980+0000","mapping_epoch":480,"log_start":"499'8466","ondisk_log_start":"499'8466","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6300,"log_dups_size":0,"ondisk_log_size":6300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:04.649404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14766,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":1077,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385810+0000","last_change":"2024-08-29T15:43:26.588054+0000","last_active":"2024-08-29T15:48:50.385810+0000","last_peered":"2024-08-29T15:48:50.385810+0000","last_clean":"2024-08-29T15:48:50.385810+0000","last_became_active":"2024-08-29T15:43:26.285267+0000","last_became_peered":"2024-08-29T15:43:26.285267+0000","last_unstale":"2024-08-29T15:48:50.385810+0000","last_undegraded":"2024-08-29T15:48:50.385810+0000","last_fullsized":"2024-08-29T15:48:50.385810+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.587986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.587986+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:15:21.198019+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1399,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385362+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:48:50.385362+0000","last_peered":"2024-08-29T15:48:50.385362+0000","last_clean":"2024-08-29T15:48:50.385362+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:48:50.385362+0000","last_undegraded":"2024-08-29T15:48:50.385362+0000","last_fullsized":"2024-08-29T15:48:50.385362+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":965,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429483+0000","last_change":"2024-08-29T15:47:42.288211+0000","last_active":"2024-08-29T15:48:50.429483+0000","last_peered":"2024-08-29T15:48:50.429483+0000","last_clean":"2024-08-29T15:48:50.429483+0000","last_became_active":"2024-08-29T15:47:42.287594+0000","last_became_peered":"2024-08-29T15:47:42.287594+0000","last_unstale":"2024-08-29T15:48:50.429483+0000","last_undegraded":"2024-08-29T15:48:50.429483+0000","last_fullsized":"2024-08-29T15:48:50.429483+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:32:18.869863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":80,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429534+0000","last_change":"2024-08-29T15:46:58.369628+0000","last_active":"2024-08-29T15:48:50.429534+0000","last_peered":"2024-08-29T15:48:50.429534+0000","last_clean":"2024-08-29T15:48:50.429534+0000","last_became_active":"2024-08-29T15:46:58.369315+0000","last_became_peered":"2024-08-29T15:46:58.369315+0000","last_unstale":"2024-08-29T15:48:50.429534+0000","last_undegraded":"2024-08-29T15:48:50.429534+0000","last_fullsized":"2024-08-29T15:48:50.429534+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:52:05.895354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"530'14961","reported_seq":18732,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:58.722814+0000","last_change":"2024-08-29T15:48:46.403491+0000","last_active":"2024-08-29T15:48:58.722814+0000","last_peered":"2024-08-29T15:48:58.722814+0000","last_clean":"2024-08-29T15:48:58.722814+0000","last_became_active":"2024-08-29T15:48:46.352035+0000","last_became_peered":"2024-08-29T15:48:46.352035+0000","last_unstale":"2024-08-29T15:48:58.722814+0000","last_undegraded":"2024-08-29T15:48:58.722814+0000","last_fullsized":"2024-08-29T15:48:58.722814+0000","mapping_epoch":526,"log_start":"493'7600","ondisk_log_start":"493'7600","created":480,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:03.034399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49741824,"num_objects":759,"num_object_clones":0,"num_object_copies":2277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14961,"num_write_kb":503040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1845,"num_bytes_recovered":120913920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":1105,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430066+0000","last_change":"2024-08-29T15:48:15.289740+0000","last_active":"2024-08-29T15:48:50.430066+0000","last_peered":"2024-08-29T15:48:50.430066+0000","last_clean":"2024-08-29T15:48:50.430066+0000","last_became_active":"2024-08-29T15:48:15.289419+0000","last_became_peered":"2024-08-29T15:48:15.289419+0000","last_unstale":"2024-08-29T15:48:50.430066+0000","last_undegraded":"2024-08-29T15:48:50.430066+0000","last_fullsized":"2024-08-29T15:48:50.430066+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.841106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.841106+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:04:15.648107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":967,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386317+0000","last_change":"2024-08-29T15:37:29.345555+0000","last_active":"2024-08-29T15:48:50.386317+0000","last_peered":"2024-08-29T15:48:50.386317+0000","last_clean":"2024-08-29T15:48:50.386317+0000","last_became_active":"2024-08-29T15:37:29.344738+0000","last_became_peered":"2024-08-29T15:37:29.344738+0000","last_unstale":"2024-08-29T15:48:50.386317+0000","last_undegraded":"2024-08-29T15:48:50.386317+0000","last_fullsized":"2024-08-29T15:48:50.386317+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:14.103195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":272,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.385637+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:48:49.385637+0000","last_peered":"2024-08-29T15:48:49.385637+0000","last_clean":"2024-08-29T15:48:49.385637+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:48:49.385637+0000","last_undegraded":"2024-08-29T15:48:49.385637+0000","last_fullsized":"2024-08-29T15:48:49.385637+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385983+0000","last_change":"2024-08-29T15:47:12.472465+0000","last_active":"2024-08-29T15:48:50.385983+0000","last_peered":"2024-08-29T15:48:50.385983+0000","last_clean":"2024-08-29T15:48:50.385983+0000","last_became_active":"2024-08-29T15:47:12.472167+0000","last_became_peered":"2024-08-29T15:47:12.472167+0000","last_unstale":"2024-08-29T15:48:50.385983+0000","last_undegraded":"2024-08-29T15:48:50.385983+0000","last_fullsized":"2024-08-29T15:48:50.385983+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:44:56.401983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"530'15174","reported_seq":21533,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:56.008115+0000","last_change":"2024-08-29T15:48:31.428964+0000","last_active":"2024-08-29T15:48:56.008115+0000","last_peered":"2024-08-29T15:48:56.008115+0000","last_clean":"2024-08-29T15:48:56.008115+0000","last_became_active":"2024-08-29T15:48:31.362574+0000","last_became_peered":"2024-08-29T15:48:31.362574+0000","last_unstale":"2024-08-29T15:48:56.008115+0000","last_undegraded":"2024-08-29T15:48:56.008115+0000","last_fullsized":"2024-08-29T15:48:56.008115+0000","mapping_epoch":521,"log_start":"499'8895","ondisk_log_start":"499'8895","created":480,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:27:20.861141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15174,"num_write_kb":513920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3126,"num_bytes_recovered":204865536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":942,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383110+0000","last_change":"2024-08-29T15:47:01.407034+0000","last_active":"2024-08-29T15:48:49.383110+0000","last_peered":"2024-08-29T15:48:49.383110+0000","last_clean":"2024-08-29T15:48:49.383110+0000","last_became_active":"2024-08-29T15:47:01.406648+0000","last_became_peered":"2024-08-29T15:47:01.406648+0000","last_unstale":"2024-08-29T15:48:49.383110+0000","last_undegraded":"2024-08-29T15:48:49.383110+0000","last_fullsized":"2024-08-29T15:48:49.383110+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_clean_scrub_stamp":"2024-08-29T15:44:26.709569+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:15:11.694724+0000","scrub_duration":0.0031838119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":480,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385614+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:48:50.385614+0000","last_peered":"2024-08-29T15:48:50.385614+0000","last_clean":"2024-08-29T15:48:50.385614+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:48:50.385614+0000","last_undegraded":"2024-08-29T15:48:50.385614+0000","last_fullsized":"2024-08-29T15:48:50.385614+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":319,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389444+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:48:49.389444+0000","last_peered":"2024-08-29T15:48:49.389444+0000","last_clean":"2024-08-29T15:48:49.389444+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:48:49.389444+0000","last_undegraded":"2024-08-29T15:48:49.389444+0000","last_fullsized":"2024-08-29T15:48:49.389444+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":387,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385388+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:48:50.385388+0000","last_peered":"2024-08-29T15:48:50.385388+0000","last_clean":"2024-08-29T15:48:50.385388+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:48:50.385388+0000","last_undegraded":"2024-08-29T15:48:50.385388+0000","last_fullsized":"2024-08-29T15:48:50.385388+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"530'14656","reported_seq":14762,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:56.021550+0000","last_change":"2024-08-29T15:46:36.596834+0000","last_active":"2024-08-29T15:48:56.021550+0000","last_peered":"2024-08-29T15:48:56.021550+0000","last_clean":"2024-08-29T15:48:56.021550+0000","last_became_active":"2024-08-29T15:46:36.586137+0000","last_became_peered":"2024-08-29T15:46:36.586137+0000","last_unstale":"2024-08-29T15:48:56.021550+0000","last_undegraded":"2024-08-29T15:48:56.021550+0000","last_fullsized":"2024-08-29T15:48:56.021550+0000","mapping_epoch":492,"log_start":"497'8319","ondisk_log_start":"497'8319","created":480,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:18:18.073025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57278464,"num_objects":874,"num_object_clones":0,"num_object_copies":2622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14656,"num_write_kb":496960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":994,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387519+0000","last_change":"2024-08-29T15:47:46.311743+0000","last_active":"2024-08-29T15:48:50.387519+0000","last_peered":"2024-08-29T15:48:50.387519+0000","last_clean":"2024-08-29T15:48:50.387519+0000","last_became_active":"2024-08-29T15:47:46.311238+0000","last_became_peered":"2024-08-29T15:47:46.311238+0000","last_unstale":"2024-08-29T15:48:50.387519+0000","last_undegraded":"2024-08-29T15:48:50.387519+0000","last_fullsized":"2024-08-29T15:48:50.387519+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.292191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.292191+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:43.047588+0000","scrub_duration":0.0075757899999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":412,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385557+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:48:50.385557+0000","last_peered":"2024-08-29T15:48:50.385557+0000","last_clean":"2024-08-29T15:48:50.385557+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:48:50.385557+0000","last_undegraded":"2024-08-29T15:48:50.385557+0000","last_fullsized":"2024-08-29T15:48:50.385557+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1363,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385982+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:48:50.385982+0000","last_peered":"2024-08-29T15:48:50.385982+0000","last_clean":"2024-08-29T15:48:50.385982+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:48:50.385982+0000","last_undegraded":"2024-08-29T15:48:50.385982+0000","last_fullsized":"2024-08-29T15:48:50.385982+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387343+0000","last_change":"2024-08-29T15:47:12.474328+0000","last_active":"2024-08-29T15:48:50.387343+0000","last_peered":"2024-08-29T15:48:50.387343+0000","last_clean":"2024-08-29T15:48:50.387343+0000","last_became_active":"2024-08-29T15:47:12.473903+0000","last_became_peered":"2024-08-29T15:47:12.473903+0000","last_unstale":"2024-08-29T15:48:50.387343+0000","last_undegraded":"2024-08-29T15:48:50.387343+0000","last_fullsized":"2024-08-29T15:48:50.387343+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:42.229361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"530'14830","reported_seq":19082,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:56.021683+0000","last_change":"2024-08-29T15:48:47.433077+0000","last_active":"2024-08-29T15:48:56.021683+0000","last_peered":"2024-08-29T15:48:56.021683+0000","last_clean":"2024-08-29T15:48:56.021683+0000","last_became_active":"2024-08-29T15:48:47.367348+0000","last_became_peered":"2024-08-29T15:48:47.367348+0000","last_unstale":"2024-08-29T15:48:56.021683+0000","last_undegraded":"2024-08-29T15:48:56.021683+0000","last_fullsized":"2024-08-29T15:48:56.021683+0000","mapping_epoch":527,"log_start":"498'8493","ondisk_log_start":"498'8493","created":480,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:14:04.463759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59965464,"num_objects":916,"num_object_clones":0,"num_object_copies":2748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":916,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14830,"num_write_kb":503809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2065,"num_bytes_recovered":135266328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":938,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429425+0000","last_change":"2024-08-29T15:47:01.410197+0000","last_active":"2024-08-29T15:48:50.429425+0000","last_peered":"2024-08-29T15:48:50.429425+0000","last_clean":"2024-08-29T15:48:50.429425+0000","last_became_active":"2024-08-29T15:47:01.406541+0000","last_became_peered":"2024-08-29T15:47:01.406541+0000","last_unstale":"2024-08-29T15:48:50.429425+0000","last_undegraded":"2024-08-29T15:48:50.429425+0000","last_fullsized":"2024-08-29T15:48:50.429425+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:23.864183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:23.864183+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:13:01.243784+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":945,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385948+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:48:50.385948+0000","last_peered":"2024-08-29T15:48:50.385948+0000","last_clean":"2024-08-29T15:48:50.385948+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:48:50.385948+0000","last_undegraded":"2024-08-29T15:48:50.385948+0000","last_fullsized":"2024-08-29T15:48:50.385948+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":1049,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.386238+0000","last_change":"2024-08-29T15:46:58.399147+0000","last_active":"2024-08-29T15:48:49.386238+0000","last_peered":"2024-08-29T15:48:49.386238+0000","last_clean":"2024-08-29T15:48:49.386238+0000","last_became_active":"2024-08-29T15:46:58.371595+0000","last_became_peered":"2024-08-29T15:46:58.371595+0000","last_unstale":"2024-08-29T15:48:49.386238+0000","last_undegraded":"2024-08-29T15:48:49.386238+0000","last_fullsized":"2024-08-29T15:48:49.386238+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:46:46.943596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":936,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430159+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:48:50.430159+0000","last_peered":"2024-08-29T15:48:50.430159+0000","last_clean":"2024-08-29T15:48:50.430159+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:48:50.430159+0000","last_undegraded":"2024-08-29T15:48:50.430159+0000","last_fullsized":"2024-08-29T15:48:50.430159+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"530'14922","reported_seq":14982,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:55.059645+0000","last_change":"2024-08-29T15:45:07.146261+0000","last_active":"2024-08-29T15:48:55.059645+0000","last_peered":"2024-08-29T15:48:55.059645+0000","last_clean":"2024-08-29T15:48:55.059645+0000","last_became_active":"2024-08-29T15:45:07.145979+0000","last_became_peered":"2024-08-29T15:45:07.145979+0000","last_unstale":"2024-08-29T15:48:55.059645+0000","last_undegraded":"2024-08-29T15:48:55.059645+0000","last_fullsized":"2024-08-29T15:48:55.059645+0000","mapping_epoch":480,"log_start":"500'8900","ondisk_log_start":"500'8900","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6022,"log_dups_size":0,"ondisk_log_size":6022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:28.876377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59375616,"num_objects":906,"num_object_clones":0,"num_object_copies":2718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14922,"num_write_kb":506496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":946,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389455+0000","last_change":"2024-08-29T15:44:22.861452+0000","last_active":"2024-08-29T15:48:49.389455+0000","last_peered":"2024-08-29T15:48:49.389455+0000","last_clean":"2024-08-29T15:48:49.389455+0000","last_became_active":"2024-08-29T15:44:22.611166+0000","last_became_peered":"2024-08-29T15:44:22.611166+0000","last_unstale":"2024-08-29T15:48:49.389455+0000","last_undegraded":"2024-08-29T15:48:49.389455+0000","last_fullsized":"2024-08-29T15:48:49.389455+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.861280+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.861280+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:45.254970+0000","scrub_duration":0.0040975610000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":930,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387804+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:48:50.387804+0000","last_peered":"2024-08-29T15:48:50.387804+0000","last_clean":"2024-08-29T15:48:50.387804+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:48:50.387804+0000","last_undegraded":"2024-08-29T15:48:50.387804+0000","last_fullsized":"2024-08-29T15:48:50.387804+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1581,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386277+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:48:50.386277+0000","last_peered":"2024-08-29T15:48:50.386277+0000","last_clean":"2024-08-29T15:48:50.386277+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:48:50.386277+0000","last_undegraded":"2024-08-29T15:48:50.386277+0000","last_fullsized":"2024-08-29T15:48:50.386277+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":282,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385848+0000","last_change":"2024-08-29T15:48:12.266718+0000","last_active":"2024-08-29T15:48:50.385848+0000","last_peered":"2024-08-29T15:48:50.385848+0000","last_clean":"2024-08-29T15:48:50.385848+0000","last_became_active":"2024-08-29T15:48:12.266363+0000","last_became_peered":"2024-08-29T15:48:12.266363+0000","last_unstale":"2024-08-29T15:48:50.385848+0000","last_undegraded":"2024-08-29T15:48:50.385848+0000","last_fullsized":"2024-08-29T15:48:50.385848+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:53.550145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"530'15006","reported_seq":15118,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:57.321826+0000","last_change":"2024-08-29T15:45:07.148809+0000","last_active":"2024-08-29T15:48:57.321826+0000","last_peered":"2024-08-29T15:48:57.321826+0000","last_clean":"2024-08-29T15:48:57.321826+0000","last_became_active":"2024-08-29T15:45:07.148514+0000","last_became_peered":"2024-08-29T15:45:07.148514+0000","last_unstale":"2024-08-29T15:48:57.321826+0000","last_undegraded":"2024-08-29T15:48:57.321826+0000","last_fullsized":"2024-08-29T15:48:57.321826+0000","mapping_epoch":480,"log_start":"499'8598","ondisk_log_start":"499'8598","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:42:28.486131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15006,"num_write_kb":505856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":1109,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385557+0000","last_change":"2024-08-29T15:43:26.775058+0000","last_active":"2024-08-29T15:48:50.385557+0000","last_peered":"2024-08-29T15:48:50.385557+0000","last_clean":"2024-08-29T15:48:50.385557+0000","last_became_active":"2024-08-29T15:43:26.284121+0000","last_became_peered":"2024-08-29T15:43:26.284121+0000","last_unstale":"2024-08-29T15:48:50.385557+0000","last_undegraded":"2024-08-29T15:48:50.385557+0000","last_fullsized":"2024-08-29T15:48:50.385557+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.774861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.774861+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:50:14.013328+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":84,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429739+0000","last_change":"2024-08-29T15:46:58.380711+0000","last_active":"2024-08-29T15:48:50.429739+0000","last_peered":"2024-08-29T15:48:50.429739+0000","last_clean":"2024-08-29T15:48:50.429739+0000","last_became_active":"2024-08-29T15:46:58.369572+0000","last_became_peered":"2024-08-29T15:46:58.369572+0000","last_unstale":"2024-08-29T15:48:50.429739+0000","last_undegraded":"2024-08-29T15:48:50.429739+0000","last_fullsized":"2024-08-29T15:48:50.429739+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:49:53.912727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":333,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.385600+0000","last_change":"2024-08-29T15:48:12.274839+0000","last_active":"2024-08-29T15:48:49.385600+0000","last_peered":"2024-08-29T15:48:49.385600+0000","last_clean":"2024-08-29T15:48:49.385600+0000","last_became_active":"2024-08-29T15:48:12.268376+0000","last_became_peered":"2024-08-29T15:48:12.268376+0000","last_unstale":"2024-08-29T15:48:49.385600+0000","last_undegraded":"2024-08-29T15:48:49.385600+0000","last_fullsized":"2024-08-29T15:48:49.385600+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:19.911889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":278,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383196+0000","last_change":"2024-08-29T15:46:36.588929+0000","last_active":"2024-08-29T15:48:49.383196+0000","last_peered":"2024-08-29T15:48:49.383196+0000","last_clean":"2024-08-29T15:48:49.383196+0000","last_became_active":"2024-08-29T15:46:36.588017+0000","last_became_peered":"2024-08-29T15:46:36.588017+0000","last_unstale":"2024-08-29T15:48:49.383196+0000","last_undegraded":"2024-08-29T15:48:49.383196+0000","last_fullsized":"2024-08-29T15:48:49.383196+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:45:31.813350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"90'149","reported_seq":937,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385790+0000","last_change":"2024-08-29T15:44:59.239507+0000","last_active":"2024-08-29T15:48:50.385790+0000","last_peered":"2024-08-29T15:48:50.385790+0000","last_clean":"2024-08-29T15:48:50.385790+0000","last_became_active":"2024-08-29T15:44:59.239049+0000","last_became_peered":"2024-08-29T15:44:59.239049+0000","last_unstale":"2024-08-29T15:48:50.385790+0000","last_undegraded":"2024-08-29T15:48:50.385790+0000","last_fullsized":"2024-08-29T15:48:50.385790+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:28.746327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:28.746327+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:38:22.918512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":318,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389595+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:48:49.389595+0000","last_peered":"2024-08-29T15:48:49.389595+0000","last_clean":"2024-08-29T15:48:49.389595+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:48:49.389595+0000","last_undegraded":"2024-08-29T15:48:49.389595+0000","last_fullsized":"2024-08-29T15:48:49.389595+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":328,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389698+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:48:49.389698+0000","last_peered":"2024-08-29T15:48:49.389698+0000","last_clean":"2024-08-29T15:48:49.389698+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:48:49.389698+0000","last_undegraded":"2024-08-29T15:48:49.389698+0000","last_fullsized":"2024-08-29T15:48:49.389698+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"90'149","reported_seq":998,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383950+0000","last_change":"2024-08-29T15:47:01.397530+0000","last_active":"2024-08-29T15:48:49.383950+0000","last_peered":"2024-08-29T15:48:49.383950+0000","last_clean":"2024-08-29T15:48:49.383950+0000","last_became_active":"2024-08-29T15:47:01.397287+0000","last_became_peered":"2024-08-29T15:47:01.397287+0000","last_unstale":"2024-08-29T15:48:49.383950+0000","last_undegraded":"2024-08-29T15:48:49.383950+0000","last_fullsized":"2024-08-29T15:48:49.383950+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:28.337426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:28.337426+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:58:58.372823+0000","scrub_duration":0.001450535,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387195+0000","last_change":"2024-08-29T15:47:12.486814+0000","last_active":"2024-08-29T15:48:50.387195+0000","last_peered":"2024-08-29T15:48:50.387195+0000","last_clean":"2024-08-29T15:48:50.387195+0000","last_became_active":"2024-08-29T15:47:12.486372+0000","last_became_peered":"2024-08-29T15:47:12.486372+0000","last_unstale":"2024-08-29T15:48:50.387195+0000","last_undegraded":"2024-08-29T15:48:50.387195+0000","last_fullsized":"2024-08-29T15:48:50.387195+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:38.593835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":692,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386942+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:48:50.386942+0000","last_peered":"2024-08-29T15:48:50.386942+0000","last_clean":"2024-08-29T15:48:50.386942+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:48:50.386942+0000","last_undegraded":"2024-08-29T15:48:50.386942+0000","last_fullsized":"2024-08-29T15:48:50.386942+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'149","reported_seq":1112,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429140+0000","last_change":"2024-08-29T15:44:59.244921+0000","last_active":"2024-08-29T15:48:50.429140+0000","last_peered":"2024-08-29T15:48:50.429140+0000","last_clean":"2024-08-29T15:48:50.429140+0000","last_became_active":"2024-08-29T15:44:59.244391+0000","last_became_peered":"2024-08-29T15:44:59.244391+0000","last_unstale":"2024-08-29T15:48:50.429140+0000","last_undegraded":"2024-08-29T15:48:50.429140+0000","last_fullsized":"2024-08-29T15:48:50.429140+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.778004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.778004+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:12:04.168824+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":407,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383392+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:48:49.383392+0000","last_peered":"2024-08-29T15:48:49.383392+0000","last_clean":"2024-08-29T15:48:49.383392+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:48:49.383392+0000","last_undegraded":"2024-08-29T15:48:49.383392+0000","last_fullsized":"2024-08-29T15:48:49.383392+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":893,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386160+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:48:50.386160+0000","last_peered":"2024-08-29T15:48:50.386160+0000","last_clean":"2024-08-29T15:48:50.386160+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:48:50.386160+0000","last_undegraded":"2024-08-29T15:48:50.386160+0000","last_fullsized":"2024-08-29T15:48:50.386160+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"90'149","reported_seq":841,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.386159+0000","last_change":"2024-08-29T15:45:26.882752+0000","last_active":"2024-08-29T15:48:49.386159+0000","last_peered":"2024-08-29T15:48:49.386159+0000","last_clean":"2024-08-29T15:48:49.386159+0000","last_became_active":"2024-08-29T15:44:59.248534+0000","last_became_peered":"2024-08-29T15:44:59.248534+0000","last_unstale":"2024-08-29T15:48:49.386159+0000","last_undegraded":"2024-08-29T15:48:49.386159+0000","last_fullsized":"2024-08-29T15:48:49.386159+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.882638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.882638+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:29:07.801722+0000","scrub_duration":0.001489043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":80,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430619+0000","last_change":"2024-08-29T15:46:58.368403+0000","last_active":"2024-08-29T15:48:50.430619+0000","last_peered":"2024-08-29T15:48:50.430619+0000","last_clean":"2024-08-29T15:48:50.430619+0000","last_became_active":"2024-08-29T15:46:58.368078+0000","last_became_peered":"2024-08-29T15:46:58.368078+0000","last_unstale":"2024-08-29T15:48:50.430619+0000","last_undegraded":"2024-08-29T15:48:50.430619+0000","last_fullsized":"2024-08-29T15:48:50.430619+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:08:22.938874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":945,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385980+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:48:50.385980+0000","last_peered":"2024-08-29T15:48:50.385980+0000","last_clean":"2024-08-29T15:48:50.385980+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:48:50.385980+0000","last_undegraded":"2024-08-29T15:48:50.385980+0000","last_fullsized":"2024-08-29T15:48:50.385980+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"90'149","reported_seq":773,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389235+0000","last_change":"2024-08-29T15:45:26.912257+0000","last_active":"2024-08-29T15:48:49.389235+0000","last_peered":"2024-08-29T15:48:49.389235+0000","last_clean":"2024-08-29T15:48:49.389235+0000","last_became_active":"2024-08-29T15:44:59.244250+0000","last_became_peered":"2024-08-29T15:44:59.244250+0000","last_unstale":"2024-08-29T15:48:49.389235+0000","last_undegraded":"2024-08-29T15:48:49.389235+0000","last_fullsized":"2024-08-29T15:48:49.389235+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.912179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.912179+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:51.014992+0000","scrub_duration":0.0012045789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":389,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.384148+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:48:49.384148+0000","last_peered":"2024-08-29T15:48:49.384148+0000","last_clean":"2024-08-29T15:48:49.384148+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:48:49.384148+0000","last_undegraded":"2024-08-29T15:48:49.384148+0000","last_fullsized":"2024-08-29T15:48:49.384148+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":187,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430006+0000","last_change":"2024-08-29T15:47:12.486034+0000","last_active":"2024-08-29T15:48:50.430006+0000","last_peered":"2024-08-29T15:48:50.430006+0000","last_clean":"2024-08-29T15:48:50.430006+0000","last_became_active":"2024-08-29T15:47:12.483444+0000","last_became_peered":"2024-08-29T15:47:12.483444+0000","last_unstale":"2024-08-29T15:48:50.430006+0000","last_undegraded":"2024-08-29T15:48:50.430006+0000","last_fullsized":"2024-08-29T15:48:50.430006+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:52:10.488628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"90'149","reported_seq":1058,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.386584+0000","last_change":"2024-08-29T15:47:01.398313+0000","last_active":"2024-08-29T15:48:49.386584+0000","last_peered":"2024-08-29T15:48:49.386584+0000","last_clean":"2024-08-29T15:48:49.386584+0000","last_became_active":"2024-08-29T15:47:01.397916+0000","last_became_peered":"2024-08-29T15:47:01.397916+0000","last_unstale":"2024-08-29T15:48:49.386584+0000","last_undegraded":"2024-08-29T15:48:49.386584+0000","last_fullsized":"2024-08-29T15:48:49.386584+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:23.954739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:23.954739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:01:09.974317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":94,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429850+0000","last_change":"2024-08-29T15:48:12.267373+0000","last_active":"2024-08-29T15:48:50.429850+0000","last_peered":"2024-08-29T15:48:50.429850+0000","last_clean":"2024-08-29T15:48:50.429850+0000","last_became_active":"2024-08-29T15:48:12.267059+0000","last_became_peered":"2024-08-29T15:48:12.267059+0000","last_unstale":"2024-08-29T15:48:50.429850+0000","last_undegraded":"2024-08-29T15:48:50.429850+0000","last_fullsized":"2024-08-29T15:48:50.429850+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:25:55.282346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":924,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385410+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:48:50.385410+0000","last_peered":"2024-08-29T15:48:50.385410+0000","last_clean":"2024-08-29T15:48:50.385410+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:48:50.385410+0000","last_undegraded":"2024-08-29T15:48:50.385410+0000","last_fullsized":"2024-08-29T15:48:50.385410+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'149","reported_seq":785,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383193+0000","last_change":"2024-08-29T15:45:27.312630+0000","last_active":"2024-08-29T15:48:49.383193+0000","last_peered":"2024-08-29T15:48:49.383193+0000","last_clean":"2024-08-29T15:48:49.383193+0000","last_became_active":"2024-08-29T15:44:59.247776+0000","last_became_peered":"2024-08-29T15:44:59.247776+0000","last_unstale":"2024-08-29T15:48:49.383193+0000","last_undegraded":"2024-08-29T15:48:49.383193+0000","last_fullsized":"2024-08-29T15:48:49.383193+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:27.312539+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:27.312539+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:07.441954+0000","scrub_duration":0.0034059530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":928,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386789+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:48:50.386789+0000","last_peered":"2024-08-29T15:48:50.386789+0000","last_clean":"2024-08-29T15:48:50.386789+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:48:50.386789+0000","last_undegraded":"2024-08-29T15:48:50.386789+0000","last_fullsized":"2024-08-29T15:48:50.386789+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":317,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389485+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:48:49.389485+0000","last_peered":"2024-08-29T15:48:49.389485+0000","last_clean":"2024-08-29T15:48:49.389485+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:48:49.389485+0000","last_undegraded":"2024-08-29T15:48:49.389485+0000","last_fullsized":"2024-08-29T15:48:49.389485+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'149","reported_seq":1096,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386077+0000","last_change":"2024-08-29T15:47:01.424749+0000","last_active":"2024-08-29T15:48:50.386077+0000","last_peered":"2024-08-29T15:48:50.386077+0000","last_clean":"2024-08-29T15:48:50.386077+0000","last_became_active":"2024-08-29T15:47:01.424333+0000","last_became_peered":"2024-08-29T15:47:01.424333+0000","last_unstale":"2024-08-29T15:48:50.386077+0000","last_undegraded":"2024-08-29T15:48:50.386077+0000","last_fullsized":"2024-08-29T15:48:50.386077+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:31.851590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:31.851590+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:01.272901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":969,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386046+0000","last_change":"2024-08-29T15:39:11.832387+0000","last_active":"2024-08-29T15:48:50.386046+0000","last_peered":"2024-08-29T15:48:50.386046+0000","last_clean":"2024-08-29T15:48:50.386046+0000","last_became_active":"2024-08-29T15:39:11.832056+0000","last_became_peered":"2024-08-29T15:39:11.832056+0000","last_unstale":"2024-08-29T15:48:50.386046+0000","last_undegraded":"2024-08-29T15:48:50.386046+0000","last_fullsized":"2024-08-29T15:48:50.386046+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:47:42.049015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":80,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429591+0000","last_change":"2024-08-29T15:46:58.367038+0000","last_active":"2024-08-29T15:48:50.429591+0000","last_peered":"2024-08-29T15:48:50.429591+0000","last_clean":"2024-08-29T15:48:50.429591+0000","last_became_active":"2024-08-29T15:46:58.366692+0000","last_became_peered":"2024-08-29T15:46:58.366692+0000","last_unstale":"2024-08-29T15:48:50.429591+0000","last_undegraded":"2024-08-29T15:48:50.429591+0000","last_fullsized":"2024-08-29T15:48:50.429591+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:40:02.240790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"90'149","reported_seq":943,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386896+0000","last_change":"2024-08-29T15:44:59.241503+0000","last_active":"2024-08-29T15:48:50.386896+0000","last_peered":"2024-08-29T15:48:50.386896+0000","last_clean":"2024-08-29T15:48:50.386896+0000","last_became_active":"2024-08-29T15:44:59.241088+0000","last_became_peered":"2024-08-29T15:44:59.241088+0000","last_unstale":"2024-08-29T15:48:50.386896+0000","last_undegraded":"2024-08-29T15:48:50.386896+0000","last_fullsized":"2024-08-29T15:48:50.386896+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:27.788587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:27.788587+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:39.021866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":353,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389357+0000","last_change":"2024-08-29T15:47:12.481314+0000","last_active":"2024-08-29T15:48:49.389357+0000","last_peered":"2024-08-29T15:48:49.389357+0000","last_clean":"2024-08-29T15:48:49.389357+0000","last_became_active":"2024-08-29T15:47:12.481010+0000","last_became_peered":"2024-08-29T15:47:12.481010+0000","last_unstale":"2024-08-29T15:48:49.389357+0000","last_undegraded":"2024-08-29T15:48:49.389357+0000","last_fullsized":"2024-08-29T15:48:49.389357+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:58:04.897625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":450,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383207+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:48:49.383207+0000","last_peered":"2024-08-29T15:48:49.383207+0000","last_clean":"2024-08-29T15:48:49.383207+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:48:49.383207+0000","last_undegraded":"2024-08-29T15:48:49.383207+0000","last_fullsized":"2024-08-29T15:48:49.383207+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":228,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385646+0000","last_change":"2024-08-29T15:46:36.588938+0000","last_active":"2024-08-29T15:48:50.385646+0000","last_peered":"2024-08-29T15:48:50.385646+0000","last_clean":"2024-08-29T15:48:50.385646+0000","last_became_active":"2024-08-29T15:46:36.588605+0000","last_became_peered":"2024-08-29T15:46:36.588605+0000","last_unstale":"2024-08-29T15:48:50.385646+0000","last_undegraded":"2024-08-29T15:48:50.385646+0000","last_fullsized":"2024-08-29T15:48:50.385646+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:29.576817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":450,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383528+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:48:49.383528+0000","last_peered":"2024-08-29T15:48:49.383528+0000","last_clean":"2024-08-29T15:48:49.383528+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:48:49.383528+0000","last_undegraded":"2024-08-29T15:48:49.383528+0000","last_fullsized":"2024-08-29T15:48:49.383528+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"90'149","reported_seq":1127,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385732+0000","last_change":"2024-08-29T15:44:59.243558+0000","last_active":"2024-08-29T15:48:50.385732+0000","last_peered":"2024-08-29T15:48:50.385732+0000","last_clean":"2024-08-29T15:48:50.385732+0000","last_became_active":"2024-08-29T15:44:59.243114+0000","last_became_peered":"2024-08-29T15:44:59.243114+0000","last_unstale":"2024-08-29T15:48:50.385732+0000","last_undegraded":"2024-08-29T15:48:50.385732+0000","last_fullsized":"2024-08-29T15:48:50.385732+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:25.805561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:25.805561+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:03:59.085143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":960,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386335+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:48:50.386335+0000","last_peered":"2024-08-29T15:48:50.386335+0000","last_clean":"2024-08-29T15:48:50.386335+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:48:50.386335+0000","last_undegraded":"2024-08-29T15:48:50.386335+0000","last_fullsized":"2024-08-29T15:48:50.386335+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":963,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387024+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:48:50.387024+0000","last_peered":"2024-08-29T15:48:50.387024+0000","last_clean":"2024-08-29T15:48:50.387024+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:48:50.387024+0000","last_undegraded":"2024-08-29T15:48:50.387024+0000","last_fullsized":"2024-08-29T15:48:50.387024+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'149","reported_seq":914,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385414+0000","last_change":"2024-08-29T15:47:46.316018+0000","last_active":"2024-08-29T15:48:50.385414+0000","last_peered":"2024-08-29T15:48:50.385414+0000","last_clean":"2024-08-29T15:48:50.385414+0000","last_became_active":"2024-08-29T15:47:46.315610+0000","last_became_peered":"2024-08-29T15:47:46.315610+0000","last_unstale":"2024-08-29T15:48:50.385414+0000","last_undegraded":"2024-08-29T15:48:50.385414+0000","last_fullsized":"2024-08-29T15:48:50.385414+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:24.970456+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:45:24.970456+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:30:31.344363+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":318,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.068951+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:48:50.068951+0000","last_peered":"2024-08-29T15:48:50.068951+0000","last_clean":"2024-08-29T15:48:50.068951+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:48:50.068951+0000","last_undegraded":"2024-08-29T15:48:50.068951+0000","last_fullsized":"2024-08-29T15:48:50.068951+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":86,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.430653+0000","last_change":"2024-08-29T15:46:58.378379+0000","last_active":"2024-08-29T15:48:50.430653+0000","last_peered":"2024-08-29T15:48:50.430653+0000","last_clean":"2024-08-29T15:48:50.430653+0000","last_became_active":"2024-08-29T15:46:58.368284+0000","last_became_peered":"2024-08-29T15:46:58.368284+0000","last_unstale":"2024-08-29T15:48:50.430653+0000","last_undegraded":"2024-08-29T15:48:50.430653+0000","last_fullsized":"2024-08-29T15:48:50.430653+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:57:45.539791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":873463832,"num_objects":13329,"num_object_clones":0,"num_object_copies":39987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13329,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":238927,"num_write_kb":8072129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17538,"num_bytes_recovered":1149304856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2905419776,"data_stored":2905407560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105605,"ondisk_log_size":105605,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":7780,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234651,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1171856,"kb_used_data":274440,"kb_used_omap":1,"kb_used_meta":897406,"kb_avail":92573296,"statfs":{"total":95995035648,"available":94795055104,"internally_reserved":0,"allocated":281026560,"data_stored":277676581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":918944203},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496282,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145160,"kb_used_data":460224,"kb_used_omap":1,"kb_used_meta":684926,"kb_avail":92599992,"statfs":{"total":95995035648,"available":94822391808,"internally_reserved":0,"allocated":471269376,"data_stored":467926739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":701364679},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426870,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":878016,"kb_used_data":241272,"kb_used_omap":8,"kb_used_meta":636727,"kb_avail":92867136,"statfs":{"total":95995035648,"available":95095947264,"internally_reserved":0,"allocated":247062528,"data_stored":243724371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":652009354},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019561,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484108,"kb_used_data":524356,"kb_used_omap":1,"kb_used_meta":959742,"kb_avail":92261044,"statfs":{"total":95995035648,"available":94475309056,"internally_reserved":0,"allocated":536940544,"data_stored":533593868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":982776262},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435746,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263928,"kb_used_data":267440,"kb_used_omap":4,"kb_used_meta":996475,"kb_avail":92481224,"statfs":{"total":95995035648,"available":94700773376,"internally_reserved":0,"allocated":273858560,"data_stored":270527813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":1020390665},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903598,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1094520,"kb_used_data":321264,"kb_used_omap":10,"kb_used_meta":773109,"kb_avail":92650632,"statfs":{"total":95995035648,"available":94874247168,"internally_reserved":0,"allocated":328974336,"data_stored":325643479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":791664144},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762799,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1310836,"kb_used_data":314284,"kb_used_omap":7,"kb_used_meta":996536,"kb_avail":92434316,"statfs":{"total":95995035648,"available":94652739584,"internally_reserved":0,"allocated":321826816,"data_stored":318498949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":1020453218},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926322,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1491640,"kb_used_data":593648,"kb_used_omap":7,"kb_used_meta":897976,"kb_avail":92253512,"statfs":{"total":95995035648,"available":94467596288,"internally_reserved":0,"allocated":607895552,"data_stored":604563902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":919527567},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":580521984,"data_stored":580517912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":302645248,"data_stored":302645248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":320733184,"data_stored":320733184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":254869504,"data_stored":254869504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":499978240,"data_stored":499974168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":228982784,"data_stored":228982784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":460394496,"data_stored":460390424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":257294336,"data_stored":257294336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:49:00.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:49:00.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:00.689 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:00.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:00.791 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:00.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:00.823 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:00.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:00.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:00.952 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:00.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:00.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:01.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:01.091 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:01.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:01.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:01.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:01.236 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:01.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:01.338 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:01.375 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:01.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:01.480 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:01.508 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:01.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:01.610 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:01.645 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:01.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:01.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:01.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:01.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:01.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:01.910 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:01.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:01.945 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:01.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:02.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:02.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:02.082 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:02.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:02.182 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:02.214 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:02.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:02.315 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:02.353 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:02.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:02.454 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:02.489 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:02.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:02.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:02.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:02 smithi171 ceph-mon[90031]: pgmap v1869: 119 pgs: 119 active+clean; 676 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:49:02.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:02 smithi027 ceph-mon[98970]: pgmap v1869: 119 pgs: 119 active+clean; 676 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:49:02.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:02 smithi027 ceph-mon[93921]: pgmap v1869: 119 pgs: 119 active+clean; 676 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 2.8 MiB/s, 44 objects/s recovering 2024-08-29T15:49:02.592 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:02.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:02.626 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:02.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:02.728 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:02.765 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:02.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:02.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:02.901 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:02.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:02.973 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:02.976 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:03.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:03.073 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:03.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:03.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:03.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:03.201 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:03.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:03.303 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:03.342 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:03.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:03.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:03.486 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:03.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:03.588 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:03.620 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:03.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:03.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:03.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:03.745 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:03.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:03.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:03.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:03.888 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:03.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:03.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:03.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:04.027 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:04.129 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:04.162 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:04.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:04.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:04.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.290 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:04.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:04.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:04.429 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:04.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:04 smithi171 ceph-mon[90031]: pgmap v1870: 119 pgs: 119 active+clean; 573 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:49:04.532 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:04.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:04.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:04.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:04 smithi027 ceph-mon[93921]: pgmap v1870: 119 pgs: 119 active+clean; 573 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:49:04.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:04 smithi027 ceph-mon[98970]: pgmap v1870: 119 pgs: 119 active+clean; 573 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-08-29T15:49:04.665 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:04.700 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:04.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:04.802 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.835 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:04.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:04.936 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:04.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:04.967 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:04.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:05.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:05.103 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:05.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:05.164 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:05.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:05.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:05.236 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:05.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:05.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:05.379 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:05.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:05.481 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:05.515 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:05.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:05.616 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:05.655 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:05.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:05.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:05.757 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:05.790 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:05.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:05.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:05.929 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:05.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:05.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:06.032 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:06.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:06.063 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:06.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:06.164 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:06.200 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:06.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:06.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:06.347 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:06.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:06.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:06.393 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:06 smithi027 ceph-mon[93921]: pgmap v1871: 119 pgs: 119 active+clean; 493 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-08-29T15:49:06.393 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:06 smithi027 ceph-mon[98970]: pgmap v1871: 119 pgs: 119 active+clean; 493 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-08-29T15:49:06.449 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:06.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:06.486 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:06.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:06.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:06 smithi171 ceph-mon[90031]: pgmap v1871: 119 pgs: 119 active+clean; 493 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-08-29T15:49:06.586 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:06.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:06.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:06.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:06.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:06.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:06.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:06.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:06.895 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:06.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:06.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:06.997 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:07.034 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:07.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:07.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:07.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:07.181 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:07.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:07.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:07.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:07.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:07.313 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:07.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:07.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:07.457 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:07.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:07.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:07.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:07.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:07.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:07.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:07.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:07.745 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:07.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:07.847 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:07.873 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:07.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:07.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:07.975 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:08.010 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:08.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:08.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:08.146 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:08.248 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:08.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.281 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:08.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:08.382 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:08.409 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:08.511 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:08.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:08 smithi171 ceph-mon[90031]: pgmap v1872: 119 pgs: 119 active+clean; 285 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-29T15:49:08.538 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:08.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:08.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:08 smithi027 ceph-mon[98970]: pgmap v1872: 119 pgs: 119 active+clean; 285 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-29T15:49:08.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:08 smithi027 ceph-mon[93921]: pgmap v1872: 119 pgs: 119 active+clean; 285 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s 2024-08-29T15:49:08.640 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:08.666 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:08.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:08.809 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:08.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:08.955 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:08.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:09.049 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:49:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:49:09] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-29T15:49:09.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:09.129 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:09.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:09.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:09.230 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:09.273 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:09.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:09.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:09.374 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:09.404 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:09.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:09.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:09.542 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:09.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:09.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:09.686 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:09.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:09.787 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:09.817 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:09.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:09.919 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:09.928 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Removed 126127 objects 2024-08-29T15:49:09.928 INFO:tasks.radosbench.radosbench.0.smithi027.stdout:Clean up completed and total clean up time :141.239 2024-08-29T15:49:09.941 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:09.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:10.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:10.066 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:10.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:10.127 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:49:10.128 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:49:10.128 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:49:10.128 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:49:10.128 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:49:10.128 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:49:10.128 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:49:10.129 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:49:10.130 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:49:10.167 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:10.189 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:10.255 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is set 2024-08-29T15:49:10.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:10.317 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:10.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:10.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:10.436 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-08-29T15:49:10.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-08-29T15:49:10.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.442 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:10.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:10 smithi171 ceph-mon[90031]: pgmap v1873: 119 pgs: 119 active+clean; 189 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-29T15:49:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1260012570' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:49:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:10 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:49:10.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:10 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3935136148' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:10.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:10.564 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[98970]: pgmap v1873: 119 pgs: 119 active+clean; 189 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-29T15:49:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1260012570' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:49:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:49:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3935136148' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[93921]: pgmap v1873: 119 pgs: 119 active+clean; 189 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-29T15:49:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1260012570' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:49:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-29T15:49:10.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:10 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3935136148' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:10.666 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:10.688 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:10.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:10.790 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:10.812 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:10.914 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:10.935 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:10.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:10.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:11.037 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:11.062 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:11.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:11.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:11.186 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:11.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:11.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:11.311 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:11.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:11.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:11.435 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:11.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:11 smithi171 ceph-mon[90031]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:11 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:49:11.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:11 smithi171 ceph-mon[90031]: osdmap e531: 8 total, 8 up, 8 in 2024-08-29T15:49:11.537 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:11.546 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:11 smithi027 ceph-mon[98970]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:11.546 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:11 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:49:11.546 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:11 smithi027 ceph-mon[98970]: osdmap e531: 8 total, 8 up, 8 in 2024-08-29T15:49:11.546 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:11 smithi027 ceph-mon[93921]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:11.546 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:11 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-29T15:49:11.546 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:11 smithi027 ceph-mon[93921]: osdmap e531: 8 total, 8 up, 8 in 2024-08-29T15:49:11.563 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:11.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:11.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:11.687 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:11.789 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:11.814 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:11.916 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:11.942 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:11.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:11.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:12.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:12.070 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:12.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:12.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:12.196 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:12.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:12.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:12.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:12.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:12.422 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:12.449 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:12.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:12.478 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:49:12.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:12 smithi027 ceph-mon[98970]: pgmap v1875: 119 pgs: 119 active+clean; 70 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 968 op/s 2024-08-29T15:49:12.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:12 smithi027 ceph-mon[93921]: pgmap v1875: 119 pgs: 119 active+clean; 70 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 968 op/s 2024-08-29T15:49:12.483 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:49:12.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:12 smithi171 ceph-mon[90031]: pgmap v1875: 119 pgs: 119 active+clean; 70 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 968 op/s 2024-08-29T15:49:12.551 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:12.577 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:12.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:12.701 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:12.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:12.803 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:12.829 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:12.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:12.931 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:12.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-29T15:49:12.953 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:12.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:12.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:13.055 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:13.081 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:13.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:13.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:13.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:13.242 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:13.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:13.343 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:13.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:13.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:13.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:13.471 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:13.496 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:13.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:13.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:13 smithi027 ceph-mon[98970]: from='client.37341 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:49:13.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:13 smithi027 ceph-mon[93921]: from='client.37341 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:49:13.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:13.624 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:13.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:13.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:13.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:13.749 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:13.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:13.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:13 smithi171 ceph-mon[90031]: from='client.37341 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:49:13.851 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:13.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:13.877 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:13.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:13.941 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:13.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:14.001 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:14.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:14.103 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:14.129 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:14.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:14.253 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:14.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:14.355 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:14.381 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:14.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:14.483 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:14.501 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1875,"stamp":"2024-08-29T15:49:11.184515+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":73668581,"num_objects":1079,"num_object_clones":0,"num_object_copies":3237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1079,"num_whiteouts":0,"num_read":5988,"num_read_kb":13001,"num_write":252113,"num_write_kb":8084754,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17753,"num_bytes_recovered":1166417551,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111435,"ondisk_log_size":111435,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7268300,"kb_used_data":430344,"kb_used_omap":43,"kb_used_meta":6837716,"kb_avail":742692916,"statfs":{"total":767960285184,"available":760517545984,"internally_reserved":0,"allocated":440672256,"data_stored":413973904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":7001821676},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,3,7,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-635174936,"num_objects":-9693,"num_object_clones":0,"num_object_copies":-29079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9693,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004782"},"pg_stats":[{"pgid":"1.11","version":"90'149","reported_seq":951,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252803+0000","last_change":"2024-08-29T15:47:01.417257+0000","last_active":"2024-08-29T15:49:10.252803+0000","last_peered":"2024-08-29T15:49:10.252803+0000","last_clean":"2024-08-29T15:49:10.252803+0000","last_became_active":"2024-08-29T15:47:01.411204+0000","last_became_peered":"2024-08-29T15:47:01.411204+0000","last_unstale":"2024-08-29T15:49:10.252803+0000","last_undegraded":"2024-08-29T15:49:10.252803+0000","last_fullsized":"2024-08-29T15:49:10.252803+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:36.699540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:36.699540+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:34:08.236678+0000","scrub_duration":0.0014941869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":983,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385817+0000","last_change":"2024-08-29T15:47:12.481794+0000","last_active":"2024-08-29T15:48:50.385817+0000","last_peered":"2024-08-29T15:48:50.385817+0000","last_clean":"2024-08-29T15:48:50.385817+0000","last_became_active":"2024-08-29T15:47:12.480977+0000","last_became_peered":"2024-08-29T15:47:12.480977+0000","last_unstale":"2024-08-29T15:48:50.385817+0000","last_undegraded":"2024-08-29T15:48:50.385817+0000","last_fullsized":"2024-08-29T15:48:50.385817+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:29.573851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":274,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429991+0000","last_change":"2024-08-29T15:46:36.697686+0000","last_active":"2024-08-29T15:48:50.429991+0000","last_peered":"2024-08-29T15:48:50.429991+0000","last_clean":"2024-08-29T15:48:50.429991+0000","last_became_active":"2024-08-29T15:46:36.697367+0000","last_became_peered":"2024-08-29T15:46:36.697367+0000","last_unstale":"2024-08-29T15:48:50.429991+0000","last_undegraded":"2024-08-29T15:48:50.429991+0000","last_fullsized":"2024-08-29T15:48:50.429991+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:19:40.001922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":1024,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383103+0000","last_change":"2024-08-29T15:44:59.245746+0000","last_active":"2024-08-29T15:48:49.383103+0000","last_peered":"2024-08-29T15:48:49.383103+0000","last_clean":"2024-08-29T15:48:49.383103+0000","last_became_active":"2024-08-29T15:44:59.245400+0000","last_became_peered":"2024-08-29T15:44:59.245400+0000","last_unstale":"2024-08-29T15:48:49.383103+0000","last_undegraded":"2024-08-29T15:48:49.383103+0000","last_fullsized":"2024-08-29T15:48:49.383103+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.968809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.968809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:48:37.657803+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383451+0000","last_change":"2024-08-29T15:39:53.258205+0000","last_active":"2024-08-29T15:48:49.383451+0000","last_peered":"2024-08-29T15:48:49.383451+0000","last_clean":"2024-08-29T15:48:49.383451+0000","last_became_active":"2024-08-29T15:39:53.257784+0000","last_became_peered":"2024-08-29T15:39:53.257784+0000","last_unstale":"2024-08-29T15:48:49.383451+0000","last_undegraded":"2024-08-29T15:48:49.383451+0000","last_fullsized":"2024-08-29T15:48:49.383451+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:30:42.270982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":82,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253040+0000","last_change":"2024-08-29T15:46:58.374246+0000","last_active":"2024-08-29T15:49:10.253040+0000","last_peered":"2024-08-29T15:49:10.253040+0000","last_clean":"2024-08-29T15:49:10.253040+0000","last_became_active":"2024-08-29T15:46:58.373708+0000","last_became_peered":"2024-08-29T15:46:58.373708+0000","last_unstale":"2024-08-29T15:49:10.253040+0000","last_undegraded":"2024-08-29T15:49:10.253040+0000","last_fullsized":"2024-08-29T15:49:10.253040+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:27.833391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":1095,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385691+0000","last_change":"2024-08-29T15:47:46.309647+0000","last_active":"2024-08-29T15:48:50.385691+0000","last_peered":"2024-08-29T15:48:50.385691+0000","last_clean":"2024-08-29T15:48:50.385691+0000","last_became_active":"2024-08-29T15:47:46.309290+0000","last_became_peered":"2024-08-29T15:47:46.309290+0000","last_unstale":"2024-08-29T15:48:50.385691+0000","last_undegraded":"2024-08-29T15:48:50.385691+0000","last_fullsized":"2024-08-29T15:48:50.385691+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.615446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.615446+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:58.511333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253371+0000","last_change":"2024-08-29T15:48:12.267269+0000","last_active":"2024-08-29T15:49:10.253371+0000","last_peered":"2024-08-29T15:49:10.253371+0000","last_clean":"2024-08-29T15:49:10.253371+0000","last_became_active":"2024-08-29T15:48:12.266893+0000","last_became_peered":"2024-08-29T15:48:12.266893+0000","last_unstale":"2024-08-29T15:49:10.253371+0000","last_undegraded":"2024-08-29T15:49:10.253371+0000","last_fullsized":"2024-08-29T15:49:10.253371+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:07:31.630164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":318,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389357+0000","last_change":"2024-08-29T15:19:46.807260+0000","last_active":"2024-08-29T15:48:49.389357+0000","last_peered":"2024-08-29T15:48:49.389357+0000","last_clean":"2024-08-29T15:48:49.389357+0000","last_became_active":"2024-08-29T15:19:46.805978+0000","last_became_peered":"2024-08-29T15:19:46.805978+0000","last_unstale":"2024-08-29T15:48:49.389357+0000","last_undegraded":"2024-08-29T15:48:49.389357+0000","last_fullsized":"2024-08-29T15:48:49.389357+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:53.293782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":1044,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383067+0000","last_change":"2024-08-29T15:44:59.241957+0000","last_active":"2024-08-29T15:48:49.383067+0000","last_peered":"2024-08-29T15:48:49.383067+0000","last_clean":"2024-08-29T15:48:49.383067+0000","last_became_active":"2024-08-29T15:44:59.241187+0000","last_became_peered":"2024-08-29T15:44:59.241187+0000","last_unstale":"2024-08-29T15:48:49.383067+0000","last_undegraded":"2024-08-29T15:48:49.383067+0000","last_fullsized":"2024-08-29T15:48:49.383067+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.831081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.831081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:04:04.531194+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387332+0000","last_change":"2024-08-29T15:48:12.268685+0000","last_active":"2024-08-29T15:48:50.387332+0000","last_peered":"2024-08-29T15:48:50.387332+0000","last_clean":"2024-08-29T15:48:50.387332+0000","last_became_active":"2024-08-29T15:48:12.268311+0000","last_became_peered":"2024-08-29T15:48:12.268311+0000","last_unstale":"2024-08-29T15:48:50.387332+0000","last_undegraded":"2024-08-29T15:48:50.387332+0000","last_fullsized":"2024-08-29T15:48:50.387332+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:30:18.347971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":109,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252562+0000","last_change":"2024-08-29T15:48:30.358799+0000","last_active":"2024-08-29T15:49:10.252562+0000","last_peered":"2024-08-29T15:49:10.252562+0000","last_clean":"2024-08-29T15:49:10.252562+0000","last_became_active":"2024-08-29T15:48:30.353158+0000","last_became_peered":"2024-08-29T15:48:30.353158+0000","last_unstale":"2024-08-29T15:49:10.252562+0000","last_undegraded":"2024-08-29T15:49:10.252562+0000","last_fullsized":"2024-08-29T15:49:10.252562+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:17:11.641951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.6","version":"530'15695","reported_seq":15755,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:08.675542+0000","last_change":"2024-08-29T15:45:06.877844+0000","last_active":"2024-08-29T15:49:08.675542+0000","last_peered":"2024-08-29T15:49:08.675542+0000","last_clean":"2024-08-29T15:49:08.675542+0000","last_became_active":"2024-08-29T15:45:06.877523+0000","last_became_peered":"2024-08-29T15:45:06.877523+0000","last_unstale":"2024-08-29T15:49:08.675542+0000","last_undegraded":"2024-08-29T15:49:08.675542+0000","last_fullsized":"2024-08-29T15:49:08.675542+0000","mapping_epoch":480,"log_start":"493'8127","ondisk_log_start":"493'8127","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:37:30.523325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15695,"num_write_kb":504256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":980,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389827+0000","last_change":"2024-08-29T15:48:15.289183+0000","last_active":"2024-08-29T15:48:49.389827+0000","last_peered":"2024-08-29T15:48:49.389827+0000","last_clean":"2024-08-29T15:48:49.389827+0000","last_became_active":"2024-08-29T15:48:15.288693+0000","last_became_peered":"2024-08-29T15:48:15.288693+0000","last_unstale":"2024-08-29T15:48:49.389827+0000","last_undegraded":"2024-08-29T15:48:49.389827+0000","last_fullsized":"2024-08-29T15:48:49.389827+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.799642+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.799642+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:17:07.837651+0000","scrub_duration":0.0012023820000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":375,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.436443+0000","last_change":"2024-08-29T15:46:58.374075+0000","last_active":"2024-08-29T15:48:50.436443+0000","last_peered":"2024-08-29T15:48:50.436443+0000","last_clean":"2024-08-29T15:48:50.436443+0000","last_became_active":"2024-08-29T15:46:58.369089+0000","last_became_peered":"2024-08-29T15:46:58.369089+0000","last_unstale":"2024-08-29T15:48:50.436443+0000","last_undegraded":"2024-08-29T15:48:50.436443+0000","last_fullsized":"2024-08-29T15:48:50.436443+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:40:45.123913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":120,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383284+0000","last_change":"2024-08-29T15:39:53.258473+0000","last_active":"2024-08-29T15:48:49.383284+0000","last_peered":"2024-08-29T15:48:49.383284+0000","last_clean":"2024-08-29T15:48:49.383284+0000","last_became_active":"2024-08-29T15:39:53.258055+0000","last_became_peered":"2024-08-29T15:39:53.258055+0000","last_unstale":"2024-08-29T15:48:49.383284+0000","last_undegraded":"2024-08-29T15:48:49.383284+0000","last_fullsized":"2024-08-29T15:48:49.383284+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:38:54.441868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"530'15592","reported_seq":18427,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:07.233566+0000","last_change":"2024-08-29T15:48:50.421937+0000","last_active":"2024-08-29T15:49:07.233566+0000","last_peered":"2024-08-29T15:49:07.233566+0000","last_clean":"2024-08-29T15:49:07.233566+0000","last_became_active":"2024-08-29T15:48:50.388985+0000","last_became_peered":"2024-08-29T15:48:50.388985+0000","last_unstale":"2024-08-29T15:49:07.233566+0000","last_undegraded":"2024-08-29T15:49:07.233566+0000","last_fullsized":"2024-08-29T15:49:07.233566+0000","mapping_epoch":529,"log_start":"500'9146","ondisk_log_start":"500'9146","created":480,"last_epoch_clean":530,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6446,"log_dups_size":0,"ondisk_log_size":6446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:31:55.673298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10616832,"num_objects":162,"num_object_clones":0,"num_object_copies":486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15592,"num_write_kb":504128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1351,"num_bytes_recovered":88539136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'149","reported_seq":1139,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385666+0000","last_change":"2024-08-29T15:44:59.240657+0000","last_active":"2024-08-29T15:48:50.385666+0000","last_peered":"2024-08-29T15:48:50.385666+0000","last_clean":"2024-08-29T15:48:50.385666+0000","last_became_active":"2024-08-29T15:44:59.240261+0000","last_became_peered":"2024-08-29T15:44:59.240261+0000","last_unstale":"2024-08-29T15:48:50.385666+0000","last_undegraded":"2024-08-29T15:48:50.385666+0000","last_fullsized":"2024-08-29T15:48:50.385666+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.834998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.834998+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:47:15.459581+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":323,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.436717+0000","last_change":"2024-08-29T15:28:35.531800+0000","last_active":"2024-08-29T15:48:50.436717+0000","last_peered":"2024-08-29T15:48:50.436717+0000","last_clean":"2024-08-29T15:48:50.436717+0000","last_became_active":"2024-08-29T15:19:46.809388+0000","last_became_peered":"2024-08-29T15:19:46.809388+0000","last_unstale":"2024-08-29T15:48:50.436717+0000","last_undegraded":"2024-08-29T15:48:50.436717+0000","last_fullsized":"2024-08-29T15:48:50.436717+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:41:54.216781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":59,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386227+0000","last_change":"2024-08-29T15:47:43.263617+0000","last_active":"2024-08-29T15:48:50.386227+0000","last_peered":"2024-08-29T15:48:50.386227+0000","last_clean":"2024-08-29T15:48:50.386227+0000","last_became_active":"2024-08-29T15:47:43.263296+0000","last_became_peered":"2024-08-29T15:47:43.263296+0000","last_unstale":"2024-08-29T15:48:50.386227+0000","last_undegraded":"2024-08-29T15:48:50.386227+0000","last_fullsized":"2024-08-29T15:48:50.386227+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:46:43.176620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"530'15520","reported_seq":25982,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252929+0000","last_change":"2024-08-29T15:48:00.824043+0000","last_active":"2024-08-29T15:49:10.252929+0000","last_peered":"2024-08-29T15:49:10.252929+0000","last_clean":"2024-08-29T15:49:10.252929+0000","last_became_active":"2024-08-29T15:48:00.770733+0000","last_became_peered":"2024-08-29T15:48:00.770733+0000","last_unstale":"2024-08-29T15:49:10.252929+0000","last_undegraded":"2024-08-29T15:49:10.252929+0000","last_fullsized":"2024-08-29T15:49:10.252929+0000","mapping_epoch":512,"log_start":"502'9192","ondisk_log_start":"502'9192","created":480,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:03:45.715507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15520,"num_write_kb":496640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5171,"num_bytes_recovered":338886656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":970,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.384053+0000","last_change":"2024-08-29T15:44:59.247796+0000","last_active":"2024-08-29T15:48:49.384053+0000","last_peered":"2024-08-29T15:48:49.384053+0000","last_clean":"2024-08-29T15:48:49.384053+0000","last_became_active":"2024-08-29T15:44:59.246494+0000","last_became_peered":"2024-08-29T15:44:59.246494+0000","last_unstale":"2024-08-29T15:48:49.384053+0000","last_undegraded":"2024-08-29T15:48:49.384053+0000","last_fullsized":"2024-08-29T15:48:49.384053+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:21.844245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:21.844245+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:26.154513+0000","scrub_duration":0.0016201309999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":124,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385486+0000","last_change":"2024-08-29T15:39:53.262167+0000","last_active":"2024-08-29T15:48:50.385486+0000","last_peered":"2024-08-29T15:48:50.385486+0000","last_clean":"2024-08-29T15:48:50.385486+0000","last_became_active":"2024-08-29T15:39:53.258406+0000","last_became_peered":"2024-08-29T15:39:53.258406+0000","last_unstale":"2024-08-29T15:48:50.385486+0000","last_undegraded":"2024-08-29T15:48:50.385486+0000","last_fullsized":"2024-08-29T15:48:50.385486+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:05:19.731559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":556,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387476+0000","last_change":"2024-08-29T15:32:26.142339+0000","last_active":"2024-08-29T15:48:50.387476+0000","last_peered":"2024-08-29T15:48:50.387476+0000","last_clean":"2024-08-29T15:48:50.387476+0000","last_became_active":"2024-08-29T15:32:26.142029+0000","last_became_peered":"2024-08-29T15:32:26.142029+0000","last_unstale":"2024-08-29T15:48:50.387476+0000","last_undegraded":"2024-08-29T15:48:50.387476+0000","last_fullsized":"2024-08-29T15:48:50.387476+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:03.147183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"530'15748","reported_seq":15862,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253226+0000","last_change":"2024-08-29T15:45:07.147672+0000","last_active":"2024-08-29T15:49:10.253226+0000","last_peered":"2024-08-29T15:49:10.253226+0000","last_clean":"2024-08-29T15:49:10.253226+0000","last_became_active":"2024-08-29T15:45:07.147363+0000","last_became_peered":"2024-08-29T15:45:07.147363+0000","last_unstale":"2024-08-29T15:49:10.253226+0000","last_undegraded":"2024-08-29T15:49:10.253226+0000","last_fullsized":"2024-08-29T15:49:10.253226+0000","mapping_epoch":480,"log_start":"502'9484","ondisk_log_start":"502'9484","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:03.126615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15748,"num_write_kb":503936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":831,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.390018+0000","last_change":"2024-08-29T15:45:25.871113+0000","last_active":"2024-08-29T15:48:49.390018+0000","last_peered":"2024-08-29T15:48:49.390018+0000","last_clean":"2024-08-29T15:48:49.390018+0000","last_became_active":"2024-08-29T15:44:59.246190+0000","last_became_peered":"2024-08-29T15:44:59.246190+0000","last_unstale":"2024-08-29T15:48:49.390018+0000","last_undegraded":"2024-08-29T15:48:49.390018+0000","last_fullsized":"2024-08-29T15:48:49.390018+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.870329+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.870329+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:42.080608+0000","scrub_duration":0.0042968349999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":427,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385624+0000","last_change":"2024-08-29T15:17:40.725103+0000","last_active":"2024-08-29T15:48:50.385624+0000","last_peered":"2024-08-29T15:48:50.385624+0000","last_clean":"2024-08-29T15:48:50.385624+0000","last_became_active":"2024-08-29T15:17:40.720793+0000","last_became_peered":"2024-08-29T15:17:40.720793+0000","last_unstale":"2024-08-29T15:48:50.385624+0000","last_undegraded":"2024-08-29T15:48:50.385624+0000","last_fullsized":"2024-08-29T15:48:50.385624+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:35.775092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":937,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253108+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:49:10.253108+0000","last_peered":"2024-08-29T15:49:10.253108+0000","last_clean":"2024-08-29T15:49:10.253108+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:49:10.253108+0000","last_undegraded":"2024-08-29T15:49:10.253108+0000","last_fullsized":"2024-08-29T15:49:10.253108+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"530'15565","reported_seq":15677,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:06.834052+0000","last_change":"2024-08-29T15:45:06.877939+0000","last_active":"2024-08-29T15:49:06.834052+0000","last_peered":"2024-08-29T15:49:06.834052+0000","last_clean":"2024-08-29T15:49:06.834052+0000","last_became_active":"2024-08-29T15:45:06.877593+0000","last_became_peered":"2024-08-29T15:45:06.877593+0000","last_unstale":"2024-08-29T15:49:06.834052+0000","last_undegraded":"2024-08-29T15:49:06.834052+0000","last_fullsized":"2024-08-29T15:49:06.834052+0000","mapping_epoch":480,"log_start":"500'8824","ondisk_log_start":"500'8824","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6741,"log_dups_size":0,"ondisk_log_size":6741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:06:32.260774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11206656,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15565,"num_write_kb":503552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":1078,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386126+0000","last_change":"2024-08-29T15:44:30.836092+0000","last_active":"2024-08-29T15:48:50.386126+0000","last_peered":"2024-08-29T15:48:50.386126+0000","last_clean":"2024-08-29T15:48:50.386126+0000","last_became_active":"2024-08-29T15:44:22.624657+0000","last_became_peered":"2024-08-29T15:44:22.624657+0000","last_unstale":"2024-08-29T15:48:50.386126+0000","last_undegraded":"2024-08-29T15:48:50.386126+0000","last_fullsized":"2024-08-29T15:48:50.386126+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:30.836022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:30.836022+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:36:10.537009+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":973,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386606+0000","last_change":"2024-08-29T15:46:58.377694+0000","last_active":"2024-08-29T15:48:50.386606+0000","last_peered":"2024-08-29T15:48:50.386606+0000","last_clean":"2024-08-29T15:48:50.386606+0000","last_became_active":"2024-08-29T15:46:58.372090+0000","last_became_peered":"2024-08-29T15:46:58.372090+0000","last_unstale":"2024-08-29T15:48:50.386606+0000","last_undegraded":"2024-08-29T15:48:50.386606+0000","last_fullsized":"2024-08-29T15:48:50.386606+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:19:50.948014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":310,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386606+0000","last_change":"2024-08-29T15:46:36.598273+0000","last_active":"2024-08-29T15:48:50.386606+0000","last_peered":"2024-08-29T15:48:50.386606+0000","last_clean":"2024-08-29T15:48:50.386606+0000","last_became_active":"2024-08-29T15:46:36.591062+0000","last_became_peered":"2024-08-29T15:46:36.591062+0000","last_unstale":"2024-08-29T15:48:50.386606+0000","last_undegraded":"2024-08-29T15:48:50.386606+0000","last_fullsized":"2024-08-29T15:48:50.386606+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:32:51.573470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"530'15631","reported_seq":23677,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:08.685813+0000","last_change":"2024-08-29T15:48:12.294106+0000","last_active":"2024-08-29T15:49:08.685813+0000","last_peered":"2024-08-29T15:49:08.685813+0000","last_clean":"2024-08-29T15:49:08.685813+0000","last_became_active":"2024-08-29T15:48:12.262915+0000","last_became_peered":"2024-08-29T15:48:12.262915+0000","last_unstale":"2024-08-29T15:49:08.685813+0000","last_undegraded":"2024-08-29T15:49:08.685813+0000","last_fullsized":"2024-08-29T15:49:08.685813+0000","mapping_epoch":515,"log_start":"493'8084","ondisk_log_start":"493'8084","created":480,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:54:51.872371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3604480,"num_objects":55,"num_object_clones":0,"num_object_copies":165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15631,"num_write_kb":501952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3980,"num_bytes_recovered":260833280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":740,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.307906+0000","last_change":"2024-08-29T15:48:50.307906+0000","last_active":"2024-08-29T15:48:50.307906+0000","last_peered":"2024-08-29T15:48:50.307906+0000","last_clean":"2024-08-29T15:48:50.307906+0000","last_became_active":"2024-08-29T15:48:33.284378+0000","last_became_peered":"2024-08-29T15:48:33.284378+0000","last_unstale":"2024-08-29T15:48:50.307906+0000","last_undegraded":"2024-08-29T15:48:50.307906+0000","last_fullsized":"2024-08-29T15:48:50.307906+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:48:50.307797+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:48:50.307797+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:46:36.775569+0000","scrub_duration":0.0020697210000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":323,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389385+0000","last_change":"2024-08-29T15:47:12.469624+0000","last_active":"2024-08-29T15:48:49.389385+0000","last_peered":"2024-08-29T15:48:49.389385+0000","last_clean":"2024-08-29T15:48:49.389385+0000","last_became_active":"2024-08-29T15:47:12.469286+0000","last_became_peered":"2024-08-29T15:47:12.469286+0000","last_unstale":"2024-08-29T15:48:49.389385+0000","last_undegraded":"2024-08-29T15:48:49.389385+0000","last_fullsized":"2024-08-29T15:48:49.389385+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:09:13.811380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":316,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386335+0000","last_change":"2024-08-29T15:46:36.703991+0000","last_active":"2024-08-29T15:48:50.386335+0000","last_peered":"2024-08-29T15:48:50.386335+0000","last_clean":"2024-08-29T15:48:50.386335+0000","last_became_active":"2024-08-29T15:46:36.703653+0000","last_became_peered":"2024-08-29T15:46:36.703653+0000","last_unstale":"2024-08-29T15:48:50.386335+0000","last_undegraded":"2024-08-29T15:48:50.386335+0000","last_fullsized":"2024-08-29T15:48:50.386335+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:10:01.192871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"530'15684","reported_seq":15796,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:06.850195+0000","last_change":"2024-08-29T15:45:06.875151+0000","last_active":"2024-08-29T15:49:06.850195+0000","last_peered":"2024-08-29T15:49:06.850195+0000","last_clean":"2024-08-29T15:49:06.850195+0000","last_became_active":"2024-08-29T15:45:06.874862+0000","last_became_peered":"2024-08-29T15:45:06.874862+0000","last_unstale":"2024-08-29T15:49:06.850195+0000","last_undegraded":"2024-08-29T15:49:06.850195+0000","last_fullsized":"2024-08-29T15:49:06.850195+0000","mapping_epoch":480,"log_start":"500'8952","ondisk_log_start":"500'8952","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6732,"log_dups_size":0,"ondisk_log_size":6732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:54:54.225355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12058624,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15684,"num_write_kb":507776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":1100,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385539+0000","last_change":"2024-08-29T15:44:59.243847+0000","last_active":"2024-08-29T15:48:50.385539+0000","last_peered":"2024-08-29T15:48:50.385539+0000","last_clean":"2024-08-29T15:48:50.385539+0000","last_became_active":"2024-08-29T15:44:59.243399+0000","last_became_peered":"2024-08-29T15:44:59.243399+0000","last_unstale":"2024-08-29T15:48:50.385539+0000","last_undegraded":"2024-08-29T15:48:50.385539+0000","last_fullsized":"2024-08-29T15:48:50.385539+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.906140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.906140+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:51:09.311132+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":319,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389274+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:48:49.389274+0000","last_peered":"2024-08-29T15:48:49.389274+0000","last_clean":"2024-08-29T15:48:49.389274+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:48:49.389274+0000","last_undegraded":"2024-08-29T15:48:49.389274+0000","last_fullsized":"2024-08-29T15:48:49.389274+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":560,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.390090+0000","last_change":"2024-08-29T15:47:12.467689+0000","last_active":"2024-08-29T15:48:49.390090+0000","last_peered":"2024-08-29T15:48:49.390090+0000","last_clean":"2024-08-29T15:48:49.390090+0000","last_became_active":"2024-08-29T15:47:12.467400+0000","last_became_peered":"2024-08-29T15:47:12.467400+0000","last_unstale":"2024-08-29T15:48:49.390090+0000","last_undegraded":"2024-08-29T15:48:49.390090+0000","last_fullsized":"2024-08-29T15:48:49.390090+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:45:54.623271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"530'16238","reported_seq":16346,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252693+0000","last_change":"2024-08-29T15:46:36.601176+0000","last_active":"2024-08-29T15:49:10.252693+0000","last_peered":"2024-08-29T15:49:10.252693+0000","last_clean":"2024-08-29T15:49:10.252693+0000","last_became_active":"2024-08-29T15:46:36.589279+0000","last_became_peered":"2024-08-29T15:46:36.589279+0000","last_unstale":"2024-08-29T15:49:10.252693+0000","last_undegraded":"2024-08-29T15:49:10.252693+0000","last_fullsized":"2024-08-29T15:49:10.252693+0000","mapping_epoch":492,"log_start":"502'9904","ondisk_log_start":"502'9904","created":480,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6334,"log_dups_size":0,"ondisk_log_size":6334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:18:23.326237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16238,"num_write_kb":519616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":911,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383202+0000","last_change":"2024-08-29T15:44:53.533605+0000","last_active":"2024-08-29T15:48:49.383202+0000","last_peered":"2024-08-29T15:48:49.383202+0000","last_clean":"2024-08-29T15:48:49.383202+0000","last_became_active":"2024-08-29T15:44:22.611492+0000","last_became_peered":"2024-08-29T15:44:22.611492+0000","last_unstale":"2024-08-29T15:48:49.383202+0000","last_undegraded":"2024-08-29T15:48:49.383202+0000","last_fullsized":"2024-08-29T15:48:49.383202+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:53.533488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:53.533488+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:10:06.648003+0000","scrub_duration":0.0059276559999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":59,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385748+0000","last_change":"2024-08-29T15:47:43.258937+0000","last_active":"2024-08-29T15:48:50.385748+0000","last_peered":"2024-08-29T15:48:50.385748+0000","last_clean":"2024-08-29T15:48:50.385748+0000","last_became_active":"2024-08-29T15:47:43.258572+0000","last_became_peered":"2024-08-29T15:47:43.258572+0000","last_unstale":"2024-08-29T15:48:50.385748+0000","last_undegraded":"2024-08-29T15:48:50.385748+0000","last_fullsized":"2024-08-29T15:48:50.385748+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:58.220504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":925,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386192+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:48:50.386192+0000","last_peered":"2024-08-29T15:48:50.386192+0000","last_clean":"2024-08-29T15:48:50.386192+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:48:50.386192+0000","last_undegraded":"2024-08-29T15:48:50.386192+0000","last_fullsized":"2024-08-29T15:48:50.386192+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"530'15427","reported_seq":15539,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:06.852512+0000","last_change":"2024-08-29T15:45:06.878140+0000","last_active":"2024-08-29T15:49:06.852512+0000","last_peered":"2024-08-29T15:49:06.852512+0000","last_clean":"2024-08-29T15:49:06.852512+0000","last_became_active":"2024-08-29T15:45:06.877878+0000","last_became_peered":"2024-08-29T15:45:06.877878+0000","last_unstale":"2024-08-29T15:49:06.852512+0000","last_undegraded":"2024-08-29T15:49:06.852512+0000","last_fullsized":"2024-08-29T15:49:06.852512+0000","mapping_epoch":480,"log_start":"499'8738","ondisk_log_start":"499'8738","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6689,"log_dups_size":0,"ondisk_log_size":6689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:33:28.011162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12124160,"num_objects":185,"num_object_clones":0,"num_object_copies":555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15427,"num_write_kb":499584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":1081,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253167+0000","last_change":"2024-08-29T15:47:01.394949+0000","last_active":"2024-08-29T15:49:10.253167+0000","last_peered":"2024-08-29T15:49:10.253167+0000","last_clean":"2024-08-29T15:49:10.253167+0000","last_became_active":"2024-08-29T15:47:01.394614+0000","last_became_peered":"2024-08-29T15:47:01.394614+0000","last_unstale":"2024-08-29T15:49:10.253167+0000","last_undegraded":"2024-08-29T15:49:10.253167+0000","last_fullsized":"2024-08-29T15:49:10.253167+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.008533+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.008533+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:06:09.577560+0000","scrub_duration":111,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":1012,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385845+0000","last_change":"2024-08-29T15:18:33.981633+0000","last_active":"2024-08-29T15:48:50.385845+0000","last_peered":"2024-08-29T15:48:50.385845+0000","last_clean":"2024-08-29T15:48:50.385845+0000","last_became_active":"2024-08-29T15:18:33.981147+0000","last_became_peered":"2024-08-29T15:18:33.981147+0000","last_unstale":"2024-08-29T15:48:50.385845+0000","last_undegraded":"2024-08-29T15:48:50.385845+0000","last_fullsized":"2024-08-29T15:48:50.385845+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:28.479221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":339,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385481+0000","last_change":"2024-08-29T15:37:26.313495+0000","last_active":"2024-08-29T15:48:50.385481+0000","last_peered":"2024-08-29T15:48:50.385481+0000","last_clean":"2024-08-29T15:48:50.385481+0000","last_became_active":"2024-08-29T15:37:26.313188+0000","last_became_peered":"2024-08-29T15:37:26.313188+0000","last_unstale":"2024-08-29T15:48:50.385481+0000","last_undegraded":"2024-08-29T15:48:50.385481+0000","last_fullsized":"2024-08-29T15:48:50.385481+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:39.580490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":286,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.439298+0000","last_change":"2024-08-29T15:32:26.140787+0000","last_active":"2024-08-29T15:48:50.439298+0000","last_peered":"2024-08-29T15:48:50.439298+0000","last_clean":"2024-08-29T15:48:50.439298+0000","last_became_active":"2024-08-29T15:32:26.140509+0000","last_became_peered":"2024-08-29T15:32:26.140509+0000","last_unstale":"2024-08-29T15:48:50.439298+0000","last_undegraded":"2024-08-29T15:48:50.439298+0000","last_fullsized":"2024-08-29T15:48:50.439298+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:50:15.561409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"530'15644","reported_seq":15758,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252959+0000","last_change":"2024-08-29T15:45:07.146463+0000","last_active":"2024-08-29T15:49:10.252959+0000","last_peered":"2024-08-29T15:49:10.252959+0000","last_clean":"2024-08-29T15:49:10.252959+0000","last_became_active":"2024-08-29T15:45:07.146148+0000","last_became_peered":"2024-08-29T15:45:07.146148+0000","last_unstale":"2024-08-29T15:49:10.252959+0000","last_undegraded":"2024-08-29T15:49:10.252959+0000","last_fullsized":"2024-08-29T15:49:10.252959+0000","mapping_epoch":480,"log_start":"502'9366","ondisk_log_start":"502'9366","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:04:04.649404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15644,"num_write_kb":500608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":1077,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385810+0000","last_change":"2024-08-29T15:43:26.588054+0000","last_active":"2024-08-29T15:48:50.385810+0000","last_peered":"2024-08-29T15:48:50.385810+0000","last_clean":"2024-08-29T15:48:50.385810+0000","last_became_active":"2024-08-29T15:43:26.285267+0000","last_became_peered":"2024-08-29T15:43:26.285267+0000","last_unstale":"2024-08-29T15:48:50.385810+0000","last_undegraded":"2024-08-29T15:48:50.385810+0000","last_fullsized":"2024-08-29T15:48:50.385810+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.587986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.587986+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:15:21.198019+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":1399,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385362+0000","last_change":"2024-08-29T15:18:10.843762+0000","last_active":"2024-08-29T15:48:50.385362+0000","last_peered":"2024-08-29T15:48:50.385362+0000","last_clean":"2024-08-29T15:48:50.385362+0000","last_became_active":"2024-08-29T15:18:10.843207+0000","last_became_peered":"2024-08-29T15:18:10.843207+0000","last_unstale":"2024-08-29T15:48:50.385362+0000","last_undegraded":"2024-08-29T15:48:50.385362+0000","last_fullsized":"2024-08-29T15:48:50.385362+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:23:57.020759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":967,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252699+0000","last_change":"2024-08-29T15:47:42.288211+0000","last_active":"2024-08-29T15:49:10.252699+0000","last_peered":"2024-08-29T15:49:10.252699+0000","last_clean":"2024-08-29T15:49:10.252699+0000","last_became_active":"2024-08-29T15:47:42.287594+0000","last_became_peered":"2024-08-29T15:47:42.287594+0000","last_unstale":"2024-08-29T15:49:10.252699+0000","last_undegraded":"2024-08-29T15:49:10.252699+0000","last_fullsized":"2024-08-29T15:49:10.252699+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:32:18.869863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253317+0000","last_change":"2024-08-29T15:46:58.369628+0000","last_active":"2024-08-29T15:49:10.253317+0000","last_peered":"2024-08-29T15:49:10.253317+0000","last_clean":"2024-08-29T15:49:10.253317+0000","last_became_active":"2024-08-29T15:46:58.369315+0000","last_became_peered":"2024-08-29T15:46:58.369315+0000","last_unstale":"2024-08-29T15:49:10.253317+0000","last_undegraded":"2024-08-29T15:49:10.253317+0000","last_fullsized":"2024-08-29T15:49:10.253317+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:52:05.895354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.9","version":"530'15632","reported_seq":19403,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:08.739813+0000","last_change":"2024-08-29T15:48:46.403491+0000","last_active":"2024-08-29T15:49:08.739813+0000","last_peered":"2024-08-29T15:49:08.739813+0000","last_clean":"2024-08-29T15:49:08.739813+0000","last_became_active":"2024-08-29T15:48:46.352035+0000","last_became_peered":"2024-08-29T15:48:46.352035+0000","last_unstale":"2024-08-29T15:49:08.739813+0000","last_undegraded":"2024-08-29T15:49:08.739813+0000","last_fullsized":"2024-08-29T15:49:08.739813+0000","mapping_epoch":526,"log_start":"493'8100","ondisk_log_start":"493'8100","created":480,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":7532,"log_dups_size":0,"ondisk_log_size":7532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:03.034399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5767168,"num_objects":88,"num_object_clones":0,"num_object_copies":264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15632,"num_write_kb":503040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1845,"num_bytes_recovered":120913920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":1107,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252918+0000","last_change":"2024-08-29T15:48:15.289740+0000","last_active":"2024-08-29T15:49:10.252918+0000","last_peered":"2024-08-29T15:49:10.252918+0000","last_clean":"2024-08-29T15:49:10.252918+0000","last_became_active":"2024-08-29T15:48:15.289419+0000","last_became_peered":"2024-08-29T15:48:15.289419+0000","last_unstale":"2024-08-29T15:49:10.252918+0000","last_undegraded":"2024-08-29T15:49:10.252918+0000","last_fullsized":"2024-08-29T15:49:10.252918+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.841106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.841106+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:04:15.648107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":967,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386317+0000","last_change":"2024-08-29T15:37:29.345555+0000","last_active":"2024-08-29T15:48:50.386317+0000","last_peered":"2024-08-29T15:48:50.386317+0000","last_clean":"2024-08-29T15:48:50.386317+0000","last_became_active":"2024-08-29T15:37:29.344738+0000","last_became_peered":"2024-08-29T15:37:29.344738+0000","last_unstale":"2024-08-29T15:48:50.386317+0000","last_undegraded":"2024-08-29T15:48:50.386317+0000","last_fullsized":"2024-08-29T15:48:50.386317+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:29:14.103195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":272,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.385637+0000","last_change":"2024-08-29T15:32:26.144604+0000","last_active":"2024-08-29T15:48:49.385637+0000","last_peered":"2024-08-29T15:48:49.385637+0000","last_clean":"2024-08-29T15:48:49.385637+0000","last_became_active":"2024-08-29T15:32:26.144316+0000","last_became_peered":"2024-08-29T15:32:26.144316+0000","last_unstale":"2024-08-29T15:48:49.385637+0000","last_undegraded":"2024-08-29T15:48:49.385637+0000","last_fullsized":"2024-08-29T15:48:49.385637+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:50:05.238254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385983+0000","last_change":"2024-08-29T15:47:12.472465+0000","last_active":"2024-08-29T15:48:50.385983+0000","last_peered":"2024-08-29T15:48:50.385983+0000","last_clean":"2024-08-29T15:48:50.385983+0000","last_became_active":"2024-08-29T15:47:12.472167+0000","last_became_peered":"2024-08-29T15:47:12.472167+0000","last_unstale":"2024-08-29T15:48:50.385983+0000","last_undegraded":"2024-08-29T15:48:50.385983+0000","last_fullsized":"2024-08-29T15:48:50.385983+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:44:56.401983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"530'16060","reported_seq":22421,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252732+0000","last_change":"2024-08-29T15:48:31.428964+0000","last_active":"2024-08-29T15:49:10.252732+0000","last_peered":"2024-08-29T15:49:10.252732+0000","last_clean":"2024-08-29T15:49:10.252732+0000","last_became_active":"2024-08-29T15:48:31.362574+0000","last_became_peered":"2024-08-29T15:48:31.362574+0000","last_unstale":"2024-08-29T15:49:10.252732+0000","last_undegraded":"2024-08-29T15:49:10.252732+0000","last_fullsized":"2024-08-29T15:49:10.252732+0000","mapping_epoch":521,"log_start":"502'9795","ondisk_log_start":"502'9795","created":480,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6265,"log_dups_size":0,"ondisk_log_size":6265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:27:20.861141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16060,"num_write_kb":513920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3126,"num_bytes_recovered":204865536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":942,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383110+0000","last_change":"2024-08-29T15:47:01.407034+0000","last_active":"2024-08-29T15:48:49.383110+0000","last_peered":"2024-08-29T15:48:49.383110+0000","last_clean":"2024-08-29T15:48:49.383110+0000","last_became_active":"2024-08-29T15:47:01.406648+0000","last_became_peered":"2024-08-29T15:47:01.406648+0000","last_unstale":"2024-08-29T15:48:49.383110+0000","last_undegraded":"2024-08-29T15:48:49.383110+0000","last_fullsized":"2024-08-29T15:48:49.383110+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_clean_scrub_stamp":"2024-08-29T15:44:26.709569+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:15:11.694724+0000","scrub_duration":0.0031838119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":480,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385614+0000","last_change":"2024-08-29T15:17:40.722322+0000","last_active":"2024-08-29T15:48:50.385614+0000","last_peered":"2024-08-29T15:48:50.385614+0000","last_clean":"2024-08-29T15:48:50.385614+0000","last_became_active":"2024-08-29T15:17:40.718719+0000","last_became_peered":"2024-08-29T15:17:40.718719+0000","last_unstale":"2024-08-29T15:48:50.385614+0000","last_undegraded":"2024-08-29T15:48:50.385614+0000","last_fullsized":"2024-08-29T15:48:50.385614+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:45:48.022790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":320,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.436111+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:48:50.436111+0000","last_peered":"2024-08-29T15:48:50.436111+0000","last_clean":"2024-08-29T15:48:50.436111+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:48:50.436111+0000","last_undegraded":"2024-08-29T15:48:50.436111+0000","last_fullsized":"2024-08-29T15:48:50.436111+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":387,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385388+0000","last_change":"2024-08-29T15:18:33.981238+0000","last_active":"2024-08-29T15:48:50.385388+0000","last_peered":"2024-08-29T15:48:50.385388+0000","last_clean":"2024-08-29T15:48:50.385388+0000","last_became_active":"2024-08-29T15:18:33.980954+0000","last_became_peered":"2024-08-29T15:18:33.980954+0000","last_unstale":"2024-08-29T15:48:50.385388+0000","last_undegraded":"2024-08-29T15:48:50.385388+0000","last_fullsized":"2024-08-29T15:48:50.385388+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:26:04.280753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"530'15530","reported_seq":15638,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253241+0000","last_change":"2024-08-29T15:46:36.596834+0000","last_active":"2024-08-29T15:49:10.253241+0000","last_peered":"2024-08-29T15:49:10.253241+0000","last_clean":"2024-08-29T15:49:10.253241+0000","last_became_active":"2024-08-29T15:46:36.586137+0000","last_became_peered":"2024-08-29T15:46:36.586137+0000","last_unstale":"2024-08-29T15:49:10.253241+0000","last_undegraded":"2024-08-29T15:49:10.253241+0000","last_fullsized":"2024-08-29T15:49:10.253241+0000","mapping_epoch":492,"log_start":"502'9219","ondisk_log_start":"502'9219","created":480,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6311,"log_dups_size":0,"ondisk_log_size":6311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:18:18.073025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15530,"num_write_kb":496960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":994,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387519+0000","last_change":"2024-08-29T15:47:46.311743+0000","last_active":"2024-08-29T15:48:50.387519+0000","last_peered":"2024-08-29T15:48:50.387519+0000","last_clean":"2024-08-29T15:48:50.387519+0000","last_became_active":"2024-08-29T15:47:46.311238+0000","last_became_peered":"2024-08-29T15:47:46.311238+0000","last_unstale":"2024-08-29T15:48:50.387519+0000","last_undegraded":"2024-08-29T15:48:50.387519+0000","last_fullsized":"2024-08-29T15:48:50.387519+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.292191+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.292191+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:04:43.047588+0000","scrub_duration":0.0075757899999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":412,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385557+0000","last_change":"2024-08-29T15:18:33.983010+0000","last_active":"2024-08-29T15:48:50.385557+0000","last_peered":"2024-08-29T15:48:50.385557+0000","last_clean":"2024-08-29T15:48:50.385557+0000","last_became_active":"2024-08-29T15:18:33.982829+0000","last_became_peered":"2024-08-29T15:18:33.982829+0000","last_unstale":"2024-08-29T15:48:50.385557+0000","last_undegraded":"2024-08-29T15:48:50.385557+0000","last_fullsized":"2024-08-29T15:48:50.385557+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T14:55:39.736594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1363,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385982+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:48:50.385982+0000","last_peered":"2024-08-29T15:48:50.385982+0000","last_clean":"2024-08-29T15:48:50.385982+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:48:50.385982+0000","last_undegraded":"2024-08-29T15:48:50.385982+0000","last_fullsized":"2024-08-29T15:48:50.385982+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387343+0000","last_change":"2024-08-29T15:47:12.474328+0000","last_active":"2024-08-29T15:48:50.387343+0000","last_peered":"2024-08-29T15:48:50.387343+0000","last_clean":"2024-08-29T15:48:50.387343+0000","last_became_active":"2024-08-29T15:47:12.473903+0000","last_became_peered":"2024-08-29T15:47:12.473903+0000","last_unstale":"2024-08-29T15:48:50.387343+0000","last_undegraded":"2024-08-29T15:48:50.387343+0000","last_fullsized":"2024-08-29T15:48:50.387343+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:42.229361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"530'15746","reported_seq":20000,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253111+0000","last_change":"2024-08-29T15:48:47.433077+0000","last_active":"2024-08-29T15:49:10.253111+0000","last_peered":"2024-08-29T15:49:10.253111+0000","last_clean":"2024-08-29T15:49:10.253111+0000","last_became_active":"2024-08-29T15:48:47.367348+0000","last_became_peered":"2024-08-29T15:48:47.367348+0000","last_unstale":"2024-08-29T15:49:10.253111+0000","last_undegraded":"2024-08-29T15:49:10.253111+0000","last_fullsized":"2024-08-29T15:49:10.253111+0000","mapping_epoch":527,"log_start":"502'9493","ondisk_log_start":"502'9493","created":480,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:14:04.463759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15746,"num_write_kb":503809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2065,"num_bytes_recovered":135266328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":941,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253100+0000","last_change":"2024-08-29T15:47:01.410197+0000","last_active":"2024-08-29T15:49:10.253100+0000","last_peered":"2024-08-29T15:49:10.253100+0000","last_clean":"2024-08-29T15:49:10.253100+0000","last_became_active":"2024-08-29T15:47:01.406541+0000","last_became_peered":"2024-08-29T15:47:01.406541+0000","last_unstale":"2024-08-29T15:49:10.253100+0000","last_undegraded":"2024-08-29T15:49:10.253100+0000","last_fullsized":"2024-08-29T15:49:10.253100+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:23.864183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:23.864183+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:13:01.243784+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":945,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385948+0000","last_change":"2024-08-29T15:18:34.056616+0000","last_active":"2024-08-29T15:48:50.385948+0000","last_peered":"2024-08-29T15:48:50.385948+0000","last_clean":"2024-08-29T15:48:50.385948+0000","last_became_active":"2024-08-29T15:18:34.056296+0000","last_became_peered":"2024-08-29T15:18:34.056296+0000","last_unstale":"2024-08-29T15:48:50.385948+0000","last_undegraded":"2024-08-29T15:48:50.385948+0000","last_fullsized":"2024-08-29T15:48:50.385948+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:31:40.368345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":1049,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.386238+0000","last_change":"2024-08-29T15:46:58.399147+0000","last_active":"2024-08-29T15:48:49.386238+0000","last_peered":"2024-08-29T15:48:49.386238+0000","last_clean":"2024-08-29T15:48:49.386238+0000","last_became_active":"2024-08-29T15:46:58.371595+0000","last_became_peered":"2024-08-29T15:46:58.371595+0000","last_unstale":"2024-08-29T15:48:49.386238+0000","last_undegraded":"2024-08-29T15:48:49.386238+0000","last_fullsized":"2024-08-29T15:48:49.386238+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:46:46.943596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":939,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253078+0000","last_change":"2024-08-29T15:17:40.723957+0000","last_active":"2024-08-29T15:49:10.253078+0000","last_peered":"2024-08-29T15:49:10.253078+0000","last_clean":"2024-08-29T15:49:10.253078+0000","last_became_active":"2024-08-29T15:17:40.723677+0000","last_became_peered":"2024-08-29T15:17:40.723677+0000","last_unstale":"2024-08-29T15:49:10.253078+0000","last_undegraded":"2024-08-29T15:49:10.253078+0000","last_fullsized":"2024-08-29T15:49:10.253078+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:16:50.909960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"530'15828","reported_seq":15888,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:09.869160+0000","last_change":"2024-08-29T15:45:07.146261+0000","last_active":"2024-08-29T15:49:09.869160+0000","last_peered":"2024-08-29T15:49:09.869160+0000","last_clean":"2024-08-29T15:49:09.869160+0000","last_became_active":"2024-08-29T15:45:07.145979+0000","last_became_peered":"2024-08-29T15:45:07.145979+0000","last_unstale":"2024-08-29T15:49:09.869160+0000","last_undegraded":"2024-08-29T15:49:09.869160+0000","last_fullsized":"2024-08-29T15:49:09.869160+0000","mapping_epoch":480,"log_start":"502'9800","ondisk_log_start":"502'9800","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6028,"log_dups_size":0,"ondisk_log_size":6028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:28.876377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15828,"num_write_kb":506496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":946,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389455+0000","last_change":"2024-08-29T15:44:22.861452+0000","last_active":"2024-08-29T15:48:49.389455+0000","last_peered":"2024-08-29T15:48:49.389455+0000","last_clean":"2024-08-29T15:48:49.389455+0000","last_became_active":"2024-08-29T15:44:22.611166+0000","last_became_peered":"2024-08-29T15:44:22.611166+0000","last_unstale":"2024-08-29T15:48:49.389455+0000","last_undegraded":"2024-08-29T15:48:49.389455+0000","last_fullsized":"2024-08-29T15:48:49.389455+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.861280+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.861280+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:45.254970+0000","scrub_duration":0.0040975610000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":930,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387804+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:48:50.387804+0000","last_peered":"2024-08-29T15:48:50.387804+0000","last_clean":"2024-08-29T15:48:50.387804+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:48:50.387804+0000","last_undegraded":"2024-08-29T15:48:50.387804+0000","last_fullsized":"2024-08-29T15:48:50.387804+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1582,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386322+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:48:50.386322+0000","last_peered":"2024-08-29T15:48:50.386322+0000","last_clean":"2024-08-29T15:48:50.386322+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:48:50.386322+0000","last_undegraded":"2024-08-29T15:48:50.386322+0000","last_fullsized":"2024-08-29T15:48:50.386322+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":282,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385848+0000","last_change":"2024-08-29T15:48:12.266718+0000","last_active":"2024-08-29T15:48:50.385848+0000","last_peered":"2024-08-29T15:48:50.385848+0000","last_clean":"2024-08-29T15:48:50.385848+0000","last_became_active":"2024-08-29T15:48:12.266363+0000","last_became_peered":"2024-08-29T15:48:12.266363+0000","last_unstale":"2024-08-29T15:48:50.385848+0000","last_undegraded":"2024-08-29T15:48:50.385848+0000","last_fullsized":"2024-08-29T15:48:50.385848+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:53.550145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"530'15665","reported_seq":15777,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:49:07.309418+0000","last_change":"2024-08-29T15:45:07.148809+0000","last_active":"2024-08-29T15:49:07.309418+0000","last_peered":"2024-08-29T15:49:07.309418+0000","last_clean":"2024-08-29T15:49:07.309418+0000","last_became_active":"2024-08-29T15:45:07.148514+0000","last_became_peered":"2024-08-29T15:45:07.148514+0000","last_unstale":"2024-08-29T15:49:07.309418+0000","last_undegraded":"2024-08-29T15:49:07.309418+0000","last_fullsized":"2024-08-29T15:49:07.309418+0000","mapping_epoch":480,"log_start":"501'9198","ondisk_log_start":"501'9198","created":480,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T15:45:05.860465+0000","last_clean_scrub_stamp":"2024-08-29T15:45:05.860465+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:42:28.486131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9371648,"num_objects":143,"num_object_clones":0,"num_object_copies":429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15665,"num_write_kb":505856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":1109,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385557+0000","last_change":"2024-08-29T15:43:26.775058+0000","last_active":"2024-08-29T15:48:50.385557+0000","last_peered":"2024-08-29T15:48:50.385557+0000","last_clean":"2024-08-29T15:48:50.385557+0000","last_became_active":"2024-08-29T15:43:26.284121+0000","last_became_peered":"2024-08-29T15:43:26.284121+0000","last_unstale":"2024-08-29T15:48:50.385557+0000","last_undegraded":"2024-08-29T15:48:50.385557+0000","last_fullsized":"2024-08-29T15:48:50.385557+0000","mapping_epoch":427,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.774861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.774861+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:50:14.013328+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":87,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253467+0000","last_change":"2024-08-29T15:46:58.380711+0000","last_active":"2024-08-29T15:49:10.253467+0000","last_peered":"2024-08-29T15:49:10.253467+0000","last_clean":"2024-08-29T15:49:10.253467+0000","last_became_active":"2024-08-29T15:46:58.369572+0000","last_became_peered":"2024-08-29T15:46:58.369572+0000","last_unstale":"2024-08-29T15:49:10.253467+0000","last_undegraded":"2024-08-29T15:49:10.253467+0000","last_fullsized":"2024-08-29T15:49:10.253467+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:49:53.912727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":333,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.385600+0000","last_change":"2024-08-29T15:48:12.274839+0000","last_active":"2024-08-29T15:48:49.385600+0000","last_peered":"2024-08-29T15:48:49.385600+0000","last_clean":"2024-08-29T15:48:49.385600+0000","last_became_active":"2024-08-29T15:48:12.268376+0000","last_became_peered":"2024-08-29T15:48:12.268376+0000","last_unstale":"2024-08-29T15:48:49.385600+0000","last_undegraded":"2024-08-29T15:48:49.385600+0000","last_fullsized":"2024-08-29T15:48:49.385600+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:19.911889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":278,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383196+0000","last_change":"2024-08-29T15:46:36.588929+0000","last_active":"2024-08-29T15:48:49.383196+0000","last_peered":"2024-08-29T15:48:49.383196+0000","last_clean":"2024-08-29T15:48:49.383196+0000","last_became_active":"2024-08-29T15:46:36.588017+0000","last_became_peered":"2024-08-29T15:46:36.588017+0000","last_unstale":"2024-08-29T15:48:49.383196+0000","last_undegraded":"2024-08-29T15:48:49.383196+0000","last_fullsized":"2024-08-29T15:48:49.383196+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:45:31.813350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"90'149","reported_seq":937,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385790+0000","last_change":"2024-08-29T15:44:59.239507+0000","last_active":"2024-08-29T15:48:50.385790+0000","last_peered":"2024-08-29T15:48:50.385790+0000","last_clean":"2024-08-29T15:48:50.385790+0000","last_became_active":"2024-08-29T15:44:59.239049+0000","last_became_peered":"2024-08-29T15:44:59.239049+0000","last_unstale":"2024-08-29T15:48:50.385790+0000","last_undegraded":"2024-08-29T15:48:50.385790+0000","last_fullsized":"2024-08-29T15:48:50.385790+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:28.746327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:28.746327+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:38:22.918512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":318,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389595+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:48:49.389595+0000","last_peered":"2024-08-29T15:48:49.389595+0000","last_clean":"2024-08-29T15:48:49.389595+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:48:49.389595+0000","last_undegraded":"2024-08-29T15:48:49.389595+0000","last_fullsized":"2024-08-29T15:48:49.389595+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":328,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389698+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:48:49.389698+0000","last_peered":"2024-08-29T15:48:49.389698+0000","last_clean":"2024-08-29T15:48:49.389698+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:48:49.389698+0000","last_undegraded":"2024-08-29T15:48:49.389698+0000","last_fullsized":"2024-08-29T15:48:49.389698+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"90'149","reported_seq":998,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383950+0000","last_change":"2024-08-29T15:47:01.397530+0000","last_active":"2024-08-29T15:48:49.383950+0000","last_peered":"2024-08-29T15:48:49.383950+0000","last_clean":"2024-08-29T15:48:49.383950+0000","last_became_active":"2024-08-29T15:47:01.397287+0000","last_became_peered":"2024-08-29T15:47:01.397287+0000","last_unstale":"2024-08-29T15:48:49.383950+0000","last_undegraded":"2024-08-29T15:48:49.383950+0000","last_fullsized":"2024-08-29T15:48:49.383950+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:28.337426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:28.337426+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:58:58.372823+0000","scrub_duration":0.001450535,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387195+0000","last_change":"2024-08-29T15:47:12.486814+0000","last_active":"2024-08-29T15:48:50.387195+0000","last_peered":"2024-08-29T15:48:50.387195+0000","last_clean":"2024-08-29T15:48:50.387195+0000","last_became_active":"2024-08-29T15:47:12.486372+0000","last_became_peered":"2024-08-29T15:47:12.486372+0000","last_unstale":"2024-08-29T15:48:50.387195+0000","last_undegraded":"2024-08-29T15:48:50.387195+0000","last_fullsized":"2024-08-29T15:48:50.387195+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:38.593835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":692,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386942+0000","last_change":"2024-08-29T15:28:35.517849+0000","last_active":"2024-08-29T15:48:50.386942+0000","last_peered":"2024-08-29T15:48:50.386942+0000","last_clean":"2024-08-29T15:48:50.386942+0000","last_became_active":"2024-08-29T15:19:46.806353+0000","last_became_peered":"2024-08-29T15:19:46.806353+0000","last_unstale":"2024-08-29T15:48:50.386942+0000","last_undegraded":"2024-08-29T15:48:50.386942+0000","last_fullsized":"2024-08-29T15:48:50.386942+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:53:49.473676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'149","reported_seq":1112,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.429140+0000","last_change":"2024-08-29T15:44:59.244921+0000","last_active":"2024-08-29T15:48:50.429140+0000","last_peered":"2024-08-29T15:48:50.429140+0000","last_clean":"2024-08-29T15:48:50.429140+0000","last_became_active":"2024-08-29T15:44:59.244391+0000","last_became_peered":"2024-08-29T15:44:59.244391+0000","last_unstale":"2024-08-29T15:48:50.429140+0000","last_undegraded":"2024-08-29T15:48:50.429140+0000","last_fullsized":"2024-08-29T15:48:50.429140+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.778004+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.778004+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:12:04.168824+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":408,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385806+0000","last_change":"2024-08-29T15:17:40.720957+0000","last_active":"2024-08-29T15:48:50.385806+0000","last_peered":"2024-08-29T15:48:50.385806+0000","last_clean":"2024-08-29T15:48:50.385806+0000","last_became_active":"2024-08-29T15:17:40.720638+0000","last_became_peered":"2024-08-29T15:17:40.720638+0000","last_unstale":"2024-08-29T15:48:50.385806+0000","last_undegraded":"2024-08-29T15:48:50.385806+0000","last_fullsized":"2024-08-29T15:48:50.385806+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:09:25.733333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":893,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386160+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:48:50.386160+0000","last_peered":"2024-08-29T15:48:50.386160+0000","last_clean":"2024-08-29T15:48:50.386160+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:48:50.386160+0000","last_undegraded":"2024-08-29T15:48:50.386160+0000","last_fullsized":"2024-08-29T15:48:50.386160+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"90'149","reported_seq":841,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.386159+0000","last_change":"2024-08-29T15:45:26.882752+0000","last_active":"2024-08-29T15:48:49.386159+0000","last_peered":"2024-08-29T15:48:49.386159+0000","last_clean":"2024-08-29T15:48:49.386159+0000","last_became_active":"2024-08-29T15:44:59.248534+0000","last_became_peered":"2024-08-29T15:44:59.248534+0000","last_unstale":"2024-08-29T15:48:49.386159+0000","last_undegraded":"2024-08-29T15:48:49.386159+0000","last_fullsized":"2024-08-29T15:48:49.386159+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.882638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.882638+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:29:07.801722+0000","scrub_duration":0.001489043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":82,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253699+0000","last_change":"2024-08-29T15:46:58.368403+0000","last_active":"2024-08-29T15:49:10.253699+0000","last_peered":"2024-08-29T15:49:10.253699+0000","last_clean":"2024-08-29T15:49:10.253699+0000","last_became_active":"2024-08-29T15:46:58.368078+0000","last_became_peered":"2024-08-29T15:46:58.368078+0000","last_unstale":"2024-08-29T15:49:10.253699+0000","last_undegraded":"2024-08-29T15:49:10.253699+0000","last_fullsized":"2024-08-29T15:49:10.253699+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:08:22.938874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":945,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385980+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:48:50.385980+0000","last_peered":"2024-08-29T15:48:50.385980+0000","last_clean":"2024-08-29T15:48:50.385980+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:48:50.385980+0000","last_undegraded":"2024-08-29T15:48:50.385980+0000","last_fullsized":"2024-08-29T15:48:50.385980+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"90'149","reported_seq":773,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389235+0000","last_change":"2024-08-29T15:45:26.912257+0000","last_active":"2024-08-29T15:48:49.389235+0000","last_peered":"2024-08-29T15:48:49.389235+0000","last_clean":"2024-08-29T15:48:49.389235+0000","last_became_active":"2024-08-29T15:44:59.244250+0000","last_became_peered":"2024-08-29T15:44:59.244250+0000","last_unstale":"2024-08-29T15:48:49.389235+0000","last_undegraded":"2024-08-29T15:48:49.389235+0000","last_fullsized":"2024-08-29T15:48:49.389235+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.912179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.912179+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:51.014992+0000","scrub_duration":0.0012045789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":390,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385898+0000","last_change":"2024-08-29T15:32:26.141004+0000","last_active":"2024-08-29T15:48:50.385898+0000","last_peered":"2024-08-29T15:48:50.385898+0000","last_clean":"2024-08-29T15:48:50.385898+0000","last_became_active":"2024-08-29T15:32:26.140664+0000","last_became_peered":"2024-08-29T15:32:26.140664+0000","last_unstale":"2024-08-29T15:48:50.385898+0000","last_undegraded":"2024-08-29T15:48:50.385898+0000","last_fullsized":"2024-08-29T15:48:50.385898+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:58:44.753739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252592+0000","last_change":"2024-08-29T15:47:12.486034+0000","last_active":"2024-08-29T15:49:10.252592+0000","last_peered":"2024-08-29T15:49:10.252592+0000","last_clean":"2024-08-29T15:49:10.252592+0000","last_became_active":"2024-08-29T15:47:12.483444+0000","last_became_peered":"2024-08-29T15:47:12.483444+0000","last_unstale":"2024-08-29T15:49:10.252592+0000","last_undegraded":"2024-08-29T15:49:10.252592+0000","last_fullsized":"2024-08-29T15:49:10.252592+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:52:10.488628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"90'149","reported_seq":1058,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.386584+0000","last_change":"2024-08-29T15:47:01.398313+0000","last_active":"2024-08-29T15:48:49.386584+0000","last_peered":"2024-08-29T15:48:49.386584+0000","last_clean":"2024-08-29T15:48:49.386584+0000","last_became_active":"2024-08-29T15:47:01.397916+0000","last_became_peered":"2024-08-29T15:47:01.397916+0000","last_unstale":"2024-08-29T15:48:49.386584+0000","last_undegraded":"2024-08-29T15:48:49.386584+0000","last_fullsized":"2024-08-29T15:48:49.386584+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:23.954739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:23.954739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:01:09.974317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":97,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253459+0000","last_change":"2024-08-29T15:48:12.267373+0000","last_active":"2024-08-29T15:49:10.253459+0000","last_peered":"2024-08-29T15:49:10.253459+0000","last_clean":"2024-08-29T15:49:10.253459+0000","last_became_active":"2024-08-29T15:48:12.267059+0000","last_became_peered":"2024-08-29T15:48:12.267059+0000","last_unstale":"2024-08-29T15:49:10.253459+0000","last_undegraded":"2024-08-29T15:49:10.253459+0000","last_fullsized":"2024-08-29T15:49:10.253459+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:25:55.282346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":925,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385459+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:48:50.385459+0000","last_peered":"2024-08-29T15:48:50.385459+0000","last_clean":"2024-08-29T15:48:50.385459+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:48:50.385459+0000","last_undegraded":"2024-08-29T15:48:50.385459+0000","last_fullsized":"2024-08-29T15:48:50.385459+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'149","reported_seq":785,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383193+0000","last_change":"2024-08-29T15:45:27.312630+0000","last_active":"2024-08-29T15:48:49.383193+0000","last_peered":"2024-08-29T15:48:49.383193+0000","last_clean":"2024-08-29T15:48:49.383193+0000","last_became_active":"2024-08-29T15:44:59.247776+0000","last_became_peered":"2024-08-29T15:44:59.247776+0000","last_unstale":"2024-08-29T15:48:49.383193+0000","last_undegraded":"2024-08-29T15:48:49.383193+0000","last_fullsized":"2024-08-29T15:48:49.383193+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:27.312539+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:27.312539+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:34:07.441954+0000","scrub_duration":0.0034059530000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":929,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386834+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:48:50.386834+0000","last_peered":"2024-08-29T15:48:50.386834+0000","last_clean":"2024-08-29T15:48:50.386834+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:48:50.386834+0000","last_undegraded":"2024-08-29T15:48:50.386834+0000","last_fullsized":"2024-08-29T15:48:50.386834+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":317,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389485+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:48:49.389485+0000","last_peered":"2024-08-29T15:48:49.389485+0000","last_clean":"2024-08-29T15:48:49.389485+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:48:49.389485+0000","last_undegraded":"2024-08-29T15:48:49.389485+0000","last_fullsized":"2024-08-29T15:48:49.389485+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'149","reported_seq":1096,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386077+0000","last_change":"2024-08-29T15:47:01.424749+0000","last_active":"2024-08-29T15:48:50.386077+0000","last_peered":"2024-08-29T15:48:50.386077+0000","last_clean":"2024-08-29T15:48:50.386077+0000","last_became_active":"2024-08-29T15:47:01.424333+0000","last_became_peered":"2024-08-29T15:47:01.424333+0000","last_unstale":"2024-08-29T15:48:50.386077+0000","last_undegraded":"2024-08-29T15:48:50.386077+0000","last_fullsized":"2024-08-29T15:48:50.386077+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:31.851590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:31.851590+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:01.272901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":970,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386086+0000","last_change":"2024-08-29T15:39:11.832387+0000","last_active":"2024-08-29T15:48:50.386086+0000","last_peered":"2024-08-29T15:48:50.386086+0000","last_clean":"2024-08-29T15:48:50.386086+0000","last_became_active":"2024-08-29T15:39:11.832056+0000","last_became_peered":"2024-08-29T15:39:11.832056+0000","last_unstale":"2024-08-29T15:48:50.386086+0000","last_undegraded":"2024-08-29T15:48:50.386086+0000","last_fullsized":"2024-08-29T15:48:50.386086+0000","mapping_epoch":405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:47:42.049015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":82,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.252757+0000","last_change":"2024-08-29T15:46:58.367038+0000","last_active":"2024-08-29T15:49:10.252757+0000","last_peered":"2024-08-29T15:49:10.252757+0000","last_clean":"2024-08-29T15:49:10.252757+0000","last_became_active":"2024-08-29T15:46:58.366692+0000","last_became_peered":"2024-08-29T15:46:58.366692+0000","last_unstale":"2024-08-29T15:49:10.252757+0000","last_undegraded":"2024-08-29T15:49:10.252757+0000","last_fullsized":"2024-08-29T15:49:10.252757+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:40:02.240790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1e","version":"90'149","reported_seq":943,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386896+0000","last_change":"2024-08-29T15:44:59.241503+0000","last_active":"2024-08-29T15:48:50.386896+0000","last_peered":"2024-08-29T15:48:50.386896+0000","last_clean":"2024-08-29T15:48:50.386896+0000","last_became_active":"2024-08-29T15:44:59.241088+0000","last_became_peered":"2024-08-29T15:44:59.241088+0000","last_unstale":"2024-08-29T15:48:50.386896+0000","last_undegraded":"2024-08-29T15:48:50.386896+0000","last_fullsized":"2024-08-29T15:48:50.386896+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:27.788587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:27.788587+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:39.021866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":353,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.389357+0000","last_change":"2024-08-29T15:47:12.481314+0000","last_active":"2024-08-29T15:48:49.389357+0000","last_peered":"2024-08-29T15:48:49.389357+0000","last_clean":"2024-08-29T15:48:49.389357+0000","last_became_active":"2024-08-29T15:47:12.481010+0000","last_became_peered":"2024-08-29T15:47:12.481010+0000","last_unstale":"2024-08-29T15:48:49.389357+0000","last_undegraded":"2024-08-29T15:48:49.389357+0000","last_fullsized":"2024-08-29T15:48:49.389357+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:58:04.897625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":450,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383207+0000","last_change":"2024-08-29T15:34:31.298658+0000","last_active":"2024-08-29T15:48:49.383207+0000","last_peered":"2024-08-29T15:48:49.383207+0000","last_clean":"2024-08-29T15:48:49.383207+0000","last_became_active":"2024-08-29T15:34:31.294456+0000","last_became_peered":"2024-08-29T15:34:31.294456+0000","last_unstale":"2024-08-29T15:48:49.383207+0000","last_undegraded":"2024-08-29T15:48:49.383207+0000","last_fullsized":"2024-08-29T15:48:49.383207+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:40:27.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":229,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385683+0000","last_change":"2024-08-29T15:46:36.588938+0000","last_active":"2024-08-29T15:48:50.385683+0000","last_peered":"2024-08-29T15:48:50.385683+0000","last_clean":"2024-08-29T15:48:50.385683+0000","last_became_active":"2024-08-29T15:46:36.588605+0000","last_became_peered":"2024-08-29T15:46:36.588605+0000","last_unstale":"2024-08-29T15:48:50.385683+0000","last_undegraded":"2024-08-29T15:48:50.385683+0000","last_fullsized":"2024-08-29T15:48:50.385683+0000","mapping_epoch":492,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:25:29.576817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":450,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:49.383528+0000","last_change":"2024-08-29T15:34:31.296758+0000","last_active":"2024-08-29T15:48:49.383528+0000","last_peered":"2024-08-29T15:48:49.383528+0000","last_clean":"2024-08-29T15:48:49.383528+0000","last_became_active":"2024-08-29T15:34:31.294477+0000","last_became_peered":"2024-08-29T15:34:31.294477+0000","last_unstale":"2024-08-29T15:48:49.383528+0000","last_undegraded":"2024-08-29T15:48:49.383528+0000","last_fullsized":"2024-08-29T15:48:49.383528+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:15.215291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"90'149","reported_seq":1127,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385732+0000","last_change":"2024-08-29T15:44:59.243558+0000","last_active":"2024-08-29T15:48:50.385732+0000","last_peered":"2024-08-29T15:48:50.385732+0000","last_clean":"2024-08-29T15:48:50.385732+0000","last_became_active":"2024-08-29T15:44:59.243114+0000","last_became_peered":"2024-08-29T15:44:59.243114+0000","last_unstale":"2024-08-29T15:48:50.385732+0000","last_undegraded":"2024-08-29T15:48:50.385732+0000","last_fullsized":"2024-08-29T15:48:50.385732+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:25.805561+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:25.805561+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:03:59.085143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":960,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.386335+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:48:50.386335+0000","last_peered":"2024-08-29T15:48:50.386335+0000","last_clean":"2024-08-29T15:48:50.386335+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:48:50.386335+0000","last_undegraded":"2024-08-29T15:48:50.386335+0000","last_fullsized":"2024-08-29T15:48:50.386335+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":964,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.387077+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:48:50.387077+0000","last_peered":"2024-08-29T15:48:50.387077+0000","last_clean":"2024-08-29T15:48:50.387077+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:48:50.387077+0000","last_undegraded":"2024-08-29T15:48:50.387077+0000","last_fullsized":"2024-08-29T15:48:50.387077+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'149","reported_seq":914,"reported_epoch":530,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.385414+0000","last_change":"2024-08-29T15:47:46.316018+0000","last_active":"2024-08-29T15:48:50.385414+0000","last_peered":"2024-08-29T15:48:50.385414+0000","last_clean":"2024-08-29T15:48:50.385414+0000","last_became_active":"2024-08-29T15:47:46.315610+0000","last_became_peered":"2024-08-29T15:47:46.315610+0000","last_unstale":"2024-08-29T15:48:50.385414+0000","last_undegraded":"2024-08-29T15:48:50.385414+0000","last_fullsized":"2024-08-29T15:48:50.385414+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:24.970456+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:45:24.970456+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:30:31.344363+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":318,"reported_epoch":529,"state":"active+clean","last_fresh":"2024-08-29T15:48:50.068951+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:48:50.068951+0000","last_peered":"2024-08-29T15:48:50.068951+0000","last_clean":"2024-08-29T15:48:50.068951+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:48:50.068951+0000","last_undegraded":"2024-08-29T15:48:50.068951+0000","last_fullsized":"2024-08-29T15:48:50.068951+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":89,"reported_epoch":531,"state":"active+clean","last_fresh":"2024-08-29T15:49:10.253699+0000","last_change":"2024-08-29T15:46:58.378379+0000","last_active":"2024-08-29T15:49:10.253699+0000","last_peered":"2024-08-29T15:49:10.253699+0000","last_clean":"2024-08-29T15:49:10.253699+0000","last_became_active":"2024-08-29T15:46:58.368284+0000","last_became_peered":"2024-08-29T15:46:58.368284+0000","last_unstale":"2024-08-29T15:49:10.253699+0000","last_undegraded":"2024-08-29T15:49:10.253699+0000","last_fullsized":"2024-08-29T15:49:10.253699+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:57:45.539791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":68878336,"num_objects":1051,"num_object_clones":0,"num_object_copies":3153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1051,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":251205,"num_write_kb":8072129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17538,"num_bytes_recovered":1149304856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":306778112,"data_stored":306774040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105783,"ondisk_log_size":105783,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":7780,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":16777235,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234654,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918032,"kb_used_data":24648,"kb_used_omap":1,"kb_used_meta":893310,"kb_avail":92827120,"statfs":{"total":95995035648,"available":95054970880,"internally_reserved":0,"allocated":25239552,"data_stored":21889573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":914749899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496285,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":718280,"kb_used_data":23488,"kb_used_omap":1,"kb_used_meta":694782,"kb_avail":93026872,"statfs":{"total":95995035648,"available":95259516928,"internally_reserved":0,"allocated":24051712,"data_stored":20709075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":711457223},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426872,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":669440,"kb_used_data":36792,"kb_used_omap":8,"kb_used_meta":632631,"kb_avail":93075712,"statfs":{"total":95995035648,"available":95309529088,"internally_reserved":0,"allocated":37675008,"data_stored":34336851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":647815050},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019564,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":981644,"kb_used_data":25988,"kb_used_omap":1,"kb_used_meta":955646,"kb_avail":92763508,"statfs":{"total":95995035648,"available":94989832192,"internally_reserved":0,"allocated":26611712,"data_stored":23265036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":978581958},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435748,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1058744,"kb_used_data":66352,"kb_used_omap":4,"kb_used_meta":992315,"kb_avail":92686408,"statfs":{"total":95995035648,"available":94910881792,"internally_reserved":0,"allocated":67944448,"data_stored":64613701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":1016130825},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903600,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846328,"kb_used_data":73200,"kb_used_omap":10,"kb_used_meta":773109,"kb_avail":92898824,"statfs":{"total":95995035648,"available":95128395776,"internally_reserved":0,"allocated":74956800,"data_stored":71625943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":791664144},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762801,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152116,"kb_used_data":168044,"kb_used_omap":7,"kb_used_meta":984056,"kb_avail":92593036,"statfs":{"total":95995035648,"available":94815268864,"internally_reserved":0,"allocated":172077056,"data_stored":168749189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":1007673698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926325,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923716,"kb_used_data":11832,"kb_used_omap":7,"kb_used_meta":911864,"kb_avail":92821436,"statfs":{"total":95995035648,"available":95049150464,"internally_reserved":0,"allocated":12115968,"data_stored":8784536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":933748879},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":151715840,"data_stored":151715840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":64749568,"data_stored":64749568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":48627712,"data_stored":48627712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":22478848,"data_stored":22478848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16060416,"data_stored":16056344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3145728,"data_stored":3145728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:49:14.507 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-29T15:49:14.507 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:49:14.507 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 7, 5, 6] 2024-08-29T15:49:14.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd out 5 2024-08-29T15:49:14.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.510 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:14.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:14.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:14.634 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:14.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:14.736 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:14.761 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:14.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:14.885 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:14.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:14.958 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2024-08-29T15:49:14.959 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:49:14.987 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:15.013 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:14 smithi171 ceph-mon[90031]: pgmap v1876: 119 pgs: 119 active+clean; 17 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-29T15:49:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:15.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:14 smithi027 ceph-mon[93921]: pgmap v1876: 119 pgs: 119 active+clean; 17 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-29T15:49:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:15.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:14 smithi027 ceph-mon[98970]: pgmap v1876: 119 pgs: 119 active+clean; 17 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-08-29T15:49:15.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:15.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:15.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:15.137 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:15.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:15.261 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:15.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:15.389 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:15.491 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:15.515 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:15.616 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:15.617 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:15.640 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:15.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:49:15.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:15.763 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:15.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:15.865 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:15.890 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:15.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:15.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:15.992 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:16.014 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:16.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:16.021 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:15 smithi027 ceph-mon[93921]: pgmap v1877: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-08-29T15:49:16.021 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:15 smithi027 ceph-mon[98970]: pgmap v1877: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-08-29T15:49:16.043 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-29T15:49:16.069 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:15 smithi171 ceph-mon[90031]: pgmap v1877: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-08-29T15:49:16.116 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:16.138 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:16.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:16.239 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:16.261 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:16.262 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:16.363 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:16.385 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:16.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:16.508 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:16.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:16.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:16.632 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:16.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:16.692 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:16.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:16.757 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:16.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:16.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:16.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:16.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:16.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:17.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:17.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:17.130 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:17.232 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:17.260 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:17.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:17.361 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:17.387 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:17.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:17.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:17.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:17.613 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:17.637 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:17.739 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:17.762 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:17.795 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:17.864 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:17.888 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:17.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:17.990 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:18.011 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:18.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:18.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.135 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:18.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:18.172 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:49:18.172 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:49:18.236 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.261 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:18.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:18.292 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:18 smithi027 ceph-mon[98970]: pgmap v1878: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 448 op/s 2024-08-29T15:49:18.292 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:18 smithi027 ceph-mon[93921]: pgmap v1878: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 448 op/s 2024-08-29T15:49:18.327 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:18 smithi171 ceph-mon[90031]: pgmap v1878: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 448 op/s 2024-08-29T15:49:18.362 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:18.385 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:18.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:18.509 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:18.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:18.633 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:18.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:18.735 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.758 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:18.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:18.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:18.881 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:18.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:18.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:18.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:19.006 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:19.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:19.008 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-29T15:49:19.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.132 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:19.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:19.216 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:19.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.256 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:19.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:19.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:19 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3918124041' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:49:19.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:19 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:49:19.271 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:19 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3918124041' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:49:19.271 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:19 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:49:19.271 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:49:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:49:19] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-08-29T15:49:19.272 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'unique_pool_5' removed 2024-08-29T15:49:19.358 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:19.379 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:19.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:19.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:19.504 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:19.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:19 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3918124041' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:49:19.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:19 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-29T15:49:19.605 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:19.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.628 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:19.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:19.729 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:19.755 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:19.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:19.857 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:19.880 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:19.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:19.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:19.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:20.004 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:20.106 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:20.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.128 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:20.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:20.229 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:20.250 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:20.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:20.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:20 smithi027 ceph-mon[93921]: pgmap v1879: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 294 op/s 2024-08-29T15:49:20.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:20 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:49:20.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:20 smithi027 ceph-mon[93921]: osdmap e532: 8 total, 8 up, 8 in 2024-08-29T15:49:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:20 smithi027 ceph-mon[98970]: pgmap v1879: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 294 op/s 2024-08-29T15:49:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:20 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:49:20.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:20 smithi027 ceph-mon[98970]: osdmap e532: 8 total, 8 up, 8 in 2024-08-29T15:49:20.319 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-29T15:49:20.320 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-08-29T15:49:20.320 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-29T15:49:20.320 INFO:tasks.workunit:Pulling workunits from ref reef 2024-08-29T15:49:20.321 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-29T15:49:20.321 DEBUG:teuthology.orchestra.run.smithi027:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-29T15:49:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.349 INFO:teuthology.orchestra.run.smithi027.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-29T15:49:20.350 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-29T15:49:20.352 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:20.374 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:20.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:20.413 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-29T15:49:20.413 DEBUG:teuthology.orchestra.run.smithi027:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-29T15:49:20.438 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:20 smithi171 ceph-mon[90031]: pgmap v1879: 119 pgs: 119 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 294 op/s 2024-08-29T15:49:20.438 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:20 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-29T15:49:20.438 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:20 smithi171 ceph-mon[90031]: osdmap e532: 8 total, 8 up, 8 in 2024-08-29T15:49:20.470 INFO:tasks.workunit:timeout=3h 2024-08-29T15:49:20.470 INFO:tasks.workunit:cleanup=True 2024-08-29T15:49:20.470 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-29T15:49:20.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.533 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:20.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:20.544 INFO:tasks.workunit.client.0.smithi027.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-29T15:49:20.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:20.657 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:20.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:20.759 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:20.780 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:20.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:20.882 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:20.904 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:20.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:20.979 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:49:20.979 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:49:21.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:21.029 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:21.130 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:21.152 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:21.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:21.254 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:21.275 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:21.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:21.400 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:21.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:21.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:21.524 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:21.626 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:21.650 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:21.657 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-29T15:49:21.752 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:21.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.777 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:21.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:21.878 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:21.901 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:21.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:21.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:21.903 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:21.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:21 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:49:21.626+0000 7f3f0d183640 -1 mon.a@0(leader).osd e532 definitely_dead 0 2024-08-29T15:49:22.004 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:22.026 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:22.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:22.173 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:22.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:22.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:22.279 INFO:teuthology.orchestra.run.smithi027.stderr:marked out osd.5. 2024-08-29T15:49:22.307 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:22.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:22.312 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:22 smithi171 ceph-mon[90031]: pgmap v1881: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:49:22.313 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/197828663' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:49:22.313 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:22 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:49:22.313 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:22 smithi171 ceph-mon[90031]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-29T15:49:22.409 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:22.431 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:22.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:22.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:22.555 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:22.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[98970]: pgmap v1881: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/197828663' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[98970]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[93921]: pgmap v1881: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/197828663' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-29T15:49:22.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:22 smithi027 ceph-mon[93921]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-29T15:49:22.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.682 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:22.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:22.784 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:22.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.807 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:22.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:22.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:22.930 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:22.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:22.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:22.982 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:49:22.982 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:49:23.001 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:49:23.001 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:49:23.016 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:49:23.016 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:49:23.033 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:23.033 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:49:23.034 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:49:23.052 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:49:23.052 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:49:23.057 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:23.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:23.069 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:49:23.070 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:49:23.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:49:23.086 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:49:23.103 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:49:23.103 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:49:23.159 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.182 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:23.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:23.278 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is set 2024-08-29T15:49:23.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:23.320 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:23.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:23.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.445 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:23.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:23.546 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-29T15:49:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[98970]: osdmap e533: 8 total, 8 up, 7 in 2024-08-29T15:49:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1329015241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:23.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4000358469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:23.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-29T15:49:23.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[93921]: osdmap e533: 8 total, 8 up, 7 in 2024-08-29T15:49:23.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1329015241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:23.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4000358469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:23.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.570 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:23.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:23.573 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:23 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-29T15:49:23.574 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:23 smithi171 ceph-mon[90031]: osdmap e533: 8 total, 8 up, 7 in 2024-08-29T15:49:23.574 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1329015241' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:23.574 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4000358469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:51080 10.9 10:9f1d1fe6:::benchmark_data_smithi027_374610_object51079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:42.736099+0000", 2024-08-29T15:49:23.583 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 220.844493438, 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.321943539, 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 51080 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.736099+0000", 2024-08-29T15:49:23.584 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.736099+0000", 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.736099+0000", 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.585 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.736125+0000", 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5621000000000001e-05 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.736131+0000", 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0929999999999999e-06 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.736135+0000", 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6199999999999998e-06 2024-08-29T15:49:23.586 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.737363+0000", 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012272520000000001 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.737484+0000", 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012131899999999999 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:49:23.587 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.737601+0000", 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011725799999999999 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.744944+0000", 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0073430040000000002 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.744981+0000", 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6878000000000002e-05 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.588 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.057879+0000", 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.312898173 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.057969+0000", 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9752000000000004e-05 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058043+0000", 2024-08-29T15:49:23.589 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3979000000000004e-05 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:51084 10.e 10:78604a72:::benchmark_data_smithi027_374610_object51083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:42.740019+0000", 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 220.840574207, 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.31804228099999998, 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.590 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 51084 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740019+0000", 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740019+0000", 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.591 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740017+0000", 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740089+0000", 2024-08-29T15:49:23.592 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1989000000000005e-05 2024-08-29T15:49:23.593 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740121+0000", 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.205e-05 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740133+0000", 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1922e-05 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.594 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740213+0000", 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9816999999999998e-05 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740351+0000", 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013813900000000001 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.740481+0000", 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012947999999999999 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.595 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.746734+0000", 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0062532880000000001 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.748479+0000", 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017445500000000001 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.057853+0000", 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.30937477699999999 2024-08-29T15:49:23.596 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.057958+0000", 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000104248 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058061+0000", 2024-08-29T15:49:23.597 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000103342 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:51088 10.e 10:7d47e90e:::benchmark_data_smithi027_374610_object51087:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:42.742642+0000", 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 220.837950823, 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.31550292800000002, 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.598 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.599 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 51088 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.742642+0000", 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.742642+0000", 2024-08-29T15:49:23.600 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.742640+0000", 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.742676+0000", 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5451000000000003e-05 2024-08-29T15:49:23.601 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.742685+0000", 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9439999999999994e-06 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.742690+0000", 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2259999999999996e-06 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.602 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.743563+0000", 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00087264700000000005 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.743715+0000", 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015238599999999999 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.743912+0000", 2024-08-29T15:49:23.603 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000196949 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.748512+0000", 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046004940000000001 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.750863+0000", 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0023502250000000001 2024-08-29T15:49:23.604 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058096+0000", 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.30723337499999998 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058123+0000", 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6914999999999999e-05 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.605 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058145+0000", 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2107e-05 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:51117 10.9 10:9e2310b9:::benchmark_data_smithi027_374610_object51116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:45:42.770330+0000", 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 220.81026304400001, 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.28781616900000001, 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.606 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 51117 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770330+0000", 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.607 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770330+0000", 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770329+0000", 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770373+0000", 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4495e-05 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770384+0000", 2024-08-29T15:49:23.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.097e-05 2024-08-29T15:49:23.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770390+0000", 2024-08-29T15:49:23.612 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4659999999999998e-06 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770426+0000", 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5905999999999998e-05 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770532+0000", 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000106125 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:49:23.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.770625+0000", 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3405000000000004e-05 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.772301+0000", 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016759780000000001 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:42.772562+0000", 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00026100899999999998 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058086+0000", 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.28552350900000001 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058113+0000", 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7036e-05 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:45:43.058146+0000", 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3251999999999999e-05 2024-08-29T15:49:23.615 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:94225 10.1 10:8da09c10:::benchmark_data_smithi027_374610_object94224:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:16.425690+0000", 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.154902502, 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26190332500000002, 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.616 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 94225 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.425690+0000", 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.425690+0000", 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.425689+0000", 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.425732+0000", 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3071999999999997e-05 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.425743+0000", 2024-08-29T15:49:23.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0489000000000001e-05 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.425749+0000", 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1290000000000001e-06 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.430889+0000", 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0051397480000000004 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.431018+0000", 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000129121 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,7", 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.431164+0000", 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014656399999999999 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.434257+0000", 2024-08-29T15:49:23.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003092638 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.434470+0000", 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00021298000000000001 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.687497+0000", 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.253027271 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.621 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.687552+0000", 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5223999999999998e-05 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.687594+0000", 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1390999999999997e-05 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:94231 10.c 10:37adfb65:::benchmark_data_smithi027_374610_object94230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2024-08-29T15:49:23.622 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:16.429724+0000", 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.15086876000001, 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25789813900000003, 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 94231 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.429724+0000", 2024-08-29T15:49:23.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.429724+0000", 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.429723+0000", 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.429787+0000", 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3165999999999998e-05 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.429794+0000", 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6190000000000002e-06 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.429798+0000", 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2830000000000003e-06 2024-08-29T15:49:23.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.434758+0000", 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049596099999999997 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.434874+0000", 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000115635 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,7", 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.434966+0000", 2024-08-29T15:49:23.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2517999999999997e-05 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.442503+0000", 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0075372049999999999 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.442628+0000", 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012442900000000001 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.687529+0000", 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.244900697 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.687584+0000", 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5118999999999999e-05 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.687622+0000", 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8600000000000003e-05 2024-08-29T15:49:23.628 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:94235 10.c 10:39cf5522:::benchmark_data_smithi027_374610_object94234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:16.432140+0000", 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.14845323200001, 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26745253000000002, 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.629 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 94235 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.432140+0000", 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.432140+0000", 2024-08-29T15:49:23.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.432139+0000", 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.432178+0000", 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9310000000000001e-05 2024-08-29T15:49:23.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.432191+0000", 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3392e-05 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.432200+0000", 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5380000000000002e-06 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437286+0000", 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005085951 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437426+0000", 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013999100000000001 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,7", 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437572+0000", 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000145933 2024-08-29T15:49:23.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.445515+0000", 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079434439999999992 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.445552+0000", 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6532000000000002e-05 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699502+0000", 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25395014700000001 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699552+0000", 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0680000000000003e-05 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699592+0000", 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9716999999999999e-05 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.635 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:94236 10.c 10:327931f7:::benchmark_data_smithi027_374610_object94235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:16.433546+0000", 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.14704643300001, 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26616769299999998, 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.636 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 94236 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.433546+0000", 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.433546+0000", 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.433545+0000", 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.433575+0000", 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0298e-05 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.433586+0000", 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0481e-05 2024-08-29T15:49:23.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.433592+0000", 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3929999999999996e-06 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.439542+0000", 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005949696 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.439678+0000", 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136413 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,7", 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.439824+0000", 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014626799999999999 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.445559+0000", 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0057342699999999996 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.445599+0000", 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0340999999999999e-05 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699630+0000", 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25403152299999998 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699677+0000", 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6230000000000003e-05 2024-08-29T15:49:23.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699714+0000", 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7418e-05 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:94240 10.c 10:3cfc0e3f:::benchmark_data_smithi027_374610_object94239:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:16.437649+0000", 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 187.14294370299999, 2024-08-29T15:49:23.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26234644000000001, 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 94240 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437649+0000", 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437649+0000", 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437648+0000", 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.644 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437684+0000", 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6520999999999998e-05 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437694+0000", 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.6490000000000003e-06 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.437700+0000", 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3130000000000004e-06 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.442653+0000", 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0049528489999999996 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.442815+0000", 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016248200000000001 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,7", 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.442979+0000", 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000163325 2024-08-29T15:49:23.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.445750+0000", 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0027708530000000002 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.446077+0000", 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00032720299999999998 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699763+0000", 2024-08-29T15:49:23.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25368634200000001 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699951+0000", 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00018808799999999999 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:16.699996+0000", 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4345000000000003e-05 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.648 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:99861 10.4 10:2a5e0dcb:::benchmark_data_smithi027_374610_object99860:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:21.706175+0000", 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 181.87441798399999, 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.24889017399999999, 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.649 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 99861 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706175+0000", 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706175+0000", 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706173+0000", 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706204+0000", 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.0734000000000001e-05 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706214+0000", 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0305999999999999e-05 2024-08-29T15:49:23.651 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706220+0000", 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6450000000000002e-06 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.722257+0000", 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.016037237999999999 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.722393+0000", 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013628800000000001 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.722540+0000", 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014657900000000001 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.731296+0000", 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0087560060000000002 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.733021+0000", 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017250519999999999 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.954994+0000", 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.221972907 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.955032+0000", 2024-08-29T15:49:23.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8739999999999998e-05 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.955065+0000", 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2614999999999999e-05 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:99864 10.4 10:283906ee:::benchmark_data_smithi027_374610_object99863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:21.706712+0000", 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 181.87388081399999, 2024-08-29T15:49:23.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.248443951, 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 99864 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706712+0000", 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706712+0000", 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706712+0000", 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706740+0000", 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7949e-05 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706746+0000", 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7270000000000001e-06 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.706751+0000", 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8080000000000004e-06 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.725000+0000", 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.018249398 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.725125+0000", 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000124444 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 1,3", 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.725277+0000", 2024-08-29T15:49:23.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015238 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.731299+0000", 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0060213259999999996 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.733057+0000", 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017583270000000001 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.955096+0000", 2024-08-29T15:49:23.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.22203943000000001 2024-08-29T15:49:23.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.955129+0000", 2024-08-29T15:49:23.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2255999999999997e-05 2024-08-29T15:49:23.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:21.955156+0000", 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7416e-05 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:102605 10.9 10:9a22226f:::benchmark_data_smithi027_374610_object102604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:24.455659+0000", 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 179.12493363600001, 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26250764500000001, 2024-08-29T15:49:23.663 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 102605 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455659+0000", 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455659+0000", 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455658+0000", 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455693+0000", 2024-08-29T15:49:23.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5967000000000003e-05 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455726+0000", 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2299999999999999e-05 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455733+0000", 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6000000000000001e-06 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455817+0000", 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.3709999999999996e-05 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.455951+0000", 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013354099999999999 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.456078+0000", 2024-08-29T15:49:23.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000127417 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.459483+0000", 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034054350000000001 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.461265+0000", 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0017813589999999999 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.718073+0000", 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25680816099999998 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.718124+0000", 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0982000000000001e-05 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.718167+0000", 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2907000000000001e-05 2024-08-29T15:49:23.669 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:102610 10.1 10:8cd4cd22:::benchmark_data_smithi027_374610_object102609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:24.460477+0000", 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 179.12011638300001, 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25761295899999997, 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.670 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 102610 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460477+0000", 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460477+0000", 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460475+0000", 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460536+0000", 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1284999999999993e-05 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460548+0000", 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1803e-05 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460555+0000", 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7460000000000003e-06 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460596+0000", 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1103999999999998e-05 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460789+0000", 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00019327799999999999 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,7", 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.460918+0000", 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000129097 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468004+0000", 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0070861129999999998 2024-08-29T15:49:23.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.471404+0000", 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033995119999999999 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.717992+0000", 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.24658824200000001 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.718048+0000", 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5448999999999997e-05 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.718089+0000", 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1779000000000001e-05 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:102619 10.1 10:8df00864:::benchmark_data_smithi027_374610_object102618:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2024-08-29T15:49:23.676 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:24.462626+0000", 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 179.11796709199999, 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.258338242, 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 102619 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.462626+0000", 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.462626+0000", 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.462625+0000", 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.462714+0000", 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9178999999999994e-05 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.462729+0000", 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4627000000000001e-05 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.462736+0000", 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5360000000000004e-06 2024-08-29T15:49:23.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.465339+0000", 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0026027149999999998 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.465471+0000", 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013244900000000001 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,7", 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.465611+0000", 2024-08-29T15:49:23.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013938000000000001 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468150+0000", 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0025392779999999998 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.471431+0000", 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0032811820000000001 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.720898+0000", 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.24946628800000001 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.720934+0000", 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6837e-05 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.720964+0000", 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9657000000000001e-05 2024-08-29T15:49:23.682 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:102620 10.9 10:9b8178ef:::benchmark_data_smithi027_374610_object102619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:24.468835+0000", 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 179.111757598, 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25202904599999998, 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.683 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 102620 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468835+0000", 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468835+0000", 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468834+0000", 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468881+0000", 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6771000000000003e-05 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468894+0000", 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2791e-05 2024-08-29T15:49:23.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468900+0000", 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.3890000000000002e-06 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468931+0000", 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1049e-05 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.469049+0000", 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117643 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.469148+0000", 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.8824000000000001e-05 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.473725+0000", 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0045778310000000001 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.476711+0000", 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029861290000000001 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.720781+0000", 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.244069124 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.720828+0000", 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7168e-05 2024-08-29T15:49:23.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.720864+0000", 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6562e-05 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:102621 10.9 10:9d2db5f6:::benchmark_data_smithi027_374610_object102620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:24.468912+0000", 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 179.11168080300001, 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25218317099999998, 2024-08-29T15:49:23.689 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 102621 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468912+0000", 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468912+0000", 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468912+0000", 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468945+0000", 2024-08-29T15:49:23.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3615000000000003e-05 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468954+0000", 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5550000000000003e-06 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.468970+0000", 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5971e-05 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.471370+0000", 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0024005419999999999 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.471514+0000", 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014326399999999999 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.471675+0000", 2024-08-29T15:49:23.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00016181100000000001 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.473806+0000", 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0021305920000000002 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.484879+0000", 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011072738 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.720992+0000", 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.23611320499999999 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.721063+0000", 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0925000000000003e-05 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.721095+0000", 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2453999999999998e-05 2024-08-29T15:49:23.695 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:214890 10.9 10:99f3e89e:::benchmark_data_smithi027_374610_object88760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:27.297404+0000", 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 56.283189239999999, 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098766548999999995, 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.696 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 214890 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297404+0000", 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297404+0000", 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297403+0000", 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297406+0000", 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.994e-06 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297412+0000", 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5489999999999997e-06 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297416+0000", 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8130000000000002e-06 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297449+0000", 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3348e-05 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.699 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297527+0000", 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7812999999999995e-05 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.297625+0000", 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7979000000000005e-05 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303231+0000", 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056065639999999996 2024-08-29T15:49:23.700 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303473+0000", 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000241325 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.396084+0000", 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.092611382000000006 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.701 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.396133+0000", 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8980999999999997e-05 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.396170+0000", 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7286000000000002e-05 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.702 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:214892 10.9 10:928cf181:::benchmark_data_smithi027_374610_object88762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:27.300200+0000", 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 56.28039295, 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.096496573000000002, 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 214892 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.703 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300200+0000", 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300200+0000", 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300199+0000", 2024-08-29T15:49:23.704 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300205+0000", 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2389999999999999e-06 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300217+0000", 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1885999999999999e-05 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.705 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300223+0000", 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6529999999999997e-06 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300265+0000", 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1616999999999998e-05 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300365+0000", 2024-08-29T15:49:23.706 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00010013099999999999 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,7", 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.300482+0000", 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000116656 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303506+0000", 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0030244650000000001 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.707 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303546+0000", 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9845999999999998e-05 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.396618+0000", 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.093072287000000004 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.396662+0000", 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3114000000000001e-05 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.708 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.396697+0000", 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4958e-05 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:249763 10.e 10:77157bab:::benchmark_data_smithi027_374610_object123633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e530)", 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:49:07.226475+0000", 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 16.354117961, 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.073339715, 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.709 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 249763 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226475+0000", 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.710 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226475+0000", 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226473+0000", 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226480+0000", 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9859999999999997e-06 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.711 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226494+0000", 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.435e-05 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226500+0000", 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9410000000000004e-06 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226540+0000", 2024-08-29T15:49:23.712 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9808999999999998e-05 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226635+0000", 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4321999999999996e-05 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.226722+0000", 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7206999999999994e-05 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.713 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230113+0000", 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033914570000000001 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230129+0000", 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5548000000000001e-05 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.299756+0000", 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.069627338999999996 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.714 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.299791+0000", 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4987999999999998e-05 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.299815+0000", 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3567e-05 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.715 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:249768 10.e 10:715dedc2:::benchmark_data_smithi027_374610_object123638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e530)", 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:49:07.230617+0000", 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 16.349975715999999, 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.069334698, 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 249768 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.716 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230617+0000", 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230617+0000", 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230615+0000", 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.717 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230622+0000", 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4180000000000002e-06 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230636+0000", 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3764999999999999e-05 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230642+0000", 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.6329999999999999e-06 2024-08-29T15:49:23.718 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230680+0000", 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7991999999999998e-05 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230793+0000", 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011320900000000001 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,6", 2024-08-29T15:49:23.719 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.230879+0000", 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6058999999999997e-05 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.233805+0000", 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0029255750000000001 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.233813+0000", 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2530000000000004e-06 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.720 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.299910+0000", 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.066096472000000003 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.299934+0000", 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3748999999999999e-05 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:07.299952+0000", 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8362000000000001e-05 2024-08-29T15:49:23.721 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.722 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.722 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.722 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:23.722 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:23.722 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:49:23.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:23.744 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:23.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:23.868 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:23.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:23.993 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:23.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:24.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:24.117 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:24.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:24.218 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:24.240 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[93921]: pgmap v1883: 103 pgs: 2 unknown, 6 peering, 95 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:24.318 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[93921]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1329015241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[93921]: osdmap e534: 8 total, 8 up, 7 in 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[98970]: pgmap v1883: 103 pgs: 2 unknown, 6 peering, 95 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[98970]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1329015241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:24.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:24 smithi027 ceph-mon[98970]: osdmap e534: 8 total, 8 up, 7 in 2024-08-29T15:49:24.342 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.364 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:24.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:24.465 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:24.486 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:24.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:24 smithi171 ceph-mon[90031]: pgmap v1883: 103 pgs: 2 unknown, 6 peering, 95 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:24 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-29T15:49:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:24 smithi171 ceph-mon[90031]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1329015241' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:24.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:24 smithi171 ceph-mon[90031]: osdmap e534: 8 total, 8 up, 7 in 2024-08-29T15:49:24.588 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:24.610 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:24.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:24.648 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-29T15:49:24.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:24.736 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:24.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:24.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:24.860 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:24.887 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:24.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:24.985 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:24.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:24.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:25.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:25.108 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:25.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:25.210 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:25.232 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:25.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:25.334 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:25.355 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:25.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:25.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.480 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:25.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:25.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:25 smithi171 ceph-mon[90031]: osdmap e535: 8 total, 8 up, 7 in 2024-08-29T15:49:25.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:25 smithi027 ceph-mon[98970]: osdmap e535: 8 total, 8 up, 7 in 2024-08-29T15:49:25.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:25 smithi027 ceph-mon[93921]: osdmap e535: 8 total, 8 up, 7 in 2024-08-29T15:49:25.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:25.603 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:25.705 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:25.727 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:25.829 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:25.852 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:25.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:25.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:25.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:25.978 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:25.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:26.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-29T15:49:26.080 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.104 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:26.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:26.205 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:26.226 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:26.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:26.279 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:26.327 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.351 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:26.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:26.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:26 smithi027 ceph-mon[98970]: pgmap v1886: 103 pgs: 3 active+undersized+remapped, 1 remapped+peering, 2 unknown, 6 peering, 91 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 4.4 KiB/s, 0 objects/s recovering 2024-08-29T15:49:26.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:26 smithi027 ceph-mon[98970]: osdmap e536: 8 total, 8 up, 7 in 2024-08-29T15:49:26.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:26 smithi027 ceph-mon[93921]: pgmap v1886: 103 pgs: 3 active+undersized+remapped, 1 remapped+peering, 2 unknown, 6 peering, 91 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 4.4 KiB/s, 0 objects/s recovering 2024-08-29T15:49:26.407 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:26 smithi027 ceph-mon[93921]: osdmap e536: 8 total, 8 up, 7 in 2024-08-29T15:49:26.452 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.478 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:26.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:26.498 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:49:26.498 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:49:26.498 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:49:26.499 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:49:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:26 smithi171 ceph-mon[90031]: pgmap v1886: 103 pgs: 3 active+undersized+remapped, 1 remapped+peering, 2 unknown, 6 peering, 91 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 4.4 KiB/s, 0 objects/s recovering 2024-08-29T15:49:26.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:26 smithi171 ceph-mon[90031]: osdmap e536: 8 total, 8 up, 7 in 2024-08-29T15:49:26.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:26.602 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:26.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:26.703 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:26.725 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:26.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:26.827 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:26.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.850 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:26.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:26.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:49:26.951 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:26.973 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:26.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:27.074 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:27.093 DEBUG:teuthology.orchestra.run.smithi027:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-29T15:49:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.098 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:27.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:27.121 INFO:tasks.workunit.client.0.smithi027.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-29T15:49:27.123 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-29T15:49:27.123 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-29T15:49:27.152 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-29T15:49:27.199 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:27.221 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:27.323 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:27.345 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:27.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:27.358 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:27.447 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:27.460 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-29T15:49:27.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.470 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:27.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:27.517 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-29T15:49:27.558 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-29T15:49:27.559 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-29T15:49:27.560 INFO:tasks.workunit.client.0.smithi027.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-29T15:49:27.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:27 smithi027 ceph-mon[98970]: osdmap e537: 8 total, 8 up, 7 in 2024-08-29T15:49:27.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:27 smithi027 ceph-mon[93921]: osdmap e537: 8 total, 8 up, 7 in 2024-08-29T15:49:27.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.594 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:27.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:27.600 INFO:tasks.workunit.client.0.smithi027.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-29T15:49:27.629 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-08-29T15:49:27.629 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-29T15:49:27.654 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:27 smithi171 ceph-mon[90031]: osdmap e537: 8 total, 8 up, 7 in 2024-08-29T15:49:27.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:27.710 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-08-29T15:49:27.712 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-08-29T15:49:27.712 DEBUG:teuthology.orchestra.run.smithi027:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-08-29T15:49:27.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.720 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:27.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:27.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:27.844 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:27.859 INFO:tasks.workunit.client.0.smithi027.stdout:Running main() from gmock_main.cc 2024-08-29T15:49:27.859 INFO:tasks.workunit.client.0.smithi027.stdout:[==========] Running 70 tests from 1 test suite. 2024-08-29T15:49:27.859 INFO:tasks.workunit.client.0.smithi027.stdout:[----------] Global test environment set-up. 2024-08-29T15:49:27.859 INFO:tasks.workunit.client.0.smithi027.stdout:[----------] 70 tests from TestClsRbd 2024-08-29T15:49:27.869 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:27.946 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:27.969 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:27.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:27.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:28.071 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:28.093 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:28.195 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:28.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.217 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:28.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:28.318 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:28.339 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:28.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:28.381 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:28 smithi027 ceph-mon[98970]: 1.18 scrub starts 2024-08-29T15:49:28.382 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:28 smithi027 ceph-mon[98970]: pgmap v1889: 103 pgs: 3 active+undersized+remapped, 1 remapped+peering, 2 unknown, 9 peering, 88 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 9 op/s; 187 KiB/s, 4 objects/s recovering 2024-08-29T15:49:28.382 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:28 smithi027 ceph-mon[93921]: 1.18 scrub starts 2024-08-29T15:49:28.382 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:28 smithi027 ceph-mon[93921]: pgmap v1889: 103 pgs: 3 active+undersized+remapped, 1 remapped+peering, 2 unknown, 9 peering, 88 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 9 op/s; 187 KiB/s, 4 objects/s recovering 2024-08-29T15:49:28.389 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:49:28.389 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:49:28.406 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:49:28.406 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:49:28.423 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:49:28.423 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:49:28.439 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:49:28.440 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:49:28.441 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:28.456 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:49:28.456 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:49:28.462 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:28.472 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:49:28.472 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:49:28.488 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:49:28.488 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:49:28.505 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:49:28.505 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:49:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:28 smithi171 ceph-mon[90031]: 1.18 scrub starts 2024-08-29T15:49:28.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:28 smithi171 ceph-mon[90031]: pgmap v1889: 103 pgs: 3 active+undersized+remapped, 1 remapped+peering, 2 unknown, 9 peering, 88 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 13 KiB/s rd, 37 KiB/s wr, 9 op/s; 187 KiB/s, 4 objects/s recovering 2024-08-29T15:49:28.565 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:28.587 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:28.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:28.713 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:28.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.837 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:28.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:28.940 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:28.962 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:28.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:29.063 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:29.085 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:29.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:29.106 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:49:29.106 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:49:29.187 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:29.209 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:29.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:29.288 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:49:29] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.51.0" 2024-08-29T15:49:29.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:29.320 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_all_features 2024-08-29T15:49:29.345 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:29.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:29.356 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_all_features (36 ms) 2024-08-29T15:49:29.356 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.copyup 2024-08-29T15:49:29.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:29.471 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 7, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 3, 4, 6, 7, 5, 2] 2024-08-29T15:49:29.471 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-29T15:49:29.472 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-29T15:49:29.472 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2024-08-29T15:49:29.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 2 0 2024-08-29T15:49:29.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.473 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:29.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:29.528 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.copyup (172 ms) 2024-08-29T15:49:29.528 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-08-29T15:49:29.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[98970]: osdmap e538: 8 total, 8 up, 7 in 2024-08-29T15:49:29.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1100833432' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi027-400956-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-29T15:49:29.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1314277200' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:29.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:29.541 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:29.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[93921]: osdmap e538: 8 total, 8 up, 7 in 2024-08-29T15:49:29.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1100833432' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi027-400956-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-29T15:49:29.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1314277200' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:29.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:29.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:29.545 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.sparse_copyup (17 ms) 2024-08-29T15:49:29.545 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-08-29T15:49:29.553 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-08-29T15:49:29.553 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-08-29T15:49:29.563 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2024-08-29T15:49:29.563 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.directory_methods 2024-08-29T15:49:29.575 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:29.594 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.directory_methods (31 ms) 2024-08-29T15:49:29.594 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.create 2024-08-29T15:49:29.596 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:29.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:29.619 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.create (25 ms) 2024-08-29T15:49:29.619 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_features 2024-08-29T15:49:29.621 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-08-29T15:49:29.621 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-08-29T15:49:29.623 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-08-29T15:49:29.623 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-08-29T15:49:29.625 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-08-29T15:49:29.625 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-08-29T15:49:29.627 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-08-29T15:49:29.627 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-08-29T15:49:29.628 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-08-29T15:49:29.628 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-08-29T15:49:29.632 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-08-29T15:49:29.633 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.get_size 2024-08-29T15:49:29.636 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-08-29T15:49:29.636 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.set_size 2024-08-29T15:49:29.640 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-08-29T15:49:29.641 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.protection_status 2024-08-29T15:49:29.661 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.protection_status (21 ms) 2024-08-29T15:49:29.662 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-08-29T15:49:29.669 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2024-08-29T15:49:29.670 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.parents_v1 2024-08-29T15:49:29.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2 2024-08-29T15:49:29.718 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.parents_v1 (49 ms) 2024-08-29T15:49:29.718 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.parents_v2 2024-08-29T15:49:29.738 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:29 smithi171 ceph-mon[90031]: osdmap e538: 8 total, 8 up, 7 in 2024-08-29T15:49:29.738 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1100833432' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi027-400956-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-29T15:49:29.738 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1314277200' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:29.738 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:29.738 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:29.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.743 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:29.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.2' 2024-08-29T15:49:29.750 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.parents_v2 (31 ms) 2024-08-29T15:49:29.750 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.snapshots 2024-08-29T15:49:29.763 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-29T15:49:29.771 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2024-08-29T15:49:29.771 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.snapid_race 2024-08-29T15:49:29.776 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2024-08-29T15:49:29.777 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.stripingv2 2024-08-29T15:49:29.786 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2024-08-29T15:49:29.786 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.object_map_save 2024-08-29T15:49:29.788 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2024-08-29T15:49:29.788 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-08-29T15:49:29.793 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2024-08-29T15:49:29.794 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.object_map_update 2024-08-29T15:49:29.801 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.object_map_update (8 ms) 2024-08-29T15:49:29.801 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-08-29T15:49:29.801 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-08-29T15:49:29.801 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-08-29T15:49:29.810 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2024-08-29T15:49:29.810 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-08-29T15:49:29.815 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2024-08-29T15:49:29.815 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.flags 2024-08-29T15:49:29.820 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-08-29T15:49:29.820 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.metadata 2024-08-29T15:49:29.844 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:29.846 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.metadata (27 ms) 2024-08-29T15:49:29.846 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.set_features 2024-08-29T15:49:29.853 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-08-29T15:49:29.853 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.mirror 2024-08-29T15:49:29.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.867 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:29.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:29.889 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.mirror (36 ms) 2024-08-29T15:49:29.889 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.mirror_image 2024-08-29T15:49:29.904 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2024-08-29T15:49:29.904 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-08-29T15:49:29.968 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:29.972 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:29.990 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:29.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:29.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:30.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:30.118 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:30.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:30.184 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:30.220 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:30.242 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:30.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:30.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:30.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:30.345 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:30 smithi171 ceph-mon[90031]: pgmap v1891: 135 pgs: 32 unknown, 3 active+undersized+remapped, 1 remapped+peering, 3 peering, 96 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 380 KiB/s rd, 279 KiB/s wr, 196 op/s; 152 KiB/s, 11 objects/s recovering 2024-08-29T15:49:30.345 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1100833432' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi027-400956-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-29T15:49:30.346 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:30 smithi171 ceph-mon[90031]: osdmap e539: 8 total, 8 up, 7 in 2024-08-29T15:49:30.346 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:30 smithi171 ceph-mon[90031]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:49:30.346 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:30 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-29T15:49:30.371 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:30.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:30.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:30.473 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:30.496 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:30.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:30.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:30.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[93921]: pgmap v1891: 135 pgs: 32 unknown, 3 active+undersized+remapped, 1 remapped+peering, 3 peering, 96 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 380 KiB/s rd, 279 KiB/s wr, 196 op/s; 152 KiB/s, 11 objects/s recovering 2024-08-29T15:49:30.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1100833432' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi027-400956-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-29T15:49:30.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[93921]: osdmap e539: 8 total, 8 up, 7 in 2024-08-29T15:49:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[93921]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:49:30.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-29T15:49:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[98970]: pgmap v1891: 135 pgs: 32 unknown, 3 active+undersized+remapped, 1 remapped+peering, 3 peering, 96 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 380 KiB/s rd, 279 KiB/s wr, 196 op/s; 152 KiB/s, 11 objects/s recovering 2024-08-29T15:49:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1100833432' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi027-400956-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-29T15:49:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[98970]: osdmap e539: 8 total, 8 up, 7 in 2024-08-29T15:49:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[98970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:49:30.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:30 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-29T15:49:30.598 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:30.619 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:30.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:30.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:30.747 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:30.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:30.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:30.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:30.875 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:30.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:30.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:30.977 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:30.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.000 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:31.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:31.101 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.124 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:31.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:31.225 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:31.247 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:31.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:31.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:31.370 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:31.407 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:31 smithi171 ceph-mon[90031]: osdmap e540: 8 total, 8 up, 7 in 2024-08-29T15:49:31.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:31.502 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:31.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:31.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:31 smithi027 ceph-mon[93921]: osdmap e540: 8 total, 8 up, 7 in 2024-08-29T15:49:31.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:31 smithi027 ceph-mon[98970]: osdmap e540: 8 total, 8 up, 7 in 2024-08-29T15:49:31.604 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:31.625 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:31.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:31.647 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:49:31.647 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:49:31.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-29T15:49:31.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:31.760 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:31.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:31.862 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:31.891 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:31.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:31.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:32.019 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:32.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.146 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:32.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:32.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:32.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.276 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:32.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:32.339 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-29T15:49:32.342 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:49:32.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:32.459 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:32 smithi171 ceph-mon[90031]: pgmap v1894: 135 pgs: 1 creating+peering, 20 unknown, 114 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 907 KiB/s rd, 642 KiB/s wr, 1.08k op/s; 162 KiB/s, 9 objects/s recovering 2024-08-29T15:49:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:32 smithi171 ceph-mon[90031]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:49:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:32 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:49:32.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:32 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2348951638' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:49:32.561 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:32.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.585 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:32.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:32.663 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.mirror_image_status (2759 ms) 2024-08-29T15:49:32.663 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-08-29T15:49:32.686 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4 2024-08-29T15:49:32.708 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:32.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.4' 2024-08-29T15:49:32.743 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.mirror_image_map (80 ms) 2024-08-29T15:49:32.743 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-08-29T15:49:32.751 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-08-29T15:49:32.751 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-08-29T15:49:32.762 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2024-08-29T15:49:32.763 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-08-29T15:49:32.765 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-08-29T15:49:32.765 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-08-29T15:49:32.767 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-08-29T15:49:32.767 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-08-29T15:49:32.770 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-08-29T15:49:32.770 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-08-29T15:49:32.776 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-08-29T15:49:32.776 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-08-29T15:49:32.779 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-08-29T15:49:32.779 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-08-29T15:49:32.783 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-08-29T15:49:32.783 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_image_add 2024-08-29T15:49:32.785 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-08-29T15:49:32.785 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-08-29T15:49:32.790 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_image_remove (5 ms) 2024-08-29T15:49:32.790 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_image_list 2024-08-29T15:49:32.792 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2024-08-29T15:49:32.793 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-08-29T15:49:32.796 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-08-29T15:49:32.796 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.image_group_add 2024-08-29T15:49:32.798 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-08-29T15:49:32.799 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-08-29T15:49:32.801 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-08-29T15:49:32.802 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.image_group_get 2024-08-29T15:49:32.804 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2024-08-29T15:49:32.804 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-08-29T15:49:32.805 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2024-08-29T15:49:32.805 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-08-29T15:49:32.807 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-08-29T15:49:32.807 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-08-29T15:49:32.810 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:32.811 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-08-29T15:49:32.811 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-08-29T15:49:32.813 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-08-29T15:49:32.813 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-08-29T15:49:32.815 INFO:tasks.workunit.client.0.smithi027.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-774-gfe43f1c9/rpm/el9/BUILD/ceph-18.2.4-774-gfe43f1c9/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-08-29T15:49:32.815 INFO:tasks.workunit.client.0.smithi027.stdout:Expected equality of these values: 2024-08-29T15:49:32.815 INFO:tasks.workunit.client.0.smithi027.stdout: 1U 2024-08-29T15:49:32.816 INFO:tasks.workunit.client.0.smithi027.stdout: Which is: 1 2024-08-29T15:49:32.816 INFO:tasks.workunit.client.0.smithi027.stdout: keys.size() 2024-08-29T15:49:32.816 INFO:tasks.workunit.client.0.smithi027.stdout: Which is: 3 2024-08-29T15:49:32.816 INFO:tasks.workunit.client.0.smithi027.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-08-29T15:49:32.816 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[93921]: pgmap v1894: 135 pgs: 1 creating+peering, 20 unknown, 114 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 907 KiB/s rd, 642 KiB/s wr, 1.08k op/s; 162 KiB/s, 9 objects/s recovering 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[93921]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2348951638' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[98970]: pgmap v1894: 135 pgs: 1 creating+peering, 20 unknown, 114 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 907 KiB/s rd, 642 KiB/s wr, 1.08k op/s; 162 KiB/s, 9 objects/s recovering 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[98970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:49:32.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:32 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2348951638' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:49:32.819 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2024-08-29T15:49:32.819 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-08-29T15:49:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.834 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:32.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:32.835 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-08-29T15:49:32.835 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-08-29T15:49:32.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:32.960 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:32.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:33.054 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (219 ms) 2024-08-29T15:49:33.054 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-08-29T15:49:33.058 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2024-08-29T15:49:33.058 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-08-29T15:49:33.060 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-08-29T15:49:33.060 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.trash_methods 2024-08-29T15:49:33.062 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:33.067 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-08-29T15:49:33.067 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.op_features 2024-08-29T15:49:33.075 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2024-08-29T15:49:33.075 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.clone_parent 2024-08-29T15:49:33.084 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:33.095 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-08-29T15:49:33.095 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-08-29T15:49:33.104 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2024-08-29T15:49:33.104 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.clone_child 2024-08-29T15:49:33.114 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-08-29T15:49:33.114 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-08-29T15:49:33.120 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2024-08-29T15:49:33.121 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.migration 2024-08-29T15:49:33.131 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.migration (11 ms) 2024-08-29T15:49:33.131 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.migration_v1 2024-08-29T15:49:33.142 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2024-08-29T15:49:33.142 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-08-29T15:49:33.185 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:33.207 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:33.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:33.247 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:33.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:33.331 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:33.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:33.456 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:33.558 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:33.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[93921]: 1.18 scrub starts 2024-08-29T15:49:33.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[93921]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:33.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2348951638' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:49:33.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[93921]: osdmap e541: 8 total, 8 up, 7 in 2024-08-29T15:49:33.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[98970]: 1.18 scrub starts 2024-08-29T15:49:33.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[98970]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:33.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2348951638' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:49:33.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:33 smithi027 ceph-mon[98970]: osdmap e541: 8 total, 8 up, 7 in 2024-08-29T15:49:33.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.579 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:33.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:33.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:33.684 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:33 smithi171 ceph-mon[90031]: 1.18 scrub starts 2024-08-29T15:49:33.684 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:33 smithi171 ceph-mon[90031]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-29T15:49:33.684 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2348951638' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:49:33.684 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:33 smithi171 ceph-mon[90031]: osdmap e541: 8 total, 8 up, 7 in 2024-08-29T15:49:33.704 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:33.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:33.831 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:33.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:33.933 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:33.955 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:33.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:34.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:34.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.081 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:34.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:34.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:34.204 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:34.305 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6 2024-08-29T15:49:34.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.329 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:34.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.6' 2024-08-29T15:49:34.363 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.2 primary-affinity to 0 (802) 2024-08-29T15:49:34.366 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1223 ms) 2024-08-29T15:49:34.366 INFO:tasks.workunit.client.0.smithi027.stdout:[ RUN ] TestClsRbd.sparsify 2024-08-29T15:49:34.410 INFO:tasks.workunit.client.0.smithi027.stdout:[ OK ] TestClsRbd.sparsify (44 ms) 2024-08-29T15:49:34.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:34.453 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:34 smithi027 ceph-mon[93921]: pgmap v1896: 135 pgs: 1 creating+peering, 134 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 1.02k op/s; 16 KiB/s, 0 objects/s recovering 2024-08-29T15:49:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:34 smithi027 ceph-mon[93921]: osdmap e542: 8 total, 8 up, 7 in 2024-08-29T15:49:34.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/704736531' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-08-29T15:49:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:34 smithi027 ceph-mon[98970]: pgmap v1896: 135 pgs: 1 creating+peering, 134 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 1.02k op/s; 16 KiB/s, 0 objects/s recovering 2024-08-29T15:49:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:34 smithi027 ceph-mon[98970]: osdmap e542: 8 total, 8 up, 7 in 2024-08-29T15:49:34.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/704736531' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-08-29T15:49:34.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0 2024-08-29T15:49:34.576 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.0' 2024-08-29T15:49:34.616 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:34 smithi171 ceph-mon[90031]: pgmap v1896: 135 pgs: 1 creating+peering, 134 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 1.02k op/s; 16 KiB/s, 0 objects/s recovering 2024-08-29T15:49:34.617 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:34 smithi171 ceph-mon[90031]: osdmap e542: 8 total, 8 up, 7 in 2024-08-29T15:49:34.617 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/704736531' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-08-29T15:49:34.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.702 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:34.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:34.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3 2024-08-29T15:49:34.825 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.3' 2024-08-29T15:49:34.927 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:34.950 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:34.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:35.051 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7 2024-08-29T15:49:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:35.075 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:35.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.7' 2024-08-29T15:49:35.176 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:35.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:49:35.198 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:35.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:35.246 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:35.297 INFO:teuthology.orchestra.run.smithi171.stdout:ERROR: (22) Invalid argument 2024-08-29T15:49:35.297 INFO:teuthology.orchestra.run.smithi171.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-29T15:49:35.300 DEBUG:teuthology.orchestra.run.smithi171:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5 2024-08-29T15:49:35.322 INFO:teuthology.orchestra.run.smithi171.stderr:sudo: docker: command not found 2024-08-29T15:49:35.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:35.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi171 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.5' 2024-08-29T15:49:35.385 INFO:tasks.workunit.client.0.smithi027.stdout:[----------] 70 tests from TestClsRbd (5090 ms total) 2024-08-29T15:49:35.385 INFO:tasks.workunit.client.0.smithi027.stdout: 2024-08-29T15:49:35.385 INFO:tasks.workunit.client.0.smithi027.stdout:[----------] Global test environment tear-down 2024-08-29T15:49:35.385 INFO:tasks.workunit.client.0.smithi027.stdout:[==========] 70 tests from 1 test suite ran. (7526 ms total) 2024-08-29T15:49:35.385 INFO:tasks.workunit.client.0.smithi027.stdout:[ PASSED ] 69 tests. 2024-08-29T15:49:35.386 INFO:tasks.workunit.client.0.smithi027.stdout:[ FAILED ] 1 test, listed below: 2024-08-29T15:49:35.386 INFO:tasks.workunit.client.0.smithi027.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-08-29T15:49:35.386 INFO:tasks.workunit.client.0.smithi027.stdout: 2024-08-29T15:49:35.386 INFO:tasks.workunit.client.0.smithi027.stdout: 1 FAILED TEST 2024-08-29T15:49:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:35.418 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-08-29T15:49:35.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-29T15:49:35.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:35.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:35.485 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:35.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:35.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:35 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/704736531' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-08-29T15:49:35.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:35 smithi171 ceph-mon[90031]: osdmap e543: 8 total, 8 up, 7 in 2024-08-29T15:49:35.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:35 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/704736531' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-08-29T15:49:35.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:35 smithi027 ceph-mon[93921]: osdmap e543: 8 total, 8 up, 7 in 2024-08-29T15:49:35.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:35 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/704736531' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-08-29T15:49:35.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:35 smithi027 ceph-mon[98970]: osdmap e543: 8 total, 8 up, 7 in 2024-08-29T15:49:35.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1 2024-08-29T15:49:35.609 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: docker: command not found 2024-08-29T15:49:35.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T15:49:35.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi027 with status 1: 'sudo docker kill -s 1 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd.1' 2024-08-29T15:49:35.646 INFO:tasks.thrashosds:joining thrashosds 2024-08-29T15:49:35.934 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-29T15:49:36.504 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:36 smithi027 ceph-mon[93921]: pgmap v1899: 135 pgs: 1 peering, 134 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 991 op/s 2024-08-29T15:49:36.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:36 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:49:36.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:36 smithi027 ceph-mon[93921]: osdmap e544: 8 total, 8 up, 7 in 2024-08-29T15:49:36.505 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:36 smithi027 ceph-mon[98970]: pgmap v1899: 135 pgs: 1 peering, 134 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 991 op/s 2024-08-29T15:49:36.505 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:36 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:49:36.505 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:36 smithi027 ceph-mon[98970]: osdmap e544: 8 total, 8 up, 7 in 2024-08-29T15:49:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:36 smithi171 ceph-mon[90031]: pgmap v1899: 135 pgs: 1 peering, 134 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 991 op/s 2024-08-29T15:49:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:36 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:49:36.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:36 smithi171 ceph-mon[90031]: osdmap e544: 8 total, 8 up, 7 in 2024-08-29T15:49:36.572 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:36.790 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0: {} 2024-08-29T15:49:36.790 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1: {} 2024-08-29T15:49:36.791 INFO:teuthology.orchestra.run.smithi027.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-29T15:49:36.791 INFO:teuthology.orchestra.run.smithi027.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-29T15:49:36.797 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2: {} 2024-08-29T15:49:36.797 INFO:teuthology.orchestra.run.smithi027.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-29T15:49:36.798 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:36.814 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3: {} 2024-08-29T15:49:36.814 INFO:teuthology.orchestra.run.smithi027.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-29T15:49:36.834 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4: {} 2024-08-29T15:49:36.834 INFO:teuthology.orchestra.run.smithi027.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-29T15:49:36.850 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5: {} 2024-08-29T15:49:36.850 INFO:teuthology.orchestra.run.smithi027.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-29T15:49:36.867 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6: {} 2024-08-29T15:49:36.867 INFO:teuthology.orchestra.run.smithi027.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-29T15:49:36.883 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7: {} 2024-08-29T15:49:36.884 INFO:teuthology.orchestra.run.smithi027.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-29T15:49:37.702 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:37 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3633584914' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:37.789 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:37 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3633584914' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:37.790 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:37 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3633584914' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-29T15:49:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:38 smithi027 ceph-mon[98970]: 1.18 scrub starts 2024-08-29T15:49:38.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:38 smithi027 ceph-mon[98970]: 1.18 scrub ok 2024-08-29T15:49:38.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:38 smithi027 ceph-mon[98970]: pgmap v1901: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:38 smithi027 ceph-mon[93921]: 1.18 scrub starts 2024-08-29T15:49:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:38 smithi027 ceph-mon[93921]: 1.18 scrub ok 2024-08-29T15:49:38.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:38 smithi027 ceph-mon[93921]: pgmap v1901: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:38.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:38 smithi171 ceph-mon[90031]: 1.18 scrub starts 2024-08-29T15:49:38.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:38 smithi171 ceph-mon[90031]: 1.18 scrub ok 2024-08-29T15:49:38.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:38 smithi171 ceph-mon[90031]: pgmap v1901: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:38.952 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:49:38.952 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:49:38.953 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:49:38.953 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:49:38.953 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:49:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:49:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:49:39] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.51.0" 2024-08-29T15:49:39.629 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-29T15:49:40.038 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:40.268 INFO:tasks.thrashosds.thrasher:inning osd 2024-08-29T15:49:40.269 INFO:tasks.thrashosds.thrasher:Adding osd 5 2024-08-29T15:49:40.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd in 5 2024-08-29T15:49:40.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:40 smithi027 ceph-mon[93921]: pgmap v1902: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:40.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:40 smithi027 ceph-mon[98970]: pgmap v1902: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:40.725 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:40 smithi171 ceph-mon[90031]: pgmap v1902: 103 pgs: 1 peering, 102 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:41.001 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:41.388 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:49:41.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[98970]: 1.1b scrub starts 2024-08-29T15:49:41.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[98970]: 1.1b scrub ok 2024-08-29T15:49:41.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2578650974' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:41.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[93921]: 1.1b scrub starts 2024-08-29T15:49:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[93921]: 1.1b scrub ok 2024-08-29T15:49:41.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2578650974' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:41.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:41 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:41 smithi171 ceph-mon[90031]: 1.1b scrub starts 2024-08-29T15:49:41.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:41 smithi171 ceph-mon[90031]: 1.1b scrub ok 2024-08-29T15:49:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:41 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2578650974' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:41.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:41 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:42.447 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[93921]: pgmap v1903: 103 pgs: 103 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[93921]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[93921]: osdmap e545: 8 total, 8 up, 7 in 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[98970]: pgmap v1903: 103 pgs: 103 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[98970]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:42.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:42 smithi027 ceph-mon[98970]: osdmap e545: 8 total, 8 up, 7 in 2024-08-29T15:49:42.600 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:42 smithi171 ceph-mon[90031]: pgmap v1903: 103 pgs: 103 active+clean; 4.4 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:42.600 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:42 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:49:42.600 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:42 smithi171 ceph-mon[90031]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-29T15:49:42.600 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:42 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:42.600 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:42 smithi171 ceph-mon[90031]: osdmap e545: 8 total, 8 up, 7 in 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148094 9.2 e453/440 9:414bc4a8:::benchmark_data_smithi027_341149_object45664:head v 453'9218, mlcod=430'2855)", 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.380994+0000", 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 407.217517724, 2024-08-29T15:49:42.607 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.033578950000000003, 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.380994+0000", 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.380994+0000", 2024-08-29T15:49:42.608 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.380993+0000", 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.381000+0000", 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0489999999999998e-06 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.381011+0000", 2024-08-29T15:49:42.609 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1219e-05 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.381017+0000", 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.365e-06 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.394752+0000", 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013734379999999999 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.610 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.394778+0000", 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.658e-05 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414517+0000", 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.019738215999999999 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414573+0000", 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6048999999999998e-05 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.611 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148100 9.2 e453/440 9:413568bd:::benchmark_data_smithi027_341149_object45670:head v 453'9219, mlcod=430'2855)", 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.387969+0000", 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 407.21054205500002, 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.036568690000000001, 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.612 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387969+0000", 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387969+0000", 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387968+0000", 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.613 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387975+0000", 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9589999999999998e-06 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387987+0000", 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1392999999999999e-05 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.387993+0000", 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.888e-06 2024-08-29T15:49:42.614 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.403310+0000", 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.015316695999999999 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.403334+0000", 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.355e-05 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.615 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424496+0000", 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.021162258 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424538+0000", 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2094000000000001e-05 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148105 9.2 e453/440 9:4d9f105c:::benchmark_data_smithi027_341149_object45675:head v 453'9220, mlcod=430'2855)", 2024-08-29T15:49:42.616 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.390364+0000", 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 407.208147043, 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034200055, 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390364+0000", 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.617 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390364+0000", 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390363+0000", 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390371+0000", 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9680000000000006e-06 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.618 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390386+0000", 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5197999999999999e-05 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.390394+0000", 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8690000000000004e-06 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414615+0000", 2024-08-29T15:49:42.619 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024220742 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.414638+0000", 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2867e-05 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424543+0000", 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0099052010000000006 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.620 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.424564+0000", 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1467e-05 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:148250 9.2 e453/440 9:475ad16a:::benchmark_data_smithi027_341149_object45820:head v 453'9230, mlcod=430'2866)", 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:55.662960+0000", 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 406.935551734, 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035166784, 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.621 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662960+0000", 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662960+0000", 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.622 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662959+0000", 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662967+0000", 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2239999999999995e-06 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662982+0000", 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5234e-05 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.623 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.662989+0000", 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5069999999999997e-06 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.663035+0000", 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.6383e-05 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.624 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.663059+0000", 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3623999999999999e-05 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.697510+0000", 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034451341000000003 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.625 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:55.698126+0000", 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00061651999999999996 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:152289 9.8 e454/435 9:1a4c53e5:::benchmark_data_smithi027_341149_object49859:head v 454'9556, mlcod=431'3113)", 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 399.34857733299998, 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.041027240999999999, 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.626 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249934+0000", 2024-08-29T15:49:42.627 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249937+0000", 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2399999999999999e-06 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249942+0000", 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9949999999999996e-06 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.628 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.249946+0000", 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.997e-06 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.282663+0000", 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032717165999999999 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.282691+0000", 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7729000000000002e-05 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.629 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.290934+0000", 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0082430369999999999 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:03.290961+0000", 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7282000000000001e-05 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:167613 9.a e460/424 9:5041f107:::benchmark_data_smithi027_341149_object65183:head v 460'10529, mlcod=460'10527)", 2024-08-29T15:49:42.630 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:31.020831+0000", 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 371.57768075000001, 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034174152999999999, 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020831+0000", 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.631 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020831+0000", 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020830+0000", 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020836+0000", 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4980000000000003e-06 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.632 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020845+0000", 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5320000000000006e-06 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.020850+0000", 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.451e-06 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.022801+0000", 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0019502860000000001 2024-08-29T15:49:42.633 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.022821+0000", 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0160999999999999e-05 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.054929+0000", 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032108593999999997 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:31.055005+0000", 2024-08-29T15:49:42.634 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5332000000000003e-05 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:172219 9.1 e461/420 9:85c0e43c:::benchmark_data_smithi027_341149_object69789:head v 461'10776, mlcod=461'10774)", 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:38.225280+0000", 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 364.37323133000001, 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.039086323999999999, 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.635 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225280+0000", 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225280+0000", 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225279+0000", 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.636 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225287+0000", 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8480000000000001e-06 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225299+0000", 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1595e-05 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.225306+0000", 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1949999999999997e-06 2024-08-29T15:49:42.637 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233238+0000", 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079318259999999995 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233257+0000", 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9757000000000001e-05 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264299+0000", 2024-08-29T15:49:42.638 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031041632999999999 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264366+0000", 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7428000000000006e-05 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:172221 9.1 e461/420 9:8160b39d:::benchmark_data_smithi027_341149_object69791:head v 461'10777, mlcod=461'10774)", 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:49:42.639 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 364.368948929, 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034854593000000003, 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:49:42.640 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229562+0000", 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.229570+0000", 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.9370000000000004e-06 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.641 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233292+0000", 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003722347 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.233300+0000", 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8529999999999992e-06 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.236669+0000", 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033689480000000001 2024-08-29T15:49:42.642 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.236687+0000", 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8088e-05 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264369+0000", 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027682521000000002 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:38.264417+0000", 2024-08-29T15:49:42.643 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7741000000000003e-05 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:185992 9.2 e467/440 9:47f34658:::benchmark_data_smithi027_341149_object83562:head v 467'11563, mlcod=432'4648)", 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 343.72392630299998, 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032982351, 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.644 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874585+0000", 2024-08-29T15:49:42.645 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874590+0000", 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4419999999999997e-06 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874599+0000", 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8289999999999997e-06 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.646 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.874603+0000", 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4039999999999997e-06 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.880284+0000", 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0056811320000000002 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.880312+0000", 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7317999999999998e-05 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.647 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.907528+0000", 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.027216081 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:43:58.907567+0000", 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9746999999999997e-05 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:186823 9.8 e467/435 9:1424c12e:::benchmark_data_smithi027_341149_object84393:head v 467'11728, mlcod=432'5285)", 2024-08-29T15:49:42.648 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 342.43389811700001, 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.037339148000000003, 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.649 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164613+0000", 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164619+0000", 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.7290000000000003e-06 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.650 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.651 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.651 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164631+0000", 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1591e-05 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.164637+0000", 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8839999999999999e-06 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.175974+0000", 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.011337017 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.652 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.175994+0000", 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9862000000000001e-05 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.201931+0000", 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.025936858 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:00.201952+0000", 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1928000000000001e-05 2024-08-29T15:49:42.653 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:189430 9.8 e467/435 9:19f93844:::benchmark_data_smithi027_341149_object87000:head v 467'11889, mlcod=432'5447)", 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:03.621554+0000", 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 338.97695781300001, 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.03492348, 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.654 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621554+0000", 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621554+0000", 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621553+0000", 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.655 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621558+0000", 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1290000000000004e-06 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621564+0000", 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6899999999999997e-06 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.621568+0000", 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0380000000000001e-06 2024-08-29T15:49:42.656 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.630821+0000", 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0092526529999999996 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.630843+0000", 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2084000000000001e-05 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.656440+0000", 2024-08-29T15:49:42.657 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.02559707 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:03.656477+0000", 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7143e-05 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:190138 9.1 e467/420 9:8604df0f:::benchmark_data_smithi027_341149_object87708:head v 467'11907, mlcod=467'11904)", 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:04.424851+0000", 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 338.17366078700002, 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034214171000000002, 2024-08-29T15:49:42.658 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424851+0000", 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424851+0000", 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.659 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424850+0000", 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424858+0000", 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0559999999999999e-06 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424863+0000", 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2469999999999999e-06 2024-08-29T15:49:42.660 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.424866+0000", 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6839999999999998e-06 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.428269+0000", 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034026820000000002 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.428288+0000", 2024-08-29T15:49:42.661 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8555000000000001e-05 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459029+0000", 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.030741764000000001 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459065+0000", 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5340999999999999e-05 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.662 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:190139 9.1 e467/420 9:8027c676:::benchmark_data_smithi027_341149_object87709:head v 467'11908, mlcod=467'11904)", 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:04.427334+0000", 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 338.17117764300002, 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.031759969999999998, 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427334+0000", 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.663 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427334+0000", 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427333+0000", 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999995 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427340+0000", 2024-08-29T15:49:42.664 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4400000000000002e-06 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427354+0000", 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.455e-05 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.427360+0000", 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1600000000000003e-06 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.665 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.434094+0000", 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0067339219999999998 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.434112+0000", 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7737e-05 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459070+0000", 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.024958270000000001 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.666 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:04.459094+0000", 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3598999999999999e-05 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191156 9.2 e467/440 9:408459e1:::benchmark_data_smithi027_341149_object88726:head v 467'11903, mlcod=432'4648)", 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.723860+0000", 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.87465165100002, 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035658529000000001, 2024-08-29T15:49:42.667 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723860+0000", 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723860+0000", 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.668 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723859+0000", 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723867+0000", 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6980000000000004e-06 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723880+0000", 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2942000000000001e-05 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.669 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.723895+0000", 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5211000000000001e-05 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.727039+0000", 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0031432750000000001 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.727060+0000", 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.1707000000000001e-05 2024-08-29T15:49:42.670 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759474+0000", 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032413497999999999 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759518+0000", 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4399e-05 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.671 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191159 9.2 e467/440 9:4454a5d8:::benchmark_data_smithi027_341149_object88729:head v 467'11904, mlcod=432'4648)", 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.724627+0000", 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.87388467300002, 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034913665000000003, 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724627+0000", 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.672 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724627+0000", 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724626+0000", 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724632+0000", 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1290000000000001e-06 2024-08-29T15:49:42.673 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724645+0000", 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2469999999999999e-05 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.724651+0000", 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9669999999999998e-06 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.732733+0000", 2024-08-29T15:49:42.674 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0080824529999999999 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.732760+0000", 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7226999999999999e-05 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759522+0000", 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.026761686 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.675 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.759540+0000", 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8464000000000001e-05 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.46778.0:191166 9.2 e467/440 9:4270f1af:::benchmark_data_smithi027_341149_object88736:head v 467'11905, mlcod=432'4648)", 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:44:05.729757+0000", 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 336.868754846, 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.035565313000000001, 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.676 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729757+0000", 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729757+0000", 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.677 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729756+0000", 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729764+0000", 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1720000000000007e-06 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729779+0000", 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4827999999999999e-05 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.678 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.729785+0000", 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4219999999999997e-06 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.738213+0000", 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.008427604 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.738232+0000", 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8895000000000001e-05 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.679 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.765289+0000", 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.02705747 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:44:05.765322+0000", 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2833000000000002e-05 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.680 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:102592 10.8 e490/480 10:168bc8fa:::benchmark_data_smithi027_374610_object102591:head v 490'6342, mlcod=490'6339)", 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:24.447580+0000", 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 198.150931152, 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26554319599999998, 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447580+0000", 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.681 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447580+0000", 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447579+0000", 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447631+0000", 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1847e-05 2024-08-29T15:49:42.682 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447643+0000", 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2488e-05 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447651+0000", 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4490000000000002e-06 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447717+0000", 2024-08-29T15:49:42.683 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5951999999999995e-05 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.447735+0000", 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8649999999999999e-05 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.713089+0000", 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.265354072 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.684 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.713123+0000", 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4084000000000001e-05 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:102604 10.8 e490/480 10:1017a31a:::benchmark_data_smithi027_374610_object102603:head v 490'6343, mlcod=490'6340)", 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:46:24.456652+0000", 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 198.141859576, 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.26706034499999998, 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.685 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.456652+0000", 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.456652+0000", 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.456651+0000", 2024-08-29T15:49:42.686 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.456690+0000", 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9394000000000003e-05 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.710425+0000", 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.25373498700000002 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.687 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.710458+0000", 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3043000000000002e-05 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.710511+0000", 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2558000000000001e-05 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.710539+0000", 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.8697999999999999e-05 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.688 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.723671+0000", 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013131930999999999 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:46:24.723712+0000", 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1025000000000001e-05 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:247330 10.1 e530/492 10:856e462e:::benchmark_data_smithi027_374610_object121200:head v 530'15888, mlcod=530'15886)", 2024-08-29T15:49:42.689 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:49:04.945028+0000", 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 37.653483731999998, 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034114458, 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945028+0000", 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945028+0000", 2024-08-29T15:49:42.690 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945027+0000", 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945036+0000", 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7440000000000002e-06 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.691 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945054+0000", 2024-08-29T15:49:42.692 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7360999999999999e-05 2024-08-29T15:49:42.692 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.692 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.692 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.692 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945062+0000", 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2679999999999994e-06 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.946745+0000", 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0016827120000000001 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.946798+0000", 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3789000000000003e-05 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.693 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.979095+0000", 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.032296525999999999 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.979142+0000", 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7144000000000001e-05 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.694 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_repop(client.47036.0:247332 10.1 e530/492 10:8d213969:::benchmark_data_smithi027_374610_object121202:head v 530'15889, mlcod=530'15886)", 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:49:04.945094+0000", 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 37.653417238999999, 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.034089903999999997, 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945094+0000", 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:42.695 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945094+0000", 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945094+0000", 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967296 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945102+0000", 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4200000000000007e-06 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.696 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945108+0000", 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.1120000000000004e-06 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.945112+0000", 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9389999999999999e-06 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.950460+0000", 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.005348635 2024-08-29T15:49:42.697 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.950486+0000", 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5551e-05 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.979148+0000", 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.028661829999999999 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:49:04.979184+0000", 2024-08-29T15:49:42.698 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.6566000000000002e-05 2024-08-29T15:49:42.699 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.699 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.699 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.699 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:42.699 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:42.699 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:49:43.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:43.507 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:43 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:43.528 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-29T15:49:43.708 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:43.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:43 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:43.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:43.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:43 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:43.725 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:44.054 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:49:43.705+0000 7f3f0d183640 -1 mon.a@0(leader).osd e545 definitely_dead 0 2024-08-29T15:49:44.404 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:44 smithi171 ceph-mon[90031]: pgmap v1905: 103 pgs: 103 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:44.404 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3483904841' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:49:44.404 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:44.404 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:44.409 INFO:teuthology.orchestra.run.smithi027.stderr:marked in osd.5. 2024-08-29T15:49:44.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[98970]: pgmap v1905: 103 pgs: 103 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:44.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3483904841' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:49:44.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:44.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:44.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[93921]: pgmap v1905: 103 pgs: 103 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:44.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3483904841' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-29T15:49:44.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:44.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:45.461 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:49:45.461 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:49:45.461 INFO:teuthology.orchestra.run.smithi171.stdout: "num_ops": 0 2024-08-29T15:49:45.461 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:49:45.461 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3483904841' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-29T15:49:45.461 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:45 smithi171 ceph-mon[90031]: osdmap e546: 8 total, 8 up, 8 in 2024-08-29T15:49:45.461 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:45.461 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:45 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:45.462 INFO:tasks.thrashosds.thrasher:Added osd 5 2024-08-29T15:49:45.463 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-08-29T15:49:45.675 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:45.796 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3483904841' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-29T15:49:45.797 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[98970]: osdmap e546: 8 total, 8 up, 8 in 2024-08-29T15:49:45.797 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:45.797 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:45.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3483904841' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-29T15:49:45.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[93921]: osdmap e546: 8 total, 8 up, 8 in 2024-08-29T15:49:45.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:45.797 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:45 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:46.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset noscrub 2024-08-29T15:49:46.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-29T15:49:46.377 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:46.676 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:46 smithi171 ceph-mon[90031]: pgmap v1907: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:46.676 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:46 smithi171 ceph-mon[90031]: osdmap e547: 8 total, 8 up, 8 in 2024-08-29T15:49:46.676 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:49:46.676 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:49:46.676 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:46.677 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:46 smithi171 ceph-mon[90031]: osdmap e548: 8 total, 8 up, 8 in 2024-08-29T15:49:46.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[98970]: pgmap v1907: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[98970]: osdmap e547: 8 total, 8 up, 8 in 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[98970]: osdmap e548: 8 total, 8 up, 8 in 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[93921]: pgmap v1907: 103 pgs: 1 peering, 102 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[93921]: osdmap e547: 8 total, 8 up, 8 in 2024-08-29T15:49:46.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:49:46.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:49:46.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:49:46.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:46 smithi027 ceph-mon[93921]: osdmap e548: 8 total, 8 up, 8 in 2024-08-29T15:49:47.308 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a: {} 2024-08-29T15:49:47.308 INFO:teuthology.orchestra.run.smithi027.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-29T15:49:47.318 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b: {} 2024-08-29T15:49:47.318 INFO:teuthology.orchestra.run.smithi027.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-29T15:49:47.328 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c: {} 2024-08-29T15:49:47.329 INFO:teuthology.orchestra.run.smithi027.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-29T15:49:47.550 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:48.086 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:49:48.086 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [], 2024-08-29T15:49:48.087 INFO:teuthology.orchestra.run.smithi171.stdout: "complaint_time": 30, 2024-08-29T15:49:48.087 INFO:teuthology.orchestra.run.smithi171.stdout: "num_blocked_ops": 0 2024-08-29T15:49:48.087 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:49:48.566 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:48 smithi171 ceph-mon[90031]: pgmap v1910: 103 pgs: 1 active+remapped, 5 active+undersized+remapped, 4 remapped+peering, 1 peering, 92 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 2 objects/s recovering 2024-08-29T15:49:48.566 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:48 smithi171 ceph-mon[90031]: osdmap e549: 8 total, 8 up, 8 in 2024-08-29T15:49:48.566 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/150011181' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-29T15:49:48.566 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:48 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:49:48.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[93921]: pgmap v1910: 103 pgs: 1 active+remapped, 5 active+undersized+remapped, 4 remapped+peering, 1 peering, 92 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 2 objects/s recovering 2024-08-29T15:49:48.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[93921]: osdmap e549: 8 total, 8 up, 8 in 2024-08-29T15:49:48.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/150011181' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-29T15:49:48.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:49:48.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[98970]: pgmap v1910: 103 pgs: 1 active+remapped, 5 active+undersized+remapped, 4 remapped+peering, 1 peering, 92 active+clean; 4.6 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 2 objects/s recovering 2024-08-29T15:49:48.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[98970]: osdmap e549: 8 total, 8 up, 8 in 2024-08-29T15:49:48.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/150011181' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-29T15:49:48.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:48 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-29T15:49:48.880 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-29T15:49:49.077 INFO:teuthology.orchestra.run.smithi171.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.b/config 2024-08-29T15:49:49.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 0 1 2024-08-29T15:49:49.283 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:49:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:49:49] "GET /metrics HTTP/1.1" 200 34150 "" "Prometheus/2.51.0" 2024-08-29T15:49:49.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:49 smithi171 ceph-mon[90031]: osdmap e550: 8 total, 8 up, 8 in 2024-08-29T15:49:49.554 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:49 smithi027 ceph-mon[93921]: osdmap e550: 8 total, 8 up, 8 in 2024-08-29T15:49:49.554 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:49 smithi027 ceph-mon[98970]: osdmap e550: 8 total, 8 up, 8 in 2024-08-29T15:49:49.727 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:50.309 INFO:teuthology.orchestra.run.smithi027.stderr:noscrub is unset 2024-08-29T15:49:50.493 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:50 smithi171 ceph-mon[90031]: pgmap v1913: 103 pgs: 1 active+undersized+remapped, 1 remapped+peering, 2 peering, 99 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 6.0 KiB/s rd, 18 KiB/s wr, 3 op/s; 109 KiB/s, 4 objects/s recovering 2024-08-29T15:49:50.493 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:50 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:49:50.493 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:50 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:49:50.494 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:50 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4148831687' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:49:50.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[93921]: pgmap v1913: 103 pgs: 1 active+undersized+remapped, 1 remapped+peering, 2 peering, 99 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 6.0 KiB/s rd, 18 KiB/s wr, 3 op/s; 109 KiB/s, 4 objects/s recovering 2024-08-29T15:49:50.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:49:50.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:49:50.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4148831687' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:49:50.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[98970]: pgmap v1913: 103 pgs: 1 active+undersized+remapped, 1 remapped+peering, 2 peering, 99 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 6.0 KiB/s rd, 18 KiB/s wr, 3 op/s; 109 KiB/s, 4 objects/s recovering 2024-08-29T15:49:50.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-29T15:49:50.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:49:50.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:50 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4148831687' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout:{ 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: "size": 20, 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 600, 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: "ops": [ 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129360 9.d 9:bde52ebc:::benchmark_data_smithi027_341149_object26930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.272298+0000", 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.640172296, 2024-08-29T15:49:50.920 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95432824800000005, 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129360 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272298+0000", 2024-08-29T15:49:50.921 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272298+0000", 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272296+0000", 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.922 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272302+0000", 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.7180000000000001e-06 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272313+0000", 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0728e-05 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272318+0000", 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9239999999999998e-06 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.923 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272365+0000", 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7036000000000002e-05 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.272376+0000", 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1297999999999999e-05 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.924 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.214809+0000", 2024-08-29T15:49:50.925 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94243309600000003 2024-08-29T15:49:50.925 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.925 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.925 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.925 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.214927+0000", 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000117853 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.215003+0000", 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6551000000000006e-05 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219227+0000", 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0042237120000000001 2024-08-29T15:49:50.926 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222490+0000", 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003262788 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226533+0000", 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040426460000000004 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.927 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226582+0000", 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9709000000000001e-05 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226626+0000", 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3510000000000002e-05 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129372 9.d 9:b04251a7:::benchmark_data_smithi027_341149_object26942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.287652+0000", 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.62481776599998, 2024-08-29T15:49:50.928 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.94708441799999998, 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129372 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287652+0000", 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.929 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287652+0000", 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287651+0000", 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.930 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287658+0000", 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0690000000000004e-06 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287671+0000", 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2781e-05 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287677+0000", 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.9500000000000004e-06 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.931 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287722+0000", 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4787999999999999e-05 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.287730+0000", 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.0430000000000008e-06 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219285+0000", 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93155439900000003 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.932 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219422+0000", 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013689600000000001 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.219511+0000", 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.9679000000000007e-05 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226757+0000", 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0072459239999999999 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.933 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226813+0000", 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5693999999999999e-05 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234663+0000", 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0078496350000000006 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.934 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234694+0000", 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.1883e-05 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234736+0000", 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2015999999999999e-05 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.935 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129383 9.d 9:b062f382:::benchmark_data_smithi027_341149_object26953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.304378+0000", 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.60809170800002, 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.93048768100000001, 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.936 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129383 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304378+0000", 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304378+0000", 2024-08-29T15:49:50.937 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304376+0000", 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304384+0000", 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7200000000000006e-06 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.938 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304400+0000", 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5546999999999999e-05 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.304407+0000", 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4379999999999998e-06 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.306628+0000", 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0022207770000000002 2024-08-29T15:49:50.939 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.306649+0000", 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.0724e-05 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222519+0000", 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91587083400000002 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222661+0000", 2024-08-29T15:49:50.940 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014186799999999999 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.222790+0000", 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012872900000000001 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226804+0000", 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0040143399999999999 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.941 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226820+0000", 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5138e-05 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234817+0000", 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079972889999999994 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234843+0000", 2024-08-29T15:49:50.942 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6574999999999999e-05 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234866+0000", 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.2379e-05 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.943 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129386 9.d 9:b11ef5c9:::benchmark_data_smithi027_341149_object26956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.310122+0000", 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.60234789700002, 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.92487085199999997, 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129386 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.944 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310122+0000", 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310122+0000", 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.945 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310120+0000", 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310129+0000", 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8000000000000004e-06 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.946 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310144+0000", 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5815e-05 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310153+0000", 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7329999999999992e-06 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310209+0000", 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.5673e-05 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.947 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.310219+0000", 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0550000000000001e-05 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226852+0000", 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91663254900000002 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.226988+0000", 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000136339 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.948 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.227083+0000", 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.4705999999999998e-05 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234786+0000", 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0077026150000000003 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234795+0000", 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.2709999999999998e-06 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.949 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234943+0000", 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00014839000000000001 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234967+0000", 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.3359000000000001e-05 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234993+0000", 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6163999999999999e-05 2024-08-29T15:49:50.950 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129392 9.d 9:b5b3a1de:::benchmark_data_smithi027_341149_object26962:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.320301+0000", 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.59216850000001, 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91480801899999997, 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.951 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129392 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320301+0000", 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320301+0000", 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.952 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320300+0000", 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320306+0000", 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6230000000000002e-06 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320315+0000", 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.3070000000000008e-06 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.953 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320320+0000", 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0420000000000002e-06 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320360+0000", 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.0213000000000003e-05 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.320367+0000", 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.691e-06 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.954 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.230939+0000", 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.91057188499999997 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.231056+0000", 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00011665799999999999 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.231130+0000", 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4276000000000005e-05 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.955 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234905+0000", 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037745859999999999 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.234912+0000", 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.5290000000000003e-06 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235038+0000", 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012573 2024-08-29T15:49:50.956 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235079+0000", 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1551999999999998e-05 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235109+0000", 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9969999999999999e-05 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129399 9.d 9:b72fd645:::benchmark_data_smithi027_341149_object26969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.957 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.327813+0000", 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.58465659799998, 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.92127104000000004, 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129399 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327813+0000", 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.958 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327813+0000", 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327812+0000", 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:50.959 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327819+0000", 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.1389999999999998e-06 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327834+0000", 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.467e-05 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.960 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327841+0000", 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7230000000000004e-06 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327895+0000", 2024-08-29T15:49:50.961 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.3366999999999999e-05 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.327905+0000", 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.076e-05 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235139+0000", 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.90723388999999999 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235216+0000", 2024-08-29T15:49:50.962 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6254000000000005e-05 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.235292+0000", 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6412e-05 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248835+0000", 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.013543026 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.963 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248879+0000", 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3544999999999997e-05 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248981+0000", 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000102285 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.964 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.965 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249045+0000", 2024-08-29T15:49:50.965 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4367000000000002e-05 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249084+0000", 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9029000000000002e-05 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129409 9.d 9:b9830034:::benchmark_data_smithi027_341149_object26979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.356970+0000", 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.55549981399997, 2024-08-29T15:49:50.966 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.89225831899999997, 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129409 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356970+0000", 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.967 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356970+0000", 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356969+0000", 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356976+0000", 2024-08-29T15:49:50.968 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0299999999999996e-06 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356989+0000", 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2847000000000001e-05 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.356995+0000", 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5699999999999998e-06 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.357046+0000", 2024-08-29T15:49:50.969 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0504999999999998e-05 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.357055+0000", 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1339999999999993e-06 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.239754+0000", 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.88269909199999996 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.970 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.239912+0000", 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00015809299999999999 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.240041+0000", 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012887200000000001 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248888+0000", 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.008847644 2024-08-29T15:49:50.971 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.248924+0000", 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5528000000000003e-05 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249130+0000", 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00020563600000000001 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.972 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249194+0000", 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4634000000000005e-05 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249228+0000", 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.4020999999999998e-05 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.973 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129444 9.d 9:b42f07b7:::benchmark_data_smithi027_341149_object27014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.417884+0000", 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.49458612900003, 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.83963427700000004, 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129444 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.974 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417884+0000", 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417884+0000", 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417882+0000", 2024-08-29T15:49:50.975 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417890+0000", 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1759999999999993e-06 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417907+0000", 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7054999999999998e-05 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.976 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417916+0000", 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4339999999999992e-06 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.417993+0000", 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.7540000000000006e-05 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.977 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.418003+0000", 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.7950000000000002e-06 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.244378+0000", 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.82637549499999996 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.978 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.244510+0000", 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013137899999999999 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.244598+0000", 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8317000000000002e-05 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249273+0000", 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0046749579999999999 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.979 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249279+0000", 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0090000000000004e-06 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257450+0000", 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0081709120000000007 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257488+0000", 2024-08-29T15:49:50.980 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.8463000000000001e-05 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257518+0000", 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.9573e-05 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.46778.0:129460 9.d 9:b66c46df:::benchmark_data_smithi027_341149_object27030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e449)", 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:42:19.450107+0000", 2024-08-29T15:49:50.981 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 451.46236270100002, 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.80755162700000005, 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.46778", 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/283518739", 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 129460 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450107+0000", 2024-08-29T15:49:50.982 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450107+0000", 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450105+0000", 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450113+0000", 2024-08-29T15:49:50.983 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3599999999999998e-06 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450124+0000", 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1460999999999999e-05 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450131+0000", 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5740000000000001e-06 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.984 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450190+0000", 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.9763999999999999e-05 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:19.450201+0000", 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0549e-05 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249301+0000", 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.79910004899999998 2024-08-29T15:49:50.985 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249448+0000", 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000147362 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 4,7", 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.249581+0000", 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013289500000000001 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.986 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257568+0000", 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0079868119999999994 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257574+0000", 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6169999999999998e-06 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257600+0000", 2024-08-29T15:49:50.987 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6412000000000001e-05 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257627+0000", 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.6397000000000002e-05 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:42:20.257659+0000", 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.2181999999999998e-05 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.988 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:175995 10.3 10:cd09f7b7:::benchmark_data_smithi027_374610_object49865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e507)", 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:47:43.421454+0000", 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 127.491015979, 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.15223995900000001, 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 175995 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.989 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.421454+0000", 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.421454+0000", 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.421450+0000", 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999957 2024-08-29T15:49:50.990 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.421458+0000", 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.7320000000000004e-06 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.421469+0000", 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0866999999999999e-05 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.421474+0000", 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.9099999999999996e-06 2024-08-29T15:49:50.991 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.465796+0000", 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.044322255999999997 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.465883+0000", 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6920000000000001e-05 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3,4", 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.465968+0000", 2024-08-29T15:49:50.992 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.4302999999999996e-05 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.472096+0000", 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0061280659999999997 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.475420+0000", 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0033244350000000002 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.993 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.478580+0000", 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.003160188 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.573582+0000", 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.095001594999999994 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:50.994 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.573617+0000", 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.5472000000000002e-05 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:47:43.573694+0000", 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6334000000000005e-05 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.995 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:214881 10.6 10:6e6dd135:::benchmark_data_smithi027_374610_object88751:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:27.290613+0000", 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.621857195999993, 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.102696543, 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 214881 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:50.996 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290613+0000", 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290613+0000", 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:50.997 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290611+0000", 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290617+0000", 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.2149999999999998e-06 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:50.998 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290631+0000", 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3103e-05 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290638+0000", 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.0600000000000002e-06 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290684+0000", 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.621e-05 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:50.999 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290836+0000", 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000151855 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.290955+0000", 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000119175 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:51.000 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.295123+0000", 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0041686580000000004 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.296353+0000", 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0012296620000000001 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.393219+0000", 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.096865484000000002 2024-08-29T15:49:51.001 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.393256+0000", 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7531999999999999e-05 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.393309+0000", 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.2988000000000002e-05 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.002 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:214896 10.6 10:635380d4:::benchmark_data_smithi027_374610_object88766:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:27.303564+0000", 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 83.608905862, 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.102742351, 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 214896 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.003 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.004 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.004 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.004 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303564+0000", 2024-08-29T15:49:51.004 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.004 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.004 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.005 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.005 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303564+0000", 2024-08-29T15:49:51.005 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.006 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.006 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.006 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.006 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303562+0000", 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999976 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303570+0000", 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.8949999999999998e-06 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.007 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303586+0000", 2024-08-29T15:49:51.008 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6115000000000001e-05 2024-08-29T15:49:51.008 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.008 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.008 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.008 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303593+0000", 2024-08-29T15:49:51.008 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.8390000000000001e-06 2024-08-29T15:49:51.008 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303639+0000", 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5988000000000001e-05 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303778+0000", 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00013986399999999999 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 0,4", 2024-08-29T15:49:51.009 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.303904+0000", 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00012593200000000001 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.307379+0000", 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0034746339999999999 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.307437+0000", 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.838e-05 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.010 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.406213+0000", 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098776099000000006 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.406264+0000", 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.0600999999999999e-05 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:27.406306+0000", 2024-08-29T15:49:51.011 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.2259000000000003e-05 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:216710 10.9 10.ebd66199 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:29.392956+0000", 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 81.519513824000001, 2024-08-29T15:49:51.012 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95207151899999998, 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 216710 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.013 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.392956+0000", 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.392956+0000", 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.392954+0000", 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999981 2024-08-29T15:49:51.014 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.392962+0000", 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.486e-06 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.392978+0000", 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.6161999999999999e-05 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.015 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.392986+0000", 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6510000000000001e-06 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.393029+0000", 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.3492000000000003e-05 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:51.016 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.393037+0000", 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.6359999999999994e-06 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345013+0000", 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.95197632399999998 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345027+0000", 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4440999999999999e-05 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.017 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:216748 10.9 10.d09a5539 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:29.437179+0000", 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 81.475290662999996, 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.90789293900000001, 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.018 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 216748 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437179+0000", 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.019 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437179+0000", 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437178+0000", 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437184+0000", 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.5010000000000001e-06 2024-08-29T15:49:51.020 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437203+0000", 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.8757999999999999e-05 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437212+0000", 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.5709999999999998e-06 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.021 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437249+0000", 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.7339999999999998e-05 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.437258+0000", 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1689999999999998e-06 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345055+0000", 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.90779697800000003 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.022 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345072+0000", 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.7068e-05 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:216760 10.9 10.d023f269 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:29.450857+0000", 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 81.461612990999996, 2024-08-29T15:49:51.023 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.89425594500000005, 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 216760 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450857+0000", 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.024 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450857+0000", 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450856+0000", 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.025 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450862+0000", 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0929999999999999e-06 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450875+0000", 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3740000000000001e-05 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.026 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450882+0000", 2024-08-29T15:49:51.027 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.4690000000000002e-06 2024-08-29T15:49:51.027 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.027 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.027 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450922+0000", 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.9888000000000002e-05 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.450932+0000", 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.9969999999999993e-06 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.028 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345100+0000", 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.89416864399999996 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345113+0000", 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2373e-05 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.029 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:216773 10.9 10.7adba49 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:29.476911+0000", 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 81.435558603000004, 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86822907900000001, 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 216773 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.030 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476911+0000", 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476911+0000", 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476910+0000", 2024-08-29T15:49:51.031 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.999999 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476916+0000", 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.0569999999999998e-06 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476928+0000", 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2019e-05 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.032 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476936+0000", 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.4420000000000001e-06 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476980+0000", 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.4088000000000002e-05 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.476989+0000", 2024-08-29T15:49:51.033 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.8240000000000006e-06 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345132+0000", 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.86814352299999997 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.034 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345140+0000", 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.2169999999999994e-06 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.036 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:216777 10.9 10.53f7d309 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:29.486014+0000", 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 81.426455515000001, 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.85915556599999998, 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 216777 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.037 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486014+0000", 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486014+0000", 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.038 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486013+0000", 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486019+0000", 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6439999999999997e-06 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486031+0000", 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.2320000000000001e-05 2024-08-29T15:49:51.039 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486037+0000", 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.8209999999999998e-06 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486082+0000", 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.5278999999999998e-05 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:51.040 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.486091+0000", 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6719999999999993e-06 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345160+0000", 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.85906890800000002 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345170+0000", 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.0165e-05 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.041 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:216785 10.9 10.500b1289 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:29.514774+0000", 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 81.397695502999994, 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.83042473299999997, 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "reached pg", 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 216785 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.042 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514774+0000", 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514774+0000", 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514771+0000", 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999971 2024-08-29T15:49:51.043 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514786+0000", 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.4192999999999999e-05 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514819+0000", 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 3.3173000000000002e-05 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514834+0000", 2024-08-29T15:49:51.044 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.5119999999999999e-05 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514934+0000", 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000100252 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for peered", 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:29.514962+0000", 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.7719999999999999e-05 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.045 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345190+0000", 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.83022807600000004 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:30.345199+0000", 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 9.1360000000000003e-06 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.046 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.047 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.047 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.047 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.047 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:221105 10.3 10:c84fb406:::benchmark_data_smithi027_374610_object94975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:36.232030+0000", 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 74.680439653999997, 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11636664300000001, 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221105 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.048 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232030+0000", 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232030+0000", 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232029+0000", 2024-08-29T15:49:51.049 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232034+0000", 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.4689999999999996e-06 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232046+0000", 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.1895e-05 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.050 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232051+0000", 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.8829999999999997e-06 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232107+0000", 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 5.6301000000000002e-05 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232194+0000", 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.6960999999999996e-05 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.051 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.232276+0000", 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 8.1155000000000005e-05 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.236021+0000", 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0037453899999999999 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.236406+0000", 2024-08-29T15:49:51.052 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.00038514199999999999 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.348307+0000", 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.11190109600000001 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.348355+0000", 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.7679000000000002e-05 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.348397+0000", 2024-08-29T15:49:51.053 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1943999999999997e-05 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "description": "osd_op(client.47036.0:221124 10.3 10:c9ed7a10:::benchmark_data_smithi027_374610_object94994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e524)", 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "initiated_at": "2024-08-29T15:48:36.252310+0000", 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "age": 74.660160114999996, 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.103542063, 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "type_data": { 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "client_info": { 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "client": "client.47036", 2024-08-29T15:49:51.054 INFO:teuthology.orchestra.run.smithi171.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "tid": 221124 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "events": [ 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "initiated", 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252310+0000", 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "throttled", 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252310+0000", 2024-08-29T15:49:51.055 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "header_read", 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252308+0000", 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4294967295.9999986 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "all_read", 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252314+0000", 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.1940000000000003e-06 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.056 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "dispatched", 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252328+0000", 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.3217999999999999e-05 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "queued_for_pg", 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252334+0000", 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 6.686e-06 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "reached_pg", 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252376+0000", 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 4.1434999999999999e-05 2024-08-29T15:49:51.057 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "started", 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252484+0000", 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.000107849 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "waiting for subops from 2,3", 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.252557+0000", 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 7.3274000000000004e-05 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "op_commit", 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.256369+0000", 2024-08-29T15:49:51.058 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.0038118029999999999 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.257560+0000", 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.001191161 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.355806+0000", 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 0.098246489000000006 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.059 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "commit_sent", 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.355832+0000", 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 2.5612000000000001e-05 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: }, 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: { 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: "event": "done", 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: "time": "2024-08-29T15:48:36.355852+0000", 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: "duration": 1.9896999999999999e-05 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: } 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout: ] 2024-08-29T15:49:51.060 INFO:teuthology.orchestra.run.smithi171.stdout:} 2024-08-29T15:49:51.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4148831687' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:49:51.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[93921]: osdmap e551: 8 total, 8 up, 8 in 2024-08-29T15:49:51.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[93921]: 1.a scrub starts 2024-08-29T15:49:51.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[93921]: 1.a scrub ok 2024-08-29T15:49:51.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4148831687' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:49:51.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[98970]: osdmap e551: 8 total, 8 up, 8 in 2024-08-29T15:49:51.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[98970]: 1.a scrub starts 2024-08-29T15:49:51.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:51 smithi027 ceph-mon[98970]: 1.a scrub ok 2024-08-29T15:49:51.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-29T15:49:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4148831687' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-29T15:49:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:51 smithi171 ceph-mon[90031]: osdmap e551: 8 total, 8 up, 8 in 2024-08-29T15:49:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:51 smithi171 ceph-mon[90031]: 1.a scrub starts 2024-08-29T15:49:51.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:51 smithi171 ceph-mon[90031]: 1.a scrub ok 2024-08-29T15:49:51.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-29T15:49:52.208 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:52.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:52 smithi027 ceph-mon[98970]: pgmap v1915: 103 pgs: 1 active+undersized+remapped, 1 peering, 101 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 7.7 KiB/s rd, 23 KiB/s wr, 5 op/s; 97 KiB/s, 3 objects/s recovering 2024-08-29T15:49:52.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:52 smithi027 ceph-mon[93921]: pgmap v1915: 103 pgs: 1 active+undersized+remapped, 1 peering, 101 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 7.7 KiB/s rd, 23 KiB/s wr, 5 op/s; 97 KiB/s, 3 objects/s recovering 2024-08-29T15:49:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:52 smithi171 ceph-mon[90031]: pgmap v1915: 103 pgs: 1 active+undersized+remapped, 1 peering, 101 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 7.7 KiB/s rd, 23 KiB/s wr, 5 op/s; 97 KiB/s, 3 objects/s recovering 2024-08-29T15:49:52.882 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:53.318 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.0 to 1 (10000) 2024-08-29T15:49:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:53 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2008918135' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:49:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:53 smithi027 ceph-mon[98970]: 1.8 scrub starts 2024-08-29T15:49:53.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:53 smithi027 ceph-mon[98970]: 1.8 scrub ok 2024-08-29T15:49:53.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:53 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2008918135' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:49:53.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:53 smithi027 ceph-mon[93921]: 1.8 scrub starts 2024-08-29T15:49:53.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:53 smithi027 ceph-mon[93921]: 1.8 scrub ok 2024-08-29T15:49:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:53 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2008918135' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:49:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:53 smithi171 ceph-mon[90031]: 1.8 scrub starts 2024-08-29T15:49:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:53 smithi171 ceph-mon[90031]: 1.8 scrub ok 2024-08-29T15:49:54.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:54 smithi027 ceph-mon[98970]: pgmap v1916: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:49:54.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:54 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2008918135' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:49:54.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:54 smithi027 ceph-mon[98970]: osdmap e552: 8 total, 8 up, 8 in 2024-08-29T15:49:54.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:54 smithi027 ceph-mon[93921]: pgmap v1916: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:49:54.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:54 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2008918135' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:49:54.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:54 smithi027 ceph-mon[93921]: osdmap e552: 8 total, 8 up, 8 in 2024-08-29T15:49:54.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 0 1 2024-08-29T15:49:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:54 smithi171 ceph-mon[90031]: pgmap v1916: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:49:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:54 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2008918135' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:49:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:54 smithi171 ceph-mon[90031]: osdmap e552: 8 total, 8 up, 8 in 2024-08-29T15:49:55.665 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:49:55.666 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:49:55.666 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2024-08-29T15:49:55.666 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:49:56.769 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:56 smithi171 ceph-mon[90031]: pgmap v1918: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:49:56.804 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:56 smithi027 ceph-mon[98970]: pgmap v1918: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:49:56.804 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:56 smithi027 ceph-mon[93921]: pgmap v1918: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-29T15:49:56.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-29T15:49:56.955 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:57.333 INFO:teuthology.orchestra.run.smithi027.stderr:nodeep-scrub is unset 2024-08-29T15:49:57.333 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:57 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2484880478' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:57.334 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:57 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2484880478' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:57 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2484880478' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-29T15:49:58.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:58 smithi027 ceph-mon[93921]: pgmap v1919: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:49:58.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2484880478' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:58.616 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:58 smithi027 ceph-mon[93921]: osdmap e553: 8 total, 8 up, 8 in 2024-08-29T15:49:58.617 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:58 smithi027 ceph-mon[98970]: pgmap v1919: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:49:58.617 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2484880478' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:58.617 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:58 smithi027 ceph-mon[98970]: osdmap e553: 8 total, 8 up, 8 in 2024-08-29T15:49:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:58 smithi171 ceph-mon[90031]: pgmap v1919: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:49:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2484880478' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-29T15:49:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:58 smithi171 ceph-mon[90031]: osdmap e553: 8 total, 8 up, 8 in 2024-08-29T15:49:58.965 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:49:59.217 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:49:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:49:59] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.51.0" 2024-08-29T15:49:59.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:49:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:59.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:49:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:49:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:49:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:00.466 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:00 smithi027 ceph-mon[98970]: pgmap v1921: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:00.466 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:00 smithi027 ceph-mon[98970]: overall HEALTH_OK 2024-08-29T15:50:00.466 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:00 smithi027 ceph-mon[93921]: pgmap v1921: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:00.467 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:00 smithi027 ceph-mon[93921]: overall HEALTH_OK 2024-08-29T15:50:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:00 smithi171 ceph-mon[90031]: pgmap v1921: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:00 smithi171 ceph-mon[90031]: overall HEALTH_OK 2024-08-29T15:50:01.133 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:50:01.133 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2024-08-29T15:50:01.133 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2024-08-29T15:50:01.133 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2024-08-29T15:50:01.133 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:50:01.350 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-08-29T15:50:01.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:01 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1503041527' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:50:01.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:01 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1503041527' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:50:01.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:01 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1503041527' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-29T15:50:02.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-29T15:50:02.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:02 smithi027 ceph-mon[98970]: pgmap v1922: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:02.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:02 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1503041527' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:50:02.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:02 smithi027 ceph-mon[98970]: osdmap e554: 8 total, 8 up, 8 in 2024-08-29T15:50:02.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:02 smithi027 ceph-mon[93921]: pgmap v1922: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:02.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:02 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1503041527' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:50:02.614 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:02 smithi027 ceph-mon[93921]: osdmap e554: 8 total, 8 up, 8 in 2024-08-29T15:50:02.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 1 1 2024-08-29T15:50:02.651 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:02 smithi171 ceph-mon[90031]: pgmap v1922: 103 pgs: 103 active+clean; 4.6 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:02 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1503041527' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-29T15:50:02.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:02 smithi171 ceph-mon[90031]: osdmap e554: 8 total, 8 up, 8 in 2024-08-29T15:50:03.394 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:04.524 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:04 smithi027 ceph-mon[93921]: pgmap v1924: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:04.525 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:04 smithi027 ceph-mon[98970]: pgmap v1924: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:04.553 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29420 9.0 e430/420 9:0e19c1e7:::benchmark_data_smithi027_341149_object29419:head v 430'1735, mlcod=430'1732)", 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.491631+0000", 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 595.05972168400001, 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35499817, 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.554 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491631+0000", 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491631+0000", 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491632+0000", 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2600000000000002e-07 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491652+0000", 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9599999999999999e-05 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491662+0000", 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0621e-05 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491669+0000", 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6159999999999998e-06 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491744+0000", 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5499000000000003e-05 2024-08-29T15:50:04.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.491764+0000", 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9318999999999999e-05 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846582+0000", 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35481835099999998 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846630+0000", 2024-08-29T15:50:04.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7438000000000002e-05 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29447 9.2 e430/420 9:4ae6b342:::benchmark_data_smithi027_341149_object29446:head v 430'1859, mlcod=430'1857)", 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.501916+0000", 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 595.04943722200005, 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34462358999999998, 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501916+0000", 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501916+0000", 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501917+0000", 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.156e-06 2024-08-29T15:50:04.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.501945+0000", 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8394e-05 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843686+0000", 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34174009900000002 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843712+0000", 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6857999999999999e-05 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844135+0000", 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042273499999999998 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844159+0000", 2024-08-29T15:50:04.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3467000000000001e-05 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846502+0000", 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023433460000000001 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846539+0000", 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7534999999999999e-05 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.563 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29456 9.6 e430/420 9:6f12b949:::benchmark_data_smithi027_341149_object29455:head v 430'1808, mlcod=430'1806)", 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.505599+0000", 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 595.04575405499997, 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34067662999999998, 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505599+0000", 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505599+0000", 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505600+0000", 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4589999999999999e-06 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.505650+0000", 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9872e-05 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843666+0000", 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33801598399999999 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843688+0000", 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2158e-05 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843781+0000", 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2003000000000007e-05 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843809+0000", 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8126000000000001e-05 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846234+0000", 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024256030000000001 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.846276+0000", 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1424999999999997e-05 2024-08-29T15:50:04.568 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46778.0:29465 9.0 e430/420 9:098160e2:::benchmark_data_smithi027_341149_object29464:head v 430'1736, mlcod=430'1733)", 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:40:09.509005+0000", 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 595.04234768000003, 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33833135600000003, 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.569 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509005+0000", 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509005+0000", 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509007+0000", 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.297e-06 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.509033+0000", 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6035000000000001e-05 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843695+0000", 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.334662401 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.843705+0000", 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0069e-05 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844810+0000", 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011047920000000001 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.844839+0000", 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9156999999999999e-05 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.847284+0000", 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024448899999999999 2024-08-29T15:50:04.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:40:09.847337+0000", 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2714999999999999e-05 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51080 10.9 e486/480 10:9f1d1fe6:::benchmark_data_smithi027_374610_object51079:head v 486'3226, mlcod=486'3224)", 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.742142+0000", 2024-08-29T15:50:04.574 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 261.80921068700002, 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31543603799999997, 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742142+0000", 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742142+0000", 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742143+0000", 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8200000000000008e-07 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742180+0000", 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6165999999999999e-05 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742196+0000", 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6762999999999999e-05 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742209+0000", 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2422999999999999e-05 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742270+0000", 2024-08-29T15:50:04.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1316999999999996e-05 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742288+0000", 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8278999999999999e-05 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057489+0000", 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31520077899999999 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057578+0000", 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9328999999999998e-05 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51084 10.e e486/480 10:78604a72:::benchmark_data_smithi027_374610_object51083:head v 486'3245, mlcod=486'3243)", 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.742236+0000", 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 261.80911659399999, 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31535221400000002, 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.579 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742236+0000", 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742236+0000", 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742237+0000", 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0600000000000001e-07 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742257+0000", 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0392999999999999e-05 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742265+0000", 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4410000000000004e-06 2024-08-29T15:50:04.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742268+0000", 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3749999999999999e-06 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742336+0000", 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7867999999999995e-05 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053962+0000", 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.311625861 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057430+0000", 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003468526 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057589+0000", 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015844399999999999 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.583 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51089 10.5 e486/480 10:a181d637:::benchmark_data_smithi027_374610_object51088:head v 486'3265, mlcod=486'3263)", 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.742479+0000", 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 261.80887388399998, 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31610776899999998, 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742479+0000", 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742479+0000", 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742480+0000", 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6000000000000003e-07 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.742500+0000", 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.02e-05 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053967+0000", 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31146699700000002 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053986+0000", 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8902e-05 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054117+0000", 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013076799999999999 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054142+0000", 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5327e-05 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.058540+0000", 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043983599999999996 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.058587+0000", 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6554999999999997e-05 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51088 10.e e486/480 10:7d47e90e:::benchmark_data_smithi027_374610_object51087:head v 486'3246, mlcod=486'3243)", 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 261.80712742200001, 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31355423500000001, 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.590 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744226+0000", 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7000000000000004e-07 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.744246+0000", 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9559e-05 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053974+0000", 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30972837800000003 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053986+0000", 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1372999999999999e-05 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.055839+0000", 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001853146 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.055861+0000", 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.181e-05 2024-08-29T15:50:04.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057753+0000", 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018923029999999999 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.057780+0000", 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6896000000000001e-05 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.594 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.47036.0:51095 10.a e486/480 10:596831c7:::benchmark_data_smithi027_374610_object51094:head v 486'3255, mlcod=486'3253)", 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:45:42.746000+0000", 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 261.80535346599999, 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31383123400000001, 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746000+0000", 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746000+0000", 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746001+0000", 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.083e-06 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:42.746064+0000", 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2955999999999998e-05 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053983+0000", 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30791897299999998 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.053994+0000", 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1012000000000001e-05 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054517+0000", 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00052327500000000002 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.054536+0000", 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8861000000000001e-05 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.059775+0000", 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052396079999999998 2024-08-29T15:50:04.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:45:43.059831+0000", 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5466000000000001e-05 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135092 10.a 10:57479ccb:::benchmark_data_smithi027_374610_object8962:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.367206+0000", 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 186.18414754700001, 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1403076480000001, 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.599 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135092 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367206+0000", 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367206+0000", 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367207+0000", 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.807e-06 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367214+0000", 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1650000000000003e-06 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367229+0000", 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5744999999999999e-05 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367239+0000", 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0156000000000001e-05 2024-08-29T15:50:04.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367390+0000", 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000150192 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367402+0000", 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2660000000000001e-05 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.505193+0000", 2024-08-29T15:50:04.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.137790463 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.507513+0000", 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023204599999999999 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135094 10.a 10:50933fa0:::benchmark_data_smithi027_374610_object8964:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.367263+0000", 2024-08-29T15:50:04.604 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 186.18408970900001, 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.141313501, 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135094 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367263+0000", 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367263+0000", 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367264+0000", 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6199999999999997e-07 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367267+0000", 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0350000000000002e-06 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367276+0000", 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.585e-06 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367282+0000", 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3949999999999998e-06 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367451+0000", 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016907700000000001 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367459+0000", 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0430000000000008e-06 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.508485+0000", 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1410257660000001 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.508577+0000", 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1938e-05 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135108 10.a 10:5a0c44de:::benchmark_data_smithi027_374610_object8978:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.367293+0000", 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 186.18406035199999, 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.167945273, 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135108 2024-08-29T15:50:04.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367293+0000", 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367293+0000", 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367293+0000", 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7399999999999998e-07 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367296+0000", 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6120000000000001e-06 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367305+0000", 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6999999999999997e-06 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367310+0000", 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8660000000000002e-06 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367486+0000", 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017537600000000001 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367493+0000", 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2169999999999997e-06 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.508608+0000", 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.141114867 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509372+0000", 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00076374500000000003 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.614 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5,6", 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.509789+0000", 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041782100000000002 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.523915+0000", 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014125418000000001 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.523930+0000", 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4836e-05 2024-08-29T15:50:04.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.523965+0000", 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5731000000000001e-05 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.535062+0000", 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011096586 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.535123+0000", 2024-08-29T15:50:04.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0928999999999999e-05 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.535238+0000", 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011509499999999999 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135109 10.a 10:5c587d32:::benchmark_data_smithi027_374610_object8979:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.367319+0000", 2024-08-29T15:50:04.617 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 186.18403361, 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.171138918, 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135109 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367319+0000", 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367319+0000", 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367320+0000", 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8400000000000005e-07 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367322+0000", 2024-08-29T15:50:04.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2230000000000001e-06 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367330+0000", 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9629999999999998e-06 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367336+0000", 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5910000000000003e-06 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367575+0000", 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023936000000000001 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367583+0000", 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0330000000000008e-06 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.517370+0000", 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.149787358 2024-08-29T15:50:04.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.517478+0000", 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107839 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5,6", 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.517598+0000", 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119846 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.527790+0000", 2024-08-29T15:50:04.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010191723999999999 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.531565+0000", 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003774941 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.531622+0000", 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6867000000000003e-05 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.623 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.538266+0000", 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066444440000000002 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.538334+0000", 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8127999999999996e-05 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.538458+0000", 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124117 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.624 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:135123 10.a 10:5e1efbb8:::benchmark_data_smithi027_374610_object8993:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:46:58.367510+0000", 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 186.18384283200001, 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.183604221, 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 135123 2024-08-29T15:50:04.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367510+0000", 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367510+0000", 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367511+0000", 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5700000000000002e-07 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367514+0000", 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2109999999999998e-06 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367523+0000", 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8769999999999999e-06 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367529+0000", 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2319999999999998e-06 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367609+0000", 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9334999999999999e-05 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:58.367616+0000", 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2139999999999999e-06 2024-08-29T15:50:04.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.524176+0000", 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1565602669999999 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.524292+0000", 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011567 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5,6", 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.524423+0000", 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131631 2024-08-29T15:50:04.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.531583+0000", 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071599899999999998 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.531805+0000", 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022148599999999999 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.534049+0000", 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022444510000000002 2024-08-29T15:50:04.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.550940+0000", 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016890742 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.551001+0000", 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1227000000000002e-05 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:46:59.551114+0000", 2024-08-29T15:50:04.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113231 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:216710 10.9 10:99866bd7:::benchmark_data_smithi027_374610_object90580:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e521)", 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:30.345408+0000", 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.205945004, 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.156228915, 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.632 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 216710 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345408+0000", 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345408+0000", 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345410+0000", 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.485e-06 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345415+0000", 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1800000000000004e-06 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345428+0000", 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3536999999999999e-05 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345438+0000", 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5389999999999996e-06 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.346558+0000", 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011204959999999999 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.346568+0000", 2024-08-29T15:50:04.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7059999999999999e-06 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.490417+0000", 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.143849363 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.492723+0000", 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023060630000000001 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.492836+0000", 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011217099999999999 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501113+0000", 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082776269999999992 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501258+0000", 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000144504 2024-08-29T15:50:04.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501409+0000", 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015120800000000001 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501462+0000", 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2787000000000001e-05 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501518+0000", 2024-08-29T15:50:04.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6026999999999997e-05 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501637+0000", 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119222 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:216748 10.9 10:9caa590b:::benchmark_data_smithi027_374610_object90618:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e521)", 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:30.345459+0000", 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.205893840000002, 2024-08-29T15:50:04.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.156588881, 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 216748 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345459+0000", 2024-08-29T15:50:04.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345459+0000", 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345460+0000", 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3399999999999999e-07 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345463+0000", 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8150000000000002e-06 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345471+0000", 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9500000000000001e-06 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345477+0000", 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4049999999999997e-06 2024-08-29T15:50:04.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.349516+0000", 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040394080000000004 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.349524+0000", 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9070000000000007e-06 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.495474+0000", 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1459496600000001 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.495601+0000", 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012750399999999999 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.495721+0000", 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119295 2024-08-29T15:50:04.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501427+0000", 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057063000000000001 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501734+0000", 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00030694700000000001 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501810+0000", 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5671000000000001e-05 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501872+0000", 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2599000000000002e-05 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501936+0000", 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4145000000000003e-05 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502048+0000", 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111741 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:216760 10.9 10:964fc40b:::benchmark_data_smithi027_374610_object90630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e521)", 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:30.345486+0000", 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.205866626000002, 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.156994439, 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.647 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 216760 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345486+0000", 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345486+0000", 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345487+0000", 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3500000000000002e-07 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345489+0000", 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.21e-06 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345496+0000", 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5069999999999997e-06 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.345501+0000", 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3120000000000001e-06 2024-08-29T15:50:04.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.349545+0000", 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040435849999999997 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.349550+0000", 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8869999999999997e-06 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.498427+0000", 2024-08-29T15:50:04.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.148876767 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.498545+0000", 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011755400000000001 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.498678+0000", 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133096 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.501651+0000", 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029732959999999998 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502284+0000", 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00063256100000000002 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502347+0000", 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3288999999999997e-05 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502390+0000", 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3108999999999997e-05 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502423+0000", 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2558000000000002e-05 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502481+0000", 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8273000000000003e-05 2024-08-29T15:50:04.654 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:216773 10.9 10:925db5e0:::benchmark_data_smithi027_374610_object90643:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e521)", 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:30.347400+0000", 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.203953119000005, 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.163111988, 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.655 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 216773 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347400+0000", 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347400+0000", 2024-08-29T15:50:04.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347401+0000", 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8100000000000001e-07 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347405+0000", 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6959999999999998e-06 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347416+0000", 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1823e-05 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347425+0000", 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5019999999999992e-06 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.349570+0000", 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021450060000000001 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.349576+0000", 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8610000000000003e-06 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502527+0000", 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.152951439 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502652+0000", 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124533 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.502780+0000", 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127828 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.509695+0000", 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069152160000000001 2024-08-29T15:50:04.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.509708+0000", 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2754e-05 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.509759+0000", 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1044000000000001e-05 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.509797+0000", 2024-08-29T15:50:04.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8172e-05 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510337+0000", 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00054011499999999997 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510512+0000", 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017511800000000001 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.662 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:216777 10.9 10:90cbefca:::benchmark_data_smithi027_374610_object90647:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e521)", 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:30.347439+0000", 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.203914269999999, 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.166847637, 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.663 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 216777 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347439+0000", 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347439+0000", 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.347439+0000", 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7199999999999999e-07 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350127+0000", 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002688111 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350144+0000", 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6599000000000002e-05 2024-08-29T15:50:04.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350152+0000", 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0069999999999997e-06 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350480+0000", 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00032813200000000002 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350489+0000", 2024-08-29T15:50:04.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.969e-06 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.505928+0000", 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.155438696 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.506038+0000", 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011007599999999999 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:50:04.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.506147+0000", 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010888899999999999 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510553+0000", 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0044060979999999998 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510588+0000", 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4827999999999999e-05 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510622+0000", 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3986999999999998e-05 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.513862+0000", 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032400799999999998 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.514171+0000", 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00030903799999999998 2024-08-29T15:50:04.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.514286+0000", 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115555 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.47036.0:216785 10.9 10:9148d00a:::benchmark_data_smithi027_374610_object90655:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e521)", 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2024-08-29T15:48:30.350172+0000", 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 94.201181310999999, 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1748685560000001, 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "continuous": false, 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.47036", 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.27:0/4160119495", 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 216785 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2024-08-29T15:50:04.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2024-08-29T15:50:04.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350172+0000", 2024-08-29T15:50:04.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2024-08-29T15:50:04.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350172+0000", 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350173+0000", 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.139e-06 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350176+0000", 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9969999999999999e-06 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350183+0000", 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8170000000000001e-06 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350188+0000", 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5639999999999996e-06 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350516+0000", 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00032724499999999999 2024-08-29T15:50:04.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:30.350522+0000", 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9040000000000003e-06 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510649+0000", 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.16012626 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510753+0000", 2024-08-29T15:50:04.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0001047 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,5,7", 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.510869+0000", 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011535300000000001 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.524737+0000", 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013867772 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2024-08-29T15:50:04.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.524750+0000", 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3192e-05 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.524784+0000", 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4471000000000003e-05 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.524848+0000", 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4287000000000003e-05 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.524921+0000", 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2566999999999998e-05 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2024-08-29T15:48:31.525040+0000", 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119288 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-08-29T15:50:04.678 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-08-29T15:50:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:04 smithi171 ceph-mon[90031]: pgmap v1924: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:06.557 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:06 smithi027 ceph-mon[93921]: pgmap v1925: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:06.558 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:06 smithi027 ceph-mon[98970]: pgmap v1925: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:06 smithi171 ceph-mon[90031]: pgmap v1925: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:08.373 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.1 to 1 (10000) 2024-08-29T15:50:08.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:08 smithi027 ceph-mon[98970]: pgmap v1926: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:08.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:08 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3018772711' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:08.734 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:08 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:08.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:08 smithi027 ceph-mon[93921]: pgmap v1926: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:08.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:08 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3018772711' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:08.734 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:08 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:08 smithi171 ceph-mon[90031]: pgmap v1926: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:08 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3018772711' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:08 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:09.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 1 1 2024-08-29T15:50:09.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:50:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:50:09] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.51.0" 2024-08-29T15:50:09.374 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:09.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:09 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:50:09.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:09 smithi027 ceph-mon[93921]: osdmap e555: 8 total, 8 up, 8 in 2024-08-29T15:50:09.649 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:09 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:50:09.649 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:09 smithi027 ceph-mon[98970]: osdmap e555: 8 total, 8 up, 8 in 2024-08-29T15:50:09.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:09 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:50:09.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:09 smithi171 ceph-mon[90031]: osdmap e555: 8 total, 8 up, 8 in 2024-08-29T15:50:10.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:10 smithi027 ceph-mon[98970]: pgmap v1928: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:10.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:10 smithi027 ceph-mon[93921]: pgmap v1928: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:10 smithi171 ceph-mon[90031]: pgmap v1928: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:12.380 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-08-29T15:50:12.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:12 smithi027 ceph-mon[98970]: pgmap v1929: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:12.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:12 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2667210226' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:12.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:12 smithi027 ceph-mon[93921]: pgmap v1929: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:12.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:12 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2667210226' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:12 smithi171 ceph-mon[90031]: pgmap v1929: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:12 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2667210226' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-29T15:50:13.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 3 1 2024-08-29T15:50:13.369 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:13.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:13 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2667210226' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:50:13.746 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:13 smithi027 ceph-mon[93921]: osdmap e556: 8 total, 8 up, 8 in 2024-08-29T15:50:13.746 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:13 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2667210226' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:50:13.746 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:13 smithi027 ceph-mon[98970]: osdmap e556: 8 total, 8 up, 8 in 2024-08-29T15:50:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:13 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2667210226' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-29T15:50:13.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:13 smithi171 ceph-mon[90031]: osdmap e556: 8 total, 8 up, 8 in 2024-08-29T15:50:14.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:14 smithi027 ceph-mon[93921]: pgmap v1931: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:14 smithi027 ceph-mon[98970]: pgmap v1931: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:14.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:14 smithi171 ceph-mon[90031]: pgmap v1931: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:15.405 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.3 to 1 (10000) 2024-08-29T15:50:15.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:15 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/868821491' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:15.813 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:15 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/868821491' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:15.813 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:15 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/868821491' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:16.024 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 3 1 2024-08-29T15:50:16.232 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:16.604 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:16 smithi027 ceph-mon[98970]: pgmap v1932: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:16.604 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:16 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/868821491' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:50:16.604 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:16 smithi027 ceph-mon[98970]: osdmap e557: 8 total, 8 up, 8 in 2024-08-29T15:50:16.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:16 smithi027 ceph-mon[93921]: pgmap v1932: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:16.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:16 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/868821491' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:50:16.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:16 smithi027 ceph-mon[93921]: osdmap e557: 8 total, 8 up, 8 in 2024-08-29T15:50:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:16 smithi171 ceph-mon[90031]: pgmap v1932: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:16 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/868821491' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:50:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:16 smithi171 ceph-mon[90031]: osdmap e557: 8 total, 8 up, 8 in 2024-08-29T15:50:18.413 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-08-29T15:50:18.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:18 smithi027 ceph-mon[93921]: pgmap v1934: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:18.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:18 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3154718411' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:18.692 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:18 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:18.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:18 smithi027 ceph-mon[98970]: pgmap v1934: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:18.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:18 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3154718411' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:18.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:18 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:18 smithi171 ceph-mon[90031]: pgmap v1934: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:18 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3154718411' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:18 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-29T15:50:19.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 4 1 2024-08-29T15:50:19.230 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:50:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:50:19] "GET /metrics HTTP/1.1" 200 34144 "" "Prometheus/2.51.0" 2024-08-29T15:50:19.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:19 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:50:19.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:19 smithi027 ceph-mon[98970]: osdmap e558: 8 total, 8 up, 8 in 2024-08-29T15:50:19.696 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:19 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:50:19.696 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:19 smithi027 ceph-mon[93921]: osdmap e558: 8 total, 8 up, 8 in 2024-08-29T15:50:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:19 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-29T15:50:19.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:19 smithi171 ceph-mon[90031]: osdmap e558: 8 total, 8 up, 8 in 2024-08-29T15:50:20.668 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:20 smithi027 ceph-mon[98970]: pgmap v1936: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:20.668 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:20 smithi027 ceph-mon[93921]: pgmap v1936: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:20 smithi171 ceph-mon[90031]: pgmap v1936: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:21.430 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.4 to 1 (10000) 2024-08-29T15:50:21.750 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:21 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2423556536' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:21.750 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:21 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2423556536' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:21.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:21 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2423556536' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:22.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 4 1 2024-08-29T15:50:22.296 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:22.671 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:22 smithi027 ceph-mon[93921]: pgmap v1937: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:22.671 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:22 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2423556536' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:50:22.671 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:22 smithi027 ceph-mon[93921]: osdmap e559: 8 total, 8 up, 8 in 2024-08-29T15:50:22.671 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:22 smithi027 ceph-mon[98970]: pgmap v1937: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:22.671 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:22 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2423556536' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:50:22.671 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:22 smithi027 ceph-mon[98970]: osdmap e559: 8 total, 8 up, 8 in 2024-08-29T15:50:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:22 smithi171 ceph-mon[90031]: pgmap v1937: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:22.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:22 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2423556536' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:50:22.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:22 smithi171 ceph-mon[90031]: osdmap e559: 8 total, 8 up, 8 in 2024-08-29T15:50:24.438 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-08-29T15:50:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:24 smithi171 ceph-mon[90031]: pgmap v1939: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:24 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3062442426' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:24 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:24.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:24 smithi027 ceph-mon[93921]: pgmap v1939: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:24.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:24 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3062442426' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:24.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:24 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:24.780 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:24 smithi027 ceph-mon[98970]: pgmap v1939: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:24.780 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:24 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3062442426' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:24.780 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:24 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-29T15:50:25.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 6 1 2024-08-29T15:50:25.338 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:25 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:50:25.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:25 smithi171 ceph-mon[90031]: osdmap e560: 8 total, 8 up, 8 in 2024-08-29T15:50:25.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:25 smithi171 ceph-mon[90031]: osdmap e561: 8 total, 8 up, 8 in 2024-08-29T15:50:25.787 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:25 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:50:25.788 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:25 smithi027 ceph-mon[93921]: osdmap e560: 8 total, 8 up, 8 in 2024-08-29T15:50:25.788 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:25 smithi027 ceph-mon[93921]: osdmap e561: 8 total, 8 up, 8 in 2024-08-29T15:50:25.788 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:25 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-29T15:50:25.788 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:25 smithi027 ceph-mon[98970]: osdmap e560: 8 total, 8 up, 8 in 2024-08-29T15:50:25.788 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:25 smithi027 ceph-mon[98970]: osdmap e561: 8 total, 8 up, 8 in 2024-08-29T15:50:26.727 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:26 smithi027 ceph-mon[93921]: pgmap v1941: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:26.727 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:26 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-08-29T15:50:26.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:26 smithi027 ceph-mon[98970]: pgmap v1941: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:26.728 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:26 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-08-29T15:50:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:26 smithi171 ceph-mon[90031]: pgmap v1941: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:26 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-08-29T15:50:27.461 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.6 to 1 (10000) 2024-08-29T15:50:27.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:27 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1161560124' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:27.733 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:27 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:27.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:27 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1161560124' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:27.733 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:27 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:27 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1161560124' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:27.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:27 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:28.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 6 1 2024-08-29T15:50:28.293 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:28.719 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:28 smithi027 ceph-mon[93921]: pgmap v1943: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:28.719 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:28 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:50:28.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:28 smithi027 ceph-mon[93921]: osdmap e562: 8 total, 8 up, 8 in 2024-08-29T15:50:28.720 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:28 smithi027 ceph-mon[98970]: pgmap v1943: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:28.720 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:28 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:50:28.720 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:28 smithi027 ceph-mon[98970]: osdmap e562: 8 total, 8 up, 8 in 2024-08-29T15:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:28 smithi171 ceph-mon[90031]: pgmap v1943: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:28 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:50:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:28 smithi171 ceph-mon[90031]: osdmap e562: 8 total, 8 up, 8 in 2024-08-29T15:50:29.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:50:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:50:29] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.51.0" 2024-08-29T15:50:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:30.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:30.019 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:30.019 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:30.020 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:30.020 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:30.749 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-08-29T15:50:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:30 smithi171 ceph-mon[90031]: pgmap v1945: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3174168891' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:31.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:30 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:30 smithi027 ceph-mon[98970]: pgmap v1945: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3174168891' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:31.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:30 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:31.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:30 smithi027 ceph-mon[93921]: pgmap v1945: 103 pgs: 6 peering, 97 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:31.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3174168891' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:31.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:30 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-29T15:50:31.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 7 1 2024-08-29T15:50:31.745 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:50:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[98970]: osdmap e563: 8 total, 8 up, 8 in 2024-08-29T15:50:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[98970]: pgmap v1947: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:31.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-29T15:50:31.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:50:31.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[93921]: osdmap e563: 8 total, 8 up, 8 in 2024-08-29T15:50:31.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[93921]: pgmap v1947: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:31.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:31 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-29T15:50:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:31 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-29T15:50:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:31 smithi171 ceph-mon[90031]: osdmap e563: 8 total, 8 up, 8 in 2024-08-29T15:50:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:31 smithi171 ceph-mon[90031]: pgmap v1947: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:32.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:31 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-29T15:50:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:32 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-29T15:50:33.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:32 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:50:33.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:32 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-29T15:50:33.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:32 smithi171 ceph-mon[90031]: osdmap e564: 8 total, 8 up, 8 in 2024-08-29T15:50:33.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:32 smithi171 ceph-mon[90031]: 1.6 scrub starts 2024-08-29T15:50:33.018 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:32 smithi171 ceph-mon[90031]: 1.6 scrub ok 2024-08-29T15:50:33.024 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[93921]: osdmap e564: 8 total, 8 up, 8 in 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[93921]: 1.6 scrub starts 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[93921]: 1.6 scrub ok 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[98970]: osdmap e564: 8 total, 8 up, 8 in 2024-08-29T15:50:33.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[98970]: 1.6 scrub starts 2024-08-29T15:50:33.026 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:32 smithi027 ceph-mon[98970]: 1.6 scrub ok 2024-08-29T15:50:33.766 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.7 to 1 (10000) 2024-08-29T15:50:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[93921]: osdmap e565: 8 total, 8 up, 8 in 2024-08-29T15:50:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[93921]: 1.f scrub starts 2024-08-29T15:50:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[93921]: 1.f scrub ok 2024-08-29T15:50:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[93921]: pgmap v1950: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:34.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3428764978' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[98970]: osdmap e565: 8 total, 8 up, 8 in 2024-08-29T15:50:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[98970]: 1.f scrub starts 2024-08-29T15:50:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[98970]: 1.f scrub ok 2024-08-29T15:50:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[98970]: pgmap v1950: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:34.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3428764978' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:33 smithi171 ceph-mon[90031]: osdmap e565: 8 total, 8 up, 8 in 2024-08-29T15:50:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:33 smithi171 ceph-mon[90031]: 1.f scrub starts 2024-08-29T15:50:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:33 smithi171 ceph-mon[90031]: 1.f scrub ok 2024-08-29T15:50:34.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:33 smithi171 ceph-mon[90031]: pgmap v1950: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:34.268 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3428764978' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:34.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 7 1 2024-08-29T15:50:34.743 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3428764978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:50:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[98970]: osdmap e566: 8 total, 8 up, 8 in 2024-08-29T15:50:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[98970]: 1.7 scrub starts 2024-08-29T15:50:35.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[98970]: 1.7 scrub ok 2024-08-29T15:50:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3428764978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:50:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[93921]: osdmap e566: 8 total, 8 up, 8 in 2024-08-29T15:50:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[93921]: 1.7 scrub starts 2024-08-29T15:50:35.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:34 smithi027 ceph-mon[93921]: 1.7 scrub ok 2024-08-29T15:50:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3428764978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:50:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:34 smithi171 ceph-mon[90031]: osdmap e566: 8 total, 8 up, 8 in 2024-08-29T15:50:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:34 smithi171 ceph-mon[90031]: 1.7 scrub starts 2024-08-29T15:50:35.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:34 smithi171 ceph-mon[90031]: 1.7 scrub ok 2024-08-29T15:50:35.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:35 smithi027 ceph-mon[93921]: pgmap v1952: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:35.959 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:35 smithi027 ceph-mon[98970]: pgmap v1952: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:36.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:35 smithi171 ceph-mon[90031]: pgmap v1952: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:36.791 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-08-29T15:50:37.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:36 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1180478999' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:37.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:36 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:37.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:36 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1180478999' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:37.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:36 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:36 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1180478999' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:37.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:36 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-29T15:50:37.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 5 1 2024-08-29T15:50:37.716 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:37.870 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:37 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:50:37.870 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:37 smithi027 ceph-mon[93921]: osdmap e567: 8 total, 8 up, 8 in 2024-08-29T15:50:37.870 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:37 smithi027 ceph-mon[93921]: pgmap v1954: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:37.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:37 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:50:37.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:37 smithi027 ceph-mon[98970]: osdmap e567: 8 total, 8 up, 8 in 2024-08-29T15:50:37.871 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:37 smithi027 ceph-mon[98970]: pgmap v1954: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:38.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:37 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-29T15:50:38.168 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:37 smithi171 ceph-mon[90031]: osdmap e567: 8 total, 8 up, 8 in 2024-08-29T15:50:38.169 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:37 smithi171 ceph-mon[90031]: pgmap v1954: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:38.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:38 smithi027 ceph-mon[98970]: osdmap e568: 8 total, 8 up, 8 in 2024-08-29T15:50:38.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:38 smithi027 ceph-mon[93921]: osdmap e568: 8 total, 8 up, 8 in 2024-08-29T15:50:39.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:38 smithi171 ceph-mon[90031]: osdmap e568: 8 total, 8 up, 8 in 2024-08-29T15:50:39.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:50:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:50:39] "GET /metrics HTTP/1.1" 200 34143 "" "Prometheus/2.51.0" 2024-08-29T15:50:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:39 smithi027 ceph-mon[93921]: pgmap v1956: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:40.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:39 smithi027 ceph-mon[93921]: 1.9 scrub starts 2024-08-29T15:50:40.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:39 smithi027 ceph-mon[93921]: 1.9 scrub ok 2024-08-29T15:50:40.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:39 smithi027 ceph-mon[98970]: pgmap v1956: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:40.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:39 smithi027 ceph-mon[98970]: 1.9 scrub starts 2024-08-29T15:50:40.069 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:39 smithi027 ceph-mon[98970]: 1.9 scrub ok 2024-08-29T15:50:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:39 smithi171 ceph-mon[90031]: pgmap v1956: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:39 smithi171 ceph-mon[90031]: 1.9 scrub starts 2024-08-29T15:50:40.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:39 smithi171 ceph-mon[90031]: 1.9 scrub ok 2024-08-29T15:50:40.803 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.5 to 1 (10000) 2024-08-29T15:50:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3413314130' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[98970]: 1.0 scrub starts 2024-08-29T15:50:41.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[98970]: 1.0 scrub ok 2024-08-29T15:50:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3413314130' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[93921]: 1.0 scrub starts 2024-08-29T15:50:41.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:40 smithi027 ceph-mon[93921]: 1.0 scrub ok 2024-08-29T15:50:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:40 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3413314130' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:40 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:40 smithi171 ceph-mon[90031]: 1.0 scrub starts 2024-08-29T15:50:41.267 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:40 smithi171 ceph-mon[90031]: 1.0 scrub ok 2024-08-29T15:50:41.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 5 1 2024-08-29T15:50:41.779 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:41 smithi027 ceph-mon[98970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:50:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:41 smithi027 ceph-mon[98970]: osdmap e569: 8 total, 8 up, 8 in 2024-08-29T15:50:42.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:41 smithi027 ceph-mon[98970]: pgmap v1958: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 9 op/s; 44 KiB/s, 0 objects/s recovering 2024-08-29T15:50:42.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:41 smithi027 ceph-mon[93921]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:50:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:41 smithi027 ceph-mon[93921]: osdmap e569: 8 total, 8 up, 8 in 2024-08-29T15:50:42.069 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:41 smithi027 ceph-mon[93921]: pgmap v1958: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 9 op/s; 44 KiB/s, 0 objects/s recovering 2024-08-29T15:50:42.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:41 smithi171 ceph-mon[90031]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:50:42.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:41 smithi171 ceph-mon[90031]: osdmap e569: 8 total, 8 up, 8 in 2024-08-29T15:50:42.202 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:41 smithi171 ceph-mon[90031]: pgmap v1958: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 12 KiB/s rd, 36 KiB/s wr, 9 op/s; 44 KiB/s, 0 objects/s recovering 2024-08-29T15:50:42.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:50:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=cleanup t=2024-08-29T15:50:42.200408112Z level=info msg="Completed cleanup jobs" duration=3.045374ms 2024-08-29T15:50:42.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:50:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana.update.checker t=2024-08-29T15:50:42.352091013Z level=info msg="Update check succeeded" duration=68.360671ms 2024-08-29T15:50:42.518 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:50:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=plugins.update.checker t=2024-08-29T15:50:42.386654303Z level=info msg="Update check succeeded" duration=97.478908ms 2024-08-29T15:50:44.279 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-08-29T15:50:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[93921]: pgmap v1959: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 8 op/s; 41 KiB/s, 0 objects/s recovering 2024-08-29T15:50:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2685008394' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:44.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[98970]: pgmap v1959: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 8 op/s; 41 KiB/s, 0 objects/s recovering 2024-08-29T15:50:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2685008394' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:44.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:44 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:44 smithi171 ceph-mon[90031]: pgmap v1959: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 8 op/s; 41 KiB/s, 0 objects/s recovering 2024-08-29T15:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:44 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2685008394' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-29T15:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:44.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:44 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:44.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd reweight 2 1 2024-08-29T15:50:45.165 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:45.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2685008394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:50:45.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[93921]: osdmap e570: 8 total, 8 up, 8 in 2024-08-29T15:50:45.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[93921]: 1.14 scrub starts 2024-08-29T15:50:45.537 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[93921]: 1.14 scrub ok 2024-08-29T15:50:45.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2685008394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:50:45.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[98970]: osdmap e570: 8 total, 8 up, 8 in 2024-08-29T15:50:45.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[98970]: 1.14 scrub starts 2024-08-29T15:50:45.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:45 smithi027 ceph-mon[98970]: 1.14 scrub ok 2024-08-29T15:50:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:45 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2685008394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-29T15:50:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:45 smithi171 ceph-mon[90031]: osdmap e570: 8 total, 8 up, 8 in 2024-08-29T15:50:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:45 smithi171 ceph-mon[90031]: 1.14 scrub starts 2024-08-29T15:50:45.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:45 smithi171 ceph-mon[90031]: 1.14 scrub ok 2024-08-29T15:50:46.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:46 smithi027 ceph-mon[93921]: pgmap v1961: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 1.8 KiB/s rd, 5.0 KiB/s wr, 1 op/s; 5.8 KiB/s, 0 objects/s recovering 2024-08-29T15:50:46.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:46 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:50:46.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:46 smithi027 ceph-mon[98970]: pgmap v1961: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 1.8 KiB/s rd, 5.0 KiB/s wr, 1 op/s; 5.8 KiB/s, 0 objects/s recovering 2024-08-29T15:50:46.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:46 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:50:46.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:46 smithi171 ceph-mon[90031]: pgmap v1961: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 1.8 KiB/s rd, 5.0 KiB/s wr, 1 op/s; 5.8 KiB/s, 0 objects/s recovering 2024-08-29T15:50:46.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:46 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-29T15:50:47.294 INFO:teuthology.orchestra.run.smithi027.stderr:reweighted osd.2 to 1 (10000) 2024-08-29T15:50:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:47 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/248253300' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:50:47.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:47 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/248253300' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:50:47.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:47 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/248253300' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:50:48.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd primary-affinity 2 1 2024-08-29T15:50:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[98970]: pgmap v1962: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 1.6 KiB/s rd, 4.6 KiB/s wr, 1 op/s; 5.4 KiB/s, 0 objects/s recovering 2024-08-29T15:50:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/248253300' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:50:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[98970]: osdmap e571: 8 total, 8 up, 8 in 2024-08-29T15:50:48.318 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[98970]: 1.1c scrub starts 2024-08-29T15:50:48.319 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[98970]: 1.1c scrub ok 2024-08-29T15:50:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[93921]: pgmap v1962: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 1.6 KiB/s rd, 4.6 KiB/s wr, 1 op/s; 5.4 KiB/s, 0 objects/s recovering 2024-08-29T15:50:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/248253300' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:50:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[93921]: osdmap e571: 8 total, 8 up, 8 in 2024-08-29T15:50:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[93921]: 1.1c scrub starts 2024-08-29T15:50:48.319 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:48 smithi027 ceph-mon[93921]: 1.1c scrub ok 2024-08-29T15:50:48.413 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:48.614 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:48 smithi171 ceph-mon[90031]: pgmap v1962: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 1.6 KiB/s rd, 4.6 KiB/s wr, 1 op/s; 5.4 KiB/s, 0 objects/s recovering 2024-08-29T15:50:48.614 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:48 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/248253300' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:50:48.614 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:48 smithi171 ceph-mon[90031]: osdmap e571: 8 total, 8 up, 8 in 2024-08-29T15:50:48.614 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:48 smithi171 ceph-mon[90031]: 1.1c scrub starts 2024-08-29T15:50:48.614 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:48 smithi171 ceph-mon[90031]: 1.1c scrub ok 2024-08-29T15:50:49.265 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:50:49 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:50:49] "GET /metrics HTTP/1.1" 200 34160 "" "Prometheus/2.51.0" 2024-08-29T15:50:50.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:50 smithi027 ceph-mon[98970]: pgmap v1964: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:50.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:50 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:50.422 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:50 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:50.423 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:50 smithi027 ceph-mon[93921]: pgmap v1964: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:50.423 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:50 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:50.423 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:50 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:50 smithi171 ceph-mon[90031]: pgmap v1964: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:50 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:50.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:50 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:51.299 INFO:teuthology.orchestra.run.smithi027.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-08-29T15:50:51.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:51 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2614926670' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:50:51.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:51 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2614926670' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:50:51.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:51 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2614926670' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-29T15:50:52.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[93921]: pgmap v1965: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:52.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2614926670' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:50:52.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[93921]: osdmap e572: 8 total, 8 up, 8 in 2024-08-29T15:50:52.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[93921]: osdmap e573: 8 total, 8 up, 8 in 2024-08-29T15:50:52.542 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[98970]: pgmap v1965: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:52.542 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2614926670' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:50:52.542 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[98970]: osdmap e572: 8 total, 8 up, 8 in 2024-08-29T15:50:52.542 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:52 smithi027 ceph-mon[98970]: osdmap e573: 8 total, 8 up, 8 in 2024-08-29T15:50:52.565 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-08-29T15:50:52.565 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-08-29T15:50:52.565 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-08-29T15:50:52.566 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-08-29T15:50:52.566 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-29T15:50:52.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T15:50:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:52 smithi171 ceph-mon[90031]: pgmap v1965: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:52.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:52 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2614926670' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-29T15:50:52.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:52 smithi171 ceph-mon[90031]: osdmap e572: 8 total, 8 up, 8 in 2024-08-29T15:50:52.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:52 smithi171 ceph-mon[90031]: osdmap e573: 8 total, 8 up, 8 in 2024-08-29T15:50:53.409 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:53.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:53 smithi027 ceph-mon[98970]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-08-29T15:50:53.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:53.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:53 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:53.689 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:53 smithi027 ceph-mon[93921]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-08-29T15:50:53.689 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:53.689 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:53 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:53 smithi171 ceph-mon[90031]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-08-29T15:50:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:53.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:53 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[98970]: pgmap v1968: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:50:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:50:54.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:54.568 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[93921]: pgmap v1968: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:54.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:50:54.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:50:54.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:54 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:54 smithi171 ceph-mon[90031]: pgmap v1968: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-29T15:50:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-29T15:50:54.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:54 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' 2024-08-29T15:50:55.332 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:50:55.332 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":573,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T15:50:52.298765+0000","last_up_change":"2024-08-29T15:18:32.963612+0000","last_in_change":"2024-08-29T15:49:44.399624+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"485","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"470","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-29T14:55:31.336677+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-29T14:55:31.828806+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":549,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6803","nonce":297906423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6805","nonce":297906423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6809","nonce":297906423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6807","nonce":297906423}]},"public_addr":"172.21.15.27:6803/297906423","cluster_addr":"172.21.15.27:6805/297906423","heartbeat_back_addr":"172.21.15.27:6809/297906423","heartbeat_front_addr":"172.21.15.27:6807/297906423","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":549,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6811","nonce":2860215131}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6813","nonce":2860215131}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6817","nonce":2860215131}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6815","nonce":2860215131}]},"public_addr":"172.21.15.27:6811/2860215131","cluster_addr":"172.21.15.27:6813/2860215131","heartbeat_back_addr":"172.21.15.27:6817/2860215131","heartbeat_front_addr":"172.21.15.27:6815/2860215131","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":104,"last_clean_end":181,"up_from":217,"up_thru":572,"down_at":182,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6819","nonce":829268804}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6821","nonce":829268804}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6825","nonce":829268804}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6823","nonce":829268804}]},"public_addr":"172.21.15.27:6819/829268804","cluster_addr":"172.21.15.27:6821/829268804","heartbeat_back_addr":"172.21.15.27:6825/829268804","heartbeat_front_addr":"172.21.15.27:6823/829268804","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":549,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6827","nonce":3647667975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6829","nonce":3647667975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6833","nonce":3647667975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6831","nonce":3647667975}]},"public_addr":"172.21.15.27:6827/3647667975","cluster_addr":"172.21.15.27:6829/3647667975","heartbeat_back_addr":"172.21.15.27:6833/3647667975","heartbeat_front_addr":"172.21.15.27:6831/3647667975","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":564,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":195,"up_from":201,"up_thru":549,"down_at":196,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6809","nonce":2950085915}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6811","nonce":2950085915}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6815","nonce":2950085915}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6813","nonce":2950085915}]},"public_addr":"172.21.15.171:6809/2950085915","cluster_addr":"172.21.15.171:6811/2950085915","heartbeat_back_addr":"172.21.15.171:6815/2950085915","heartbeat_front_addr":"172.21.15.171:6813/2950085915","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":563,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":567,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":181},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":195},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.0","mappings":[{"from":1,"to":7}]},{"pgid":"3.d","mappings":[{"from":7,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":2}]},{"pgid":"3.1c","mappings":[{"from":4,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/258529454":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/92805749":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6801/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6800/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/921286138":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.171:0/2152868901":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2828185086":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.171:6835/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.171:0/2912002540":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2955701830":"2024-08-30T14:58:14.775345+0000","172.21.15.171:6834/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.171:0/3744606577":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6800/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.171:0/1480641142":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/1565341554":"2024-08-30T15:01:28.195875+0000","172.21.15.171:0/1064847764":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2657305076":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/1523003109":"2024-08-30T14:58:14.775345+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.171:0/3978338528":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T15:50:55.706 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:55 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/737926485' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:50:55.706 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:55 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/737926485' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:50:55.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:55 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/737926485' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:50:56.023 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-29T15:50:56.023 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-29T15:50:56.231 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:56.604 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:56 smithi027 ceph-mon[98970]: pgmap v1969: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:56.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:56 smithi027 ceph-mon[93921]: pgmap v1969: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:56.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:56 smithi171 ceph-mon[90031]: pgmap v1969: 103 pgs: 5 peering, 98 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:57.686 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:57 smithi027 ceph-mon[93921]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-29T15:50:57.687 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:57 smithi027 ceph-mon[93921]: Cluster is now healthy 2024-08-29T15:50:57.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:57 smithi027 ceph-mon[98970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-29T15:50:57.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:57 smithi027 ceph-mon[98970]: Cluster is now healthy 2024-08-29T15:50:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:57 smithi171 ceph-mon[90031]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-29T15:50:57.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:57 smithi171 ceph-mon[90031]: Cluster is now healthy 2024-08-29T15:50:58.118 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:50:58.118 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":573,"fsid":"af03926c-6615-11ef-bcd4-c7b262605968","created":"2024-08-29T14:48:44.626777+0000","modified":"2024-08-29T15:50:52.298765+0000","last_up_change":"2024-08-29T15:18:32.963612+0000","last_in_change":"2024-08-29T15:49:44.399624+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-29T14:51:48.913604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"485","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"470","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-29T14:54:17.406056+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-29T14:55:31.336677+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-29T14:55:31.828806+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"170eaad0-5c69-43ff-a4cc-cfdcc1763f52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":549,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6803","nonce":297906423}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6805","nonce":297906423}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6809","nonce":297906423}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":297906423},{"type":"v1","addr":"172.21.15.27:6807","nonce":297906423}]},"public_addr":"172.21.15.27:6803/297906423","cluster_addr":"172.21.15.27:6805/297906423","heartbeat_back_addr":"172.21.15.27:6809/297906423","heartbeat_front_addr":"172.21.15.27:6807/297906423","state":["exists","up"]},{"osd":1,"uuid":"73986270-de54-40cf-969c-ddb7ecba6f81","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":549,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6811","nonce":2860215131}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6813","nonce":2860215131}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6817","nonce":2860215131}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2860215131},{"type":"v1","addr":"172.21.15.27:6815","nonce":2860215131}]},"public_addr":"172.21.15.27:6811/2860215131","cluster_addr":"172.21.15.27:6813/2860215131","heartbeat_back_addr":"172.21.15.27:6817/2860215131","heartbeat_front_addr":"172.21.15.27:6815/2860215131","state":["exists","up"]},{"osd":2,"uuid":"d2b83d1d-ecf8-4576-a411-6d05ef2528c9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":104,"last_clean_end":181,"up_from":217,"up_thru":572,"down_at":182,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6819","nonce":829268804}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6821","nonce":829268804}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6825","nonce":829268804}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":829268804},{"type":"v1","addr":"172.21.15.27:6823","nonce":829268804}]},"public_addr":"172.21.15.27:6819/829268804","cluster_addr":"172.21.15.27:6821/829268804","heartbeat_back_addr":"172.21.15.27:6825/829268804","heartbeat_front_addr":"172.21.15.27:6823/829268804","state":["exists","up"]},{"osd":3,"uuid":"afc1c2c8-0579-4020-b460-b3cbef9573d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":549,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6827","nonce":3647667975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6829","nonce":3647667975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6833","nonce":3647667975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3647667975},{"type":"v1","addr":"172.21.15.27:6831","nonce":3647667975}]},"public_addr":"172.21.15.27:6827/3647667975","cluster_addr":"172.21.15.27:6829/3647667975","heartbeat_back_addr":"172.21.15.27:6833/3647667975","heartbeat_front_addr":"172.21.15.27:6831/3647667975","state":["exists","up"]},{"osd":4,"uuid":"adb26953-31d6-491f-8777-02fbb879b3a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":564,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6800","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6801","nonce":3854913111}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6802","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6803","nonce":3854913111}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6806","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6807","nonce":3854913111}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6804","nonce":3854913111},{"type":"v1","addr":"172.21.15.171:6805","nonce":3854913111}]},"public_addr":"172.21.15.171:6801/3854913111","cluster_addr":"172.21.15.171:6803/3854913111","heartbeat_back_addr":"172.21.15.171:6807/3854913111","heartbeat_front_addr":"172.21.15.171:6805/3854913111","state":["exists","up"]},{"osd":5,"uuid":"2b6e927d-bca5-4fe8-b17f-7b83a872afad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":195,"up_from":201,"up_thru":549,"down_at":196,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6808","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6809","nonce":2950085915}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6810","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6811","nonce":2950085915}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6814","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6815","nonce":2950085915}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6812","nonce":2950085915},{"type":"v1","addr":"172.21.15.171:6813","nonce":2950085915}]},"public_addr":"172.21.15.171:6809/2950085915","cluster_addr":"172.21.15.171:6811/2950085915","heartbeat_back_addr":"172.21.15.171:6815/2950085915","heartbeat_front_addr":"172.21.15.171:6813/2950085915","state":["exists","up"]},{"osd":6,"uuid":"f3c5992d-70c0-4893-9221-b8bc5268d92c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":563,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6816","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6817","nonce":2050068066}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6818","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6819","nonce":2050068066}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6822","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6823","nonce":2050068066}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6820","nonce":2050068066},{"type":"v1","addr":"172.21.15.171:6821","nonce":2050068066}]},"public_addr":"172.21.15.171:6817/2050068066","cluster_addr":"172.21.15.171:6819/2050068066","heartbeat_back_addr":"172.21.15.171:6823/2050068066","heartbeat_front_addr":"172.21.15.171:6821/2050068066","state":["exists","up"]},{"osd":7,"uuid":"3c1227a6-0c5b-4753-b5d3-f9e98383c3c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":567,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6824","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6825","nonce":3106954687}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6826","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6827","nonce":3106954687}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6830","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6831","nonce":3106954687}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.171:6828","nonce":3106954687},{"type":"v1","addr":"172.21.15.171:6829","nonce":3106954687}]},"public_addr":"172.21.15.171:6825/3106954687","cluster_addr":"172.21.15.171:6827/3106954687","heartbeat_back_addr":"172.21.15.171:6831/3106954687","heartbeat_front_addr":"172.21.15.171:6829/3106954687","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:50:58.842653+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:22.194363+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:51:45.639915+0000","dead_epoch":181},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:10.615337+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:32.024229+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:52:55.757910+0000","dead_epoch":195},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:19.556487+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-29T14:53:43.565018+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.0","mappings":[{"from":1,"to":7}]},{"pgid":"3.d","mappings":[{"from":7,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":2}]},{"pgid":"3.1c","mappings":[{"from":4,"to":0}]},{"pgid":"3.1d","mappings":[{"from":4,"to":0}]},{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/258529454":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/92805749":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6801/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:6800/301351752":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/921286138":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2218055720":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/3735938855":"2024-08-30T14:49:36.411583+0000","172.21.15.171:0/2152868901":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2828185086":"2024-08-30T15:01:28.195875+0000","172.21.15.27:0/1323146262":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/2385046935":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6800/1661583367":"2024-08-30T14:49:36.411583+0000","172.21.15.27:6801/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/2198805353":"2024-08-30T14:54:15.539878+0000","172.21.15.171:6835/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.171:0/2912002540":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/2955701830":"2024-08-30T14:58:14.775345+0000","172.21.15.171:6834/2863400284":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/3268243398":"2024-08-30T14:49:36.411583+0000","172.21.15.27:0/728365796":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/275324420":"2024-08-30T14:54:15.539878+0000","172.21.15.27:0/579941994":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2006046934":"2024-08-30T14:49:05.654793+0000","172.21.15.171:0/3744606577":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6800/2650682884":"2024-08-30T14:58:14.775345+0000","172.21.15.171:0/1480641142":"2024-08-30T14:58:56.932943+0000","172.21.15.27:0/1565341554":"2024-08-30T15:01:28.195875+0000","172.21.15.171:0/1064847764":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6800/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:6801/2777626812":"2024-08-30T14:49:05.654793+0000","172.21.15.27:0/2657305076":"2024-08-30T14:58:14.775345+0000","172.21.15.27:0/1523003109":"2024-08-30T14:58:14.775345+0000","172.21.15.27:6800/1134596600":"2024-08-30T14:54:15.539878+0000","172.21.15.171:0/3978338528":"2024-08-30T14:58:56.932943+0000","172.21.15.27:6801/1134596600":"2024-08-30T14:54:15.539878+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-29T15:50:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:58 smithi171 ceph-mon[90031]: pgmap v1970: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:58.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:58 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2033528181' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:50:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:58 smithi027 ceph-mon[98970]: pgmap v1970: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:58.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:58 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2033528181' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:50:58.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:58 smithi027 ceph-mon[93921]: pgmap v1970: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:50:58.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:58 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2033528181' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-29T15:50:58.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-29T15:50:58.899 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-29T15:50:58.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-29T15:50:58.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-29T15:50:58.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-29T15:50:58.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-29T15:50:58.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-29T15:50:58.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-29T15:50:59.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:50:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:50:59] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.51.0" 2024-08-29T15:50:59.390 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.390 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.393 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.393 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.394 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.401 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.403 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.413 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:50:59.708 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:50:59 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:59.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:50:59 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:50:59.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:50:59 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:51:00.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:00 smithi027 ceph-mon[93921]: pgmap v1971: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:00.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:00 smithi027 ceph-mon[98970]: pgmap v1971: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:00.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:00 smithi171 ceph-mon[90031]: pgmap v1971: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:02.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:02 smithi171 ceph-mon[90031]: pgmap v1972: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:02.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:02 smithi027 ceph-mon[98970]: pgmap v1972: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:02.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:02 smithi027 ceph-mon[93921]: pgmap v1972: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:04.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:04 smithi027 ceph-mon[98970]: pgmap v1973: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:04.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:04 smithi027 ceph-mon[93921]: pgmap v1973: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:04.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:04 smithi171 ceph-mon[90031]: pgmap v1973: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:06.567 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:06 smithi027 ceph-mon[98970]: pgmap v1974: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:06.567 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:06 smithi027 ceph-mon[93921]: pgmap v1974: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:06.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:06 smithi171 ceph-mon[90031]: pgmap v1974: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:08.419 INFO:teuthology.orchestra.run.smithi027.stdout:231928234678 2024-08-29T15:51:08.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-29T15:51:08.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:08 smithi171 ceph-mon[90031]: pgmap v1975: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:08.796 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:08 smithi027 ceph-mon[98970]: pgmap v1975: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:08.796 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:08 smithi027 ceph-mon[93921]: pgmap v1975: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:09.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:09 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:51:09] "GET /metrics HTTP/1.1" 200 34139 "" "Prometheus/2.51.0" 2024-08-29T15:51:10.006 INFO:teuthology.orchestra.run.smithi027.stdout:128849019588 2024-08-29T15:51:10.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-29T15:51:10.647 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:10 smithi027 ceph-mon[98970]: pgmap v1976: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:10.647 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:10 smithi027 ceph-mon[93921]: pgmap v1976: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:10.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:10 smithi171 ceph-mon[90031]: pgmap v1976: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:12.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:12 smithi171 ceph-mon[90031]: pgmap v1977: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:12.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:12 smithi027 ceph-mon[98970]: pgmap v1977: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:12.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:12 smithi027 ceph-mon[93921]: pgmap v1977: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:13.977 INFO:teuthology.orchestra.run.smithi027.stdout:468151435773 2024-08-29T15:51:13.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-29T15:51:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:14 smithi027 ceph-mon[98970]: pgmap v1978: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:14.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:14 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:51:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:14 smithi027 ceph-mon[93921]: pgmap v1978: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:14.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:14 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:51:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:14 smithi171 ceph-mon[90031]: pgmap v1978: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:14.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:14 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:51:15.680 INFO:teuthology.orchestra.run.smithi027.stdout:403726926350 2024-08-29T15:51:15.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-29T15:51:16.107 INFO:teuthology.orchestra.run.smithi027.stdout:932007903626 2024-08-29T15:51:16.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-29T15:51:16.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:16 smithi171 ceph-mon[90031]: pgmap v1979: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:16.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:16 smithi027 ceph-mon[93921]: pgmap v1979: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:16.784 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:16 smithi027 ceph-mon[98970]: pgmap v1979: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:17.159 INFO:teuthology.orchestra.run.smithi027.stdout:425201762827 2024-08-29T15:51:17.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-29T15:51:17.684 INFO:teuthology.orchestra.run.smithi027.stdout:197568496310 2024-08-29T15:51:17.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-29T15:51:18.232 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:18.669 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:18 smithi027 ceph-mon[93921]: pgmap v1980: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:18.670 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:18 smithi027 ceph-mon[98970]: pgmap v1980: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:18.693 INFO:teuthology.orchestra.run.smithi027.stdout:863288426898 2024-08-29T15:51:18.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-29T15:51:18.709 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:18.709 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:18.714 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:18.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:18 smithi171 ceph-mon[90031]: pgmap v1980: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:19.318 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:19 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:51:19] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.51.0" 2024-08-29T15:51:20.725 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:20 smithi027 ceph-mon[93921]: pgmap v1981: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:20.726 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:20 smithi027 ceph-mon[98970]: pgmap v1981: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:20.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:20 smithi171 ceph-mon[90031]: pgmap v1981: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:22.715 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:22 smithi027 ceph-mon[98970]: pgmap v1982: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:22.715 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:22 smithi027 ceph-mon[93921]: pgmap v1982: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:22.733 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:22.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:22 smithi171 ceph-mon[90031]: pgmap v1982: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:23.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:23 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3770784839' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-29T15:51:23.738 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:23 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3770784839' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-29T15:51:23.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:23 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3770784839' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-29T15:51:24.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:24 smithi027 ceph-mon[98970]: pgmap v1983: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:24.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:24 smithi027 ceph-mon[93921]: pgmap v1983: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:24.519 INFO:teuthology.orchestra.run.smithi027.stdout:231928234681 2024-08-29T15:51:24.547 INFO:teuthology.orchestra.run.smithi027.stdout:468151435776 2024-08-29T15:51:24.687 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:24.687 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:24.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:24 smithi171 ceph-mon[90031]: pgmap v1983: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:25.120 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234678 got 231928234681 for osd.7 2024-08-29T15:51:25.121 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:25.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:25 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/1941860679' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-29T15:51:25.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:25 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/1941860679' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-29T15:51:25.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:25 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/1941860679' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-29T15:51:26.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:26 smithi171 ceph-mon[90031]: pgmap v1984: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:26.790 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:26 smithi027 ceph-mon[93921]: pgmap v1984: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:26.791 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:26 smithi027 ceph-mon[98970]: pgmap v1984: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:26.961 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:27.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435773 got 468151435776 for osd.3 2024-08-29T15:51:27.461 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:28.673 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:28 smithi027 ceph-mon[98970]: pgmap v1985: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:28.674 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:28 smithi027 ceph-mon[93921]: pgmap v1985: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:28.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:28 smithi171 ceph-mon[90031]: pgmap v1985: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:28.893 INFO:teuthology.orchestra.run.smithi027.stdout:128849019592 2024-08-29T15:51:29.259 INFO:teuthology.orchestra.run.smithi027.stdout:197568496313 2024-08-29T15:51:29.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:29 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:51:29] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.51.0" 2024-08-29T15:51:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:29 smithi027 ceph-mon[98970]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:51:29.568 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4000950007' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-29T15:51:29.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:29 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/87249240' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-29T15:51:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:29 smithi027 ceph-mon[93921]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:51:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4000950007' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-29T15:51:29.569 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:29 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/87249240' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-29T15:51:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:29 smithi171 ceph-mon[90031]: from='mgr.34104 172.21.15.27:0/80820940' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-29T15:51:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4000950007' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-29T15:51:29.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:29 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/87249240' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-29T15:51:30.107 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496310 got 197568496313 for osd.6 2024-08-29T15:51:30.108 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:30.142 INFO:teuthology.orchestra.run.smithi027.stdout:932007903629 2024-08-29T15:51:30.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:30 smithi171 ceph-mon[90031]: pgmap v1986: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:30.768 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:30 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/3526354417' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-29T15:51:30.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:30 smithi027 ceph-mon[98970]: pgmap v1986: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:30.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:30 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/3526354417' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-29T15:51:30.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:30 smithi027 ceph-mon[93921]: pgmap v1986: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:30.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:30 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/3526354417' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-29T15:51:31.350 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019588 got 128849019592 for osd.4 2024-08-29T15:51:31.350 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:32.407 INFO:tasks.cephadm.ceph_manager.ceph:need seq 932007903626 got 932007903629 for osd.2 2024-08-29T15:51:32.407 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:32.666 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:32 smithi027 ceph-mon[93921]: pgmap v1987: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:32.667 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:32 smithi027 ceph-mon[98970]: pgmap v1987: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:32.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:32 smithi171 ceph-mon[90031]: pgmap v1987: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:33.093 INFO:teuthology.orchestra.run.smithi027.stdout:863288426901 2024-08-29T15:51:33.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:33 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/4021157166' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-29T15:51:33.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:33 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/4021157166' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-29T15:51:33.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:33 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/4021157166' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-29T15:51:33.924 INFO:teuthology.orchestra.run.smithi027.stdout:403726926354 2024-08-29T15:51:34.240 INFO:tasks.cephadm.ceph_manager.ceph:need seq 863288426898 got 863288426901 for osd.5 2024-08-29T15:51:34.240 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:34 smithi171 ceph-mon[90031]: pgmap v1988: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:34.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:34 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/805281588' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-29T15:51:34.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:34 smithi027 ceph-mon[98970]: pgmap v1988: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:34.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:34 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/805281588' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-29T15:51:34.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:34 smithi027 ceph-mon[93921]: pgmap v1988: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:34.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:34 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/805281588' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-29T15:51:35.507 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926350 got 403726926354 for osd.0 2024-08-29T15:51:35.507 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:36.653 INFO:teuthology.orchestra.run.smithi027.stdout:425201762831 2024-08-29T15:51:36.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:36 smithi171 ceph-mon[90031]: pgmap v1989: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:36.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:36 smithi027 ceph-mon[98970]: pgmap v1989: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:36.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:36 smithi027 ceph-mon[93921]: pgmap v1989: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:37.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762827 got 425201762831 for osd.1 2024-08-29T15:51:37.344 DEBUG:teuthology.parallel:result is None 2024-08-29T15:51:37.344 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-29T15:51:37.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:51:37.560 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:37.682 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:37 smithi027 ceph-mon[93921]: from='client.? 172.21.15.27:0/2571646539' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-29T15:51:37.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:37 smithi027 ceph-mon[98970]: from='client.? 172.21.15.27:0/2571646539' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-29T15:51:37.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:37 smithi171 ceph-mon[90031]: from='client.? 172.21.15.27:0/2571646539' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-29T15:51:38.711 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:38 smithi027 ceph-mon[93921]: pgmap v1990: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:38.711 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:38 smithi027 ceph-mon[98970]: pgmap v1990: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:38.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:38 smithi171 ceph-mon[90031]: pgmap v1990: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:39.265 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:39 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: ::ffff:172.21.15.171 - - [29/Aug/2024:15:51:39] "GET /metrics HTTP/1.1" 200 34138 "" "Prometheus/2.51.0" 2024-08-29T15:51:39.431 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:51:39.432 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:51:40.217 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1991,"stamp":"2024-08-29T15:51:39.237203+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":204,"num_bytes_recovered":17376184,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5652,"ondisk_log_size":5652,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6956900,"kb_used_data":77620,"kb_used_omap":43,"kb_used_meta":6879124,"kb_avail":743004316,"statfs":{"total":767960285184,"available":760836419584,"internally_reserved":0,"allocated":79482880,"data_stored":51255838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":7044223468},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002984"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":200,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307600+0000","last_change":"2024-08-29T15:49:45.432721+0000","last_active":"2024-08-29T15:50:52.307600+0000","last_peered":"2024-08-29T15:50:52.307600+0000","last_clean":"2024-08-29T15:50:52.307600+0000","last_became_active":"2024-08-29T15:49:45.425230+0000","last_became_peered":"2024-08-29T15:49:45.425230+0000","last_unstale":"2024-08-29T15:50:52.307600+0000","last_undegraded":"2024-08-29T15:50:52.307600+0000","last_fullsized":"2024-08-29T15:50:52.307600+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:57:53.189694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3492,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":362,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306284+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:50:52.306284+0000","last_peered":"2024-08-29T15:50:52.306284+0000","last_clean":"2024-08-29T15:50:52.306284+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:50:52.306284+0000","last_undegraded":"2024-08-29T15:50:52.306284+0000","last_fullsized":"2024-08-29T15:50:52.306284+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'149","reported_seq":1001,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423087+0000","last_change":"2024-08-29T15:47:46.316018+0000","last_active":"2024-08-29T15:50:52.423087+0000","last_peered":"2024-08-29T15:50:52.423087+0000","last_clean":"2024-08-29T15:50:52.423087+0000","last_became_active":"2024-08-29T15:47:46.315610+0000","last_became_peered":"2024-08-29T15:47:46.315610+0000","last_unstale":"2024-08-29T15:50:52.423087+0000","last_undegraded":"2024-08-29T15:50:52.423087+0000","last_fullsized":"2024-08-29T15:50:52.423087+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:24.970456+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:45:24.970456+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:30:31.344363+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1051,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305798+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:50:52.305798+0000","last_peered":"2024-08-29T15:50:52.305798+0000","last_clean":"2024-08-29T15:50:52.305798+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:50:52.305798+0000","last_undegraded":"2024-08-29T15:50:52.305798+0000","last_fullsized":"2024-08-29T15:50:52.305798+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1047,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305334+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:50:52.305334+0000","last_peered":"2024-08-29T15:50:52.305334+0000","last_clean":"2024-08-29T15:50:52.305334+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:50:52.305334+0000","last_undegraded":"2024-08-29T15:50:52.305334+0000","last_fullsized":"2024-08-29T15:50:52.305334+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"90'149","reported_seq":581,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307882+0000","last_change":"2024-08-29T15:50:47.383348+0000","last_active":"2024-08-29T15:50:51.307882+0000","last_peered":"2024-08-29T15:50:51.307882+0000","last_clean":"2024-08-29T15:50:51.307882+0000","last_became_active":"2024-08-29T15:50:37.804811+0000","last_became_peered":"2024-08-29T15:50:37.804811+0000","last_unstale":"2024-08-29T15:50:51.307882+0000","last_undegraded":"2024-08-29T15:50:51.307882+0000","last_fullsized":"2024-08-29T15:50:51.307882+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:47.383216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:47.383216+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:54:29.268648+0000","scrub_duration":0.0017543280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":41,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301359+0000","last_change":"2024-08-29T15:49:45.433633+0000","last_active":"2024-08-29T15:50:51.301359+0000","last_peered":"2024-08-29T15:50:51.301359+0000","last_clean":"2024-08-29T15:50:51.301359+0000","last_became_active":"2024-08-29T15:49:45.424864+0000","last_became_peered":"2024-08-29T15:49:45.424864+0000","last_unstale":"2024-08-29T15:50:51.301359+0000","last_undegraded":"2024-08-29T15:50:51.301359+0000","last_fullsized":"2024-08-29T15:50:51.301359+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:37:00.817911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":300,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303194+0000","last_change":"2024-08-29T15:50:25.624548+0000","last_active":"2024-08-29T15:50:51.303194+0000","last_peered":"2024-08-29T15:50:51.303194+0000","last_clean":"2024-08-29T15:50:51.303194+0000","last_became_active":"2024-08-29T15:50:25.624279+0000","last_became_peered":"2024-08-29T15:50:25.624279+0000","last_unstale":"2024-08-29T15:50:51.303194+0000","last_undegraded":"2024-08-29T15:50:51.303194+0000","last_fullsized":"2024-08-29T15:50:51.303194+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:24:27.488131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":41,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301148+0000","last_change":"2024-08-29T15:49:45.431781+0000","last_active":"2024-08-29T15:50:51.301148+0000","last_peered":"2024-08-29T15:50:51.301148+0000","last_clean":"2024-08-29T15:50:51.301148+0000","last_became_active":"2024-08-29T15:49:45.422282+0000","last_became_peered":"2024-08-29T15:49:45.422282+0000","last_unstale":"2024-08-29T15:50:51.301148+0000","last_undegraded":"2024-08-29T15:50:51.301148+0000","last_fullsized":"2024-08-29T15:50:51.301148+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:53:33.408668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":397,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303206+0000","last_change":"2024-08-29T15:47:12.481314+0000","last_active":"2024-08-29T15:50:51.303206+0000","last_peered":"2024-08-29T15:50:51.303206+0000","last_clean":"2024-08-29T15:50:51.303206+0000","last_became_active":"2024-08-29T15:47:12.481010+0000","last_became_peered":"2024-08-29T15:47:12.481010+0000","last_unstale":"2024-08-29T15:50:51.303206+0000","last_undegraded":"2024-08-29T15:50:51.303206+0000","last_fullsized":"2024-08-29T15:50:51.303206+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:58:04.897625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"90'149","reported_seq":1030,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305251+0000","last_change":"2024-08-29T15:44:59.241503+0000","last_active":"2024-08-29T15:50:52.305251+0000","last_peered":"2024-08-29T15:50:52.305251+0000","last_clean":"2024-08-29T15:50:52.305251+0000","last_became_active":"2024-08-29T15:44:59.241088+0000","last_became_peered":"2024-08-29T15:44:59.241088+0000","last_unstale":"2024-08-29T15:50:52.305251+0000","last_undegraded":"2024-08-29T15:50:52.305251+0000","last_fullsized":"2024-08-29T15:50:52.305251+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:27.788587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:27.788587+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:39.021866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":167,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306834+0000","last_change":"2024-08-29T15:46:58.367038+0000","last_active":"2024-08-29T15:50:52.306834+0000","last_peered":"2024-08-29T15:50:52.306834+0000","last_clean":"2024-08-29T15:50:52.306834+0000","last_became_active":"2024-08-29T15:46:58.366692+0000","last_became_peered":"2024-08-29T15:46:58.366692+0000","last_unstale":"2024-08-29T15:50:52.306834+0000","last_undegraded":"2024-08-29T15:50:52.306834+0000","last_fullsized":"2024-08-29T15:50:52.306834+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:40:02.240790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":988,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.424328+0000","last_change":"2024-08-29T15:50:52.424216+0000","last_active":"2024-08-29T15:50:52.424328+0000","last_peered":"2024-08-29T15:50:52.424328+0000","last_clean":"2024-08-29T15:50:52.424328+0000","last_became_active":"2024-08-29T15:50:52.423829+0000","last_became_peered":"2024-08-29T15:50:52.423829+0000","last_unstale":"2024-08-29T15:50:52.424328+0000","last_undegraded":"2024-08-29T15:50:52.424328+0000","last_fullsized":"2024-08-29T15:50:52.424328+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:30.219791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"90'149","reported_seq":1184,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305621+0000","last_change":"2024-08-29T15:47:01.424749+0000","last_active":"2024-08-29T15:50:52.305621+0000","last_peered":"2024-08-29T15:50:52.305621+0000","last_clean":"2024-08-29T15:50:52.305621+0000","last_became_active":"2024-08-29T15:47:01.424333+0000","last_became_peered":"2024-08-29T15:47:01.424333+0000","last_unstale":"2024-08-29T15:50:52.305621+0000","last_undegraded":"2024-08-29T15:50:52.305621+0000","last_fullsized":"2024-08-29T15:50:52.305621+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:31.851590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:31.851590+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:01.272901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":361,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303160+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:50:51.303160+0000","last_peered":"2024-08-29T15:50:51.303160+0000","last_clean":"2024-08-29T15:50:51.303160+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:50:51.303160+0000","last_undegraded":"2024-08-29T15:50:51.303160+0000","last_fullsized":"2024-08-29T15:50:51.303160+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1016,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305009+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:50:52.305009+0000","last_peered":"2024-08-29T15:50:52.305009+0000","last_clean":"2024-08-29T15:50:52.305009+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:50:52.305009+0000","last_undegraded":"2024-08-29T15:50:52.305009+0000","last_fullsized":"2024-08-29T15:50:52.305009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"90'149","reported_seq":803,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300656+0000","last_change":"2024-08-29T15:49:48.294772+0000","last_active":"2024-08-29T15:50:51.300656+0000","last_peered":"2024-08-29T15:50:51.300656+0000","last_clean":"2024-08-29T15:50:51.300656+0000","last_became_active":"2024-08-29T15:49:48.293676+0000","last_became_peered":"2024-08-29T15:49:48.293676+0000","last_unstale":"2024-08-29T15:50:51.300656+0000","last_undegraded":"2024-08-29T15:50:51.300656+0000","last_fullsized":"2024-08-29T15:50:51.300656+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:37.167690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:37.167690+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:36.708271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1012,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422693+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:50:52.422693+0000","last_peered":"2024-08-29T15:50:52.422693+0000","last_clean":"2024-08-29T15:50:52.422693+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:50:52.422693+0000","last_undegraded":"2024-08-29T15:50:52.422693+0000","last_fullsized":"2024-08-29T15:50:52.422693+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307439+0000","last_change":"2024-08-29T15:48:12.267373+0000","last_active":"2024-08-29T15:50:52.307439+0000","last_peered":"2024-08-29T15:50:52.307439+0000","last_clean":"2024-08-29T15:50:52.307439+0000","last_became_active":"2024-08-29T15:48:12.267059+0000","last_became_peered":"2024-08-29T15:48:12.267059+0000","last_unstale":"2024-08-29T15:50:52.307439+0000","last_undegraded":"2024-08-29T15:50:52.307439+0000","last_fullsized":"2024-08-29T15:50:52.307439+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:25:55.282346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"90'149","reported_seq":1047,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300909+0000","last_change":"2024-08-29T15:49:48.295500+0000","last_active":"2024-08-29T15:50:51.300909+0000","last_peered":"2024-08-29T15:50:51.300909+0000","last_clean":"2024-08-29T15:50:51.300909+0000","last_became_active":"2024-08-29T15:49:48.295131+0000","last_became_peered":"2024-08-29T15:49:48.295131+0000","last_unstale":"2024-08-29T15:50:51.300909+0000","last_undegraded":"2024-08-29T15:50:51.300909+0000","last_fullsized":"2024-08-29T15:50:51.300909+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:40.396348+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:40.396348+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:59:09.899452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":191,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.424352+0000","last_change":"2024-08-29T15:50:52.424253+0000","last_active":"2024-08-29T15:50:52.424352+0000","last_peered":"2024-08-29T15:50:52.424352+0000","last_clean":"2024-08-29T15:50:52.424352+0000","last_became_active":"2024-08-29T15:50:52.423932+0000","last_became_peered":"2024-08-29T15:50:52.423932+0000","last_unstale":"2024-08-29T15:50:52.424352+0000","last_undegraded":"2024-08-29T15:50:52.424352+0000","last_fullsized":"2024-08-29T15:50:52.424352+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:22.076998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":191,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306757+0000","last_change":"2024-08-29T15:50:31.763839+0000","last_active":"2024-08-29T15:50:51.306757+0000","last_peered":"2024-08-29T15:50:51.306757+0000","last_clean":"2024-08-29T15:50:51.306757+0000","last_became_active":"2024-08-29T15:50:31.763276+0000","last_became_peered":"2024-08-29T15:50:31.763276+0000","last_unstale":"2024-08-29T15:50:51.306757+0000","last_undegraded":"2024-08-29T15:50:51.306757+0000","last_fullsized":"2024-08-29T15:50:51.306757+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:21:46.127572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"90'149","reported_seq":817,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.302937+0000","last_change":"2024-08-29T15:45:26.912257+0000","last_active":"2024-08-29T15:50:51.302937+0000","last_peered":"2024-08-29T15:50:51.302937+0000","last_clean":"2024-08-29T15:50:51.302937+0000","last_became_active":"2024-08-29T15:44:59.244250+0000","last_became_peered":"2024-08-29T15:44:59.244250+0000","last_unstale":"2024-08-29T15:50:51.302937+0000","last_undegraded":"2024-08-29T15:50:51.302937+0000","last_fullsized":"2024-08-29T15:50:51.302937+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.912179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.912179+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:51.014992+0000","scrub_duration":0.0012045789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1032,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304779+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:50:52.304779+0000","last_peered":"2024-08-29T15:50:52.304779+0000","last_clean":"2024-08-29T15:50:52.304779+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:50:52.304779+0000","last_undegraded":"2024-08-29T15:50:52.304779+0000","last_fullsized":"2024-08-29T15:50:52.304779+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":181,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307671+0000","last_change":"2024-08-29T15:49:45.428868+0000","last_active":"2024-08-29T15:50:52.307671+0000","last_peered":"2024-08-29T15:50:52.307671+0000","last_clean":"2024-08-29T15:50:52.307671+0000","last_became_active":"2024-08-29T15:49:45.428396+0000","last_became_peered":"2024-08-29T15:49:45.428396+0000","last_unstale":"2024-08-29T15:50:52.307671+0000","last_undegraded":"2024-08-29T15:50:52.307671+0000","last_fullsized":"2024-08-29T15:50:52.307671+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:43:59.655509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"90'149","reported_seq":919,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307532+0000","last_change":"2024-08-29T15:49:48.296397+0000","last_active":"2024-08-29T15:50:51.307532+0000","last_peered":"2024-08-29T15:50:51.307532+0000","last_clean":"2024-08-29T15:50:51.307532+0000","last_became_active":"2024-08-29T15:49:48.296044+0000","last_became_peered":"2024-08-29T15:49:48.296044+0000","last_unstale":"2024-08-29T15:50:51.307532+0000","last_undegraded":"2024-08-29T15:50:51.307532+0000","last_fullsized":"2024-08-29T15:50:51.307532+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.882638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.882638+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:49.621431+0000","scrub_duration":0.001489043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":980,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304936+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:50:52.304936+0000","last_peered":"2024-08-29T15:50:52.304936+0000","last_clean":"2024-08-29T15:50:52.304936+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:50:52.304936+0000","last_undegraded":"2024-08-29T15:50:52.304936+0000","last_fullsized":"2024-08-29T15:50:52.304936+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301244+0000","last_change":"2024-08-29T15:49:45.424186+0000","last_active":"2024-08-29T15:50:51.301244+0000","last_peered":"2024-08-29T15:50:51.301244+0000","last_clean":"2024-08-29T15:50:51.301244+0000","last_became_active":"2024-08-29T15:49:45.423882+0000","last_became_peered":"2024-08-29T15:49:45.423882+0000","last_unstale":"2024-08-29T15:50:51.301244+0000","last_undegraded":"2024-08-29T15:50:51.301244+0000","last_fullsized":"2024-08-29T15:50:51.301244+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:52:36.923638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"90'149","reported_seq":1148,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.308659+0000","last_change":"2024-08-29T15:50:52.308570+0000","last_active":"2024-08-29T15:50:52.308659+0000","last_peered":"2024-08-29T15:50:52.308659+0000","last_clean":"2024-08-29T15:50:52.308659+0000","last_became_active":"2024-08-29T15:50:52.308209+0000","last_became_peered":"2024-08-29T15:50:52.308209+0000","last_unstale":"2024-08-29T15:50:52.308659+0000","last_undegraded":"2024-08-29T15:50:52.308659+0000","last_fullsized":"2024-08-29T15:50:52.308659+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:44.368136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:44.368136+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:10.878495+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":201,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307390+0000","last_change":"2024-08-29T15:50:37.806204+0000","last_active":"2024-08-29T15:50:51.307390+0000","last_peered":"2024-08-29T15:50:51.307390+0000","last_clean":"2024-08-29T15:50:51.307390+0000","last_became_active":"2024-08-29T15:50:37.805868+0000","last_became_peered":"2024-08-29T15:50:37.805868+0000","last_unstale":"2024-08-29T15:50:51.307390+0000","last_undegraded":"2024-08-29T15:50:51.307390+0000","last_fullsized":"2024-08-29T15:50:51.307390+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:21:01.177942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422660+0000","last_change":"2024-08-29T15:47:12.486814+0000","last_active":"2024-08-29T15:50:52.422660+0000","last_peered":"2024-08-29T15:50:52.422660+0000","last_clean":"2024-08-29T15:50:52.422660+0000","last_became_active":"2024-08-29T15:47:12.486372+0000","last_became_peered":"2024-08-29T15:47:12.486372+0000","last_unstale":"2024-08-29T15:50:52.422660+0000","last_undegraded":"2024-08-29T15:50:52.422660+0000","last_fullsized":"2024-08-29T15:50:52.422660+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:38.593835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"90'149","reported_seq":1024,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301368+0000","last_change":"2024-08-29T15:49:48.295362+0000","last_active":"2024-08-29T15:50:51.301368+0000","last_peered":"2024-08-29T15:50:51.301368+0000","last_clean":"2024-08-29T15:50:51.301368+0000","last_became_active":"2024-08-29T15:49:48.295053+0000","last_became_peered":"2024-08-29T15:49:48.295053+0000","last_unstale":"2024-08-29T15:50:51.301368+0000","last_undegraded":"2024-08-29T15:50:51.301368+0000","last_fullsized":"2024-08-29T15:50:51.301368+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:28.337426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:28.337426+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:14.219256+0000","scrub_duration":0.001450535,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":372,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303344+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:50:51.303344+0000","last_peered":"2024-08-29T15:50:51.303344+0000","last_clean":"2024-08-29T15:50:51.303344+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:50:51.303344+0000","last_undegraded":"2024-08-29T15:50:51.303344+0000","last_fullsized":"2024-08-29T15:50:51.303344+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":362,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303535+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:50:51.303535+0000","last_peered":"2024-08-29T15:50:51.303535+0000","last_clean":"2024-08-29T15:50:51.303535+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:50:51.303535+0000","last_undegraded":"2024-08-29T15:50:51.303535+0000","last_fullsized":"2024-08-29T15:50:51.303535+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"90'149","reported_seq":1024,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423052+0000","last_change":"2024-08-29T15:44:59.239507+0000","last_active":"2024-08-29T15:50:52.423052+0000","last_peered":"2024-08-29T15:50:52.423052+0000","last_clean":"2024-08-29T15:50:52.423052+0000","last_became_active":"2024-08-29T15:44:59.239049+0000","last_became_peered":"2024-08-29T15:44:59.239049+0000","last_unstale":"2024-08-29T15:50:52.423052+0000","last_undegraded":"2024-08-29T15:50:52.423052+0000","last_fullsized":"2024-08-29T15:50:52.423052+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:28.746327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:28.746327+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:38:22.918512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":302,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303075+0000","last_change":"2024-08-29T15:50:25.955660+0000","last_active":"2024-08-29T15:50:51.303075+0000","last_peered":"2024-08-29T15:50:51.303075+0000","last_clean":"2024-08-29T15:50:51.303075+0000","last_became_active":"2024-08-29T15:50:25.955251+0000","last_became_peered":"2024-08-29T15:50:25.955251+0000","last_unstale":"2024-08-29T15:50:51.303075+0000","last_undegraded":"2024-08-29T15:50:51.303075+0000","last_fullsized":"2024-08-29T15:50:51.303075+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:08:35.207019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":377,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306684+0000","last_change":"2024-08-29T15:48:12.274839+0000","last_active":"2024-08-29T15:50:51.306684+0000","last_peered":"2024-08-29T15:50:51.306684+0000","last_clean":"2024-08-29T15:50:51.306684+0000","last_became_active":"2024-08-29T15:48:12.268376+0000","last_became_peered":"2024-08-29T15:48:12.268376+0000","last_unstale":"2024-08-29T15:50:51.306684+0000","last_undegraded":"2024-08-29T15:50:51.306684+0000","last_fullsized":"2024-08-29T15:50:51.306684+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:19.911889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":172,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307347+0000","last_change":"2024-08-29T15:46:58.380711+0000","last_active":"2024-08-29T15:50:52.307347+0000","last_peered":"2024-08-29T15:50:52.307347+0000","last_clean":"2024-08-29T15:50:52.307347+0000","last_became_active":"2024-08-29T15:46:58.369572+0000","last_became_peered":"2024-08-29T15:46:58.369572+0000","last_unstale":"2024-08-29T15:50:52.307347+0000","last_undegraded":"2024-08-29T15:50:52.307347+0000","last_fullsized":"2024-08-29T15:50:52.307347+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:49:53.912727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":1216,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304981+0000","last_change":"2024-08-29T15:49:48.293377+0000","last_active":"2024-08-29T15:50:52.304981+0000","last_peered":"2024-08-29T15:50:52.304981+0000","last_clean":"2024-08-29T15:50:52.304981+0000","last_became_active":"2024-08-29T15:49:48.292925+0000","last_became_peered":"2024-08-29T15:49:48.292925+0000","last_unstale":"2024-08-29T15:50:52.304981+0000","last_undegraded":"2024-08-29T15:50:52.304981+0000","last_fullsized":"2024-08-29T15:50:52.304981+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.774861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.774861+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:07:53.726096+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":370,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304685+0000","last_change":"2024-08-29T15:48:12.266718+0000","last_active":"2024-08-29T15:50:52.304685+0000","last_peered":"2024-08-29T15:50:52.304685+0000","last_clean":"2024-08-29T15:50:52.304685+0000","last_became_active":"2024-08-29T15:48:12.266363+0000","last_became_peered":"2024-08-29T15:48:12.266363+0000","last_unstale":"2024-08-29T15:50:52.304685+0000","last_undegraded":"2024-08-29T15:50:52.304685+0000","last_fullsized":"2024-08-29T15:50:52.304685+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:53.550145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1669,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423349+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:50:52.423349+0000","last_peered":"2024-08-29T15:50:52.423349+0000","last_clean":"2024-08-29T15:50:52.423349+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:50:52.423349+0000","last_undegraded":"2024-08-29T15:50:52.423349+0000","last_fullsized":"2024-08-29T15:50:52.423349+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1017,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305644+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:50:52.305644+0000","last_peered":"2024-08-29T15:50:52.305644+0000","last_clean":"2024-08-29T15:50:52.305644+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:50:52.305644+0000","last_undegraded":"2024-08-29T15:50:52.305644+0000","last_fullsized":"2024-08-29T15:50:52.305644+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":990,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303326+0000","last_change":"2024-08-29T15:44:22.861452+0000","last_active":"2024-08-29T15:50:51.303326+0000","last_peered":"2024-08-29T15:50:51.303326+0000","last_clean":"2024-08-29T15:50:51.303326+0000","last_became_active":"2024-08-29T15:44:22.611166+0000","last_became_peered":"2024-08-29T15:44:22.611166+0000","last_unstale":"2024-08-29T15:50:51.303326+0000","last_undegraded":"2024-08-29T15:50:51.303326+0000","last_fullsized":"2024-08-29T15:50:51.303326+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.861280+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.861280+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:45.254970+0000","scrub_duration":0.0040975610000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1038,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307194+0000","last_change":"2024-08-29T15:49:45.424973+0000","last_active":"2024-08-29T15:50:52.307194+0000","last_peered":"2024-08-29T15:50:52.307194+0000","last_clean":"2024-08-29T15:50:52.307194+0000","last_became_active":"2024-08-29T15:49:45.424624+0000","last_became_peered":"2024-08-29T15:49:45.424624+0000","last_unstale":"2024-08-29T15:50:52.307194+0000","last_undegraded":"2024-08-29T15:50:52.307194+0000","last_fullsized":"2024-08-29T15:50:52.307194+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:46.192350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":1093,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307574+0000","last_change":"2024-08-29T15:46:58.399147+0000","last_active":"2024-08-29T15:50:51.307574+0000","last_peered":"2024-08-29T15:50:51.307574+0000","last_clean":"2024-08-29T15:50:51.307574+0000","last_became_active":"2024-08-29T15:46:58.371595+0000","last_became_peered":"2024-08-29T15:46:58.371595+0000","last_unstale":"2024-08-29T15:50:51.307574+0000","last_undegraded":"2024-08-29T15:50:51.307574+0000","last_fullsized":"2024-08-29T15:50:51.307574+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:46:46.943596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1046,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423114+0000","last_change":"2024-08-29T15:49:45.428512+0000","last_active":"2024-08-29T15:50:52.423114+0000","last_peered":"2024-08-29T15:50:52.423114+0000","last_clean":"2024-08-29T15:50:52.423114+0000","last_became_active":"2024-08-29T15:49:45.428171+0000","last_became_peered":"2024-08-29T15:49:45.428171+0000","last_unstale":"2024-08-29T15:50:52.423114+0000","last_undegraded":"2024-08-29T15:50:52.423114+0000","last_fullsized":"2024-08-29T15:50:52.423114+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:44:00.494227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":544,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306434+0000","last_change":"2024-08-29T15:50:31.977180+0000","last_active":"2024-08-29T15:50:51.306434+0000","last_peered":"2024-08-29T15:50:51.306434+0000","last_clean":"2024-08-29T15:50:51.306434+0000","last_became_active":"2024-08-29T15:50:31.763055+0000","last_became_peered":"2024-08-29T15:50:31.763055+0000","last_unstale":"2024-08-29T15:50:51.306434+0000","last_undegraded":"2024-08-29T15:50:51.306434+0000","last_fullsized":"2024-08-29T15:50:51.306434+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:31.977082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:31.977082+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:45:00.646384+0000","scrub_duration":0.001873375,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":170,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422791+0000","last_change":"2024-08-29T15:49:45.428070+0000","last_active":"2024-08-29T15:50:52.422791+0000","last_peered":"2024-08-29T15:50:52.422791+0000","last_clean":"2024-08-29T15:50:52.422791+0000","last_became_active":"2024-08-29T15:49:45.427658+0000","last_became_peered":"2024-08-29T15:49:45.427658+0000","last_unstale":"2024-08-29T15:50:52.422791+0000","last_undegraded":"2024-08-29T15:50:52.422791+0000","last_fullsized":"2024-08-29T15:50:52.422791+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:51:32.357331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1450,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423231+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:50:52.423231+0000","last_peered":"2024-08-29T15:50:52.423231+0000","last_clean":"2024-08-29T15:50:52.423231+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:50:52.423231+0000","last_undegraded":"2024-08-29T15:50:52.423231+0000","last_fullsized":"2024-08-29T15:50:52.423231+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301241+0000","last_change":"2024-08-29T15:49:45.426855+0000","last_active":"2024-08-29T15:50:51.301241+0000","last_peered":"2024-08-29T15:50:51.301241+0000","last_clean":"2024-08-29T15:50:51.301241+0000","last_became_active":"2024-08-29T15:49:45.425903+0000","last_became_peered":"2024-08-29T15:49:45.425903+0000","last_unstale":"2024-08-29T15:50:51.301241+0000","last_undegraded":"2024-08-29T15:50:51.301241+0000","last_fullsized":"2024-08-29T15:50:51.301241+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:16:10.531855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":544,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306481+0000","last_change":"2024-08-29T15:50:33.921689+0000","last_active":"2024-08-29T15:50:51.306481+0000","last_peered":"2024-08-29T15:50:51.306481+0000","last_clean":"2024-08-29T15:50:51.306481+0000","last_became_active":"2024-08-29T15:50:31.762574+0000","last_became_peered":"2024-08-29T15:50:31.762574+0000","last_unstale":"2024-08-29T15:50:51.306481+0000","last_undegraded":"2024-08-29T15:50:51.306481+0000","last_fullsized":"2024-08-29T15:50:51.306481+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:33.921562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:33.921562+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:53:21.195720+0000","scrub_duration":0.0033730650000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422871+0000","last_change":"2024-08-29T15:49:23.300350+0000","last_active":"2024-08-29T15:50:52.422871+0000","last_peered":"2024-08-29T15:50:52.422871+0000","last_clean":"2024-08-29T15:50:52.422871+0000","last_became_active":"2024-08-29T15:49:23.295545+0000","last_became_peered":"2024-08-29T15:49:23.295545+0000","last_unstale":"2024-08-29T15:50:52.422871+0000","last_undegraded":"2024-08-29T15:50:52.422871+0000","last_fullsized":"2024-08-29T15:50:52.422871+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:03:11.367343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":364,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307656+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:50:52.307656+0000","last_peered":"2024-08-29T15:50:52.307656+0000","last_clean":"2024-08-29T15:50:52.307656+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:50:52.307656+0000","last_undegraded":"2024-08-29T15:50:52.307656+0000","last_fullsized":"2024-08-29T15:50:52.307656+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":55,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300714+0000","last_change":"2024-08-29T15:49:45.445767+0000","last_active":"2024-08-29T15:50:51.300714+0000","last_peered":"2024-08-29T15:50:51.300714+0000","last_clean":"2024-08-29T15:50:51.300714+0000","last_became_active":"2024-08-29T15:49:45.426097+0000","last_became_peered":"2024-08-29T15:49:45.426097+0000","last_unstale":"2024-08-29T15:50:51.300714+0000","last_undegraded":"2024-08-29T15:50:51.300714+0000","last_fullsized":"2024-08-29T15:50:51.300714+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:04.260523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":967,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300688+0000","last_change":"2024-08-29T15:49:48.295920+0000","last_active":"2024-08-29T15:50:51.300688+0000","last_peered":"2024-08-29T15:50:51.300688+0000","last_clean":"2024-08-29T15:50:51.300688+0000","last_became_active":"2024-08-29T15:49:48.294838+0000","last_became_peered":"2024-08-29T15:49:48.294838+0000","last_unstale":"2024-08-29T15:50:51.300688+0000","last_undegraded":"2024-08-29T15:50:51.300688+0000","last_fullsized":"2024-08-29T15:50:51.300688+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_clean_scrub_stamp":"2024-08-29T15:44:26.709569+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:48:16.461251+0000","scrub_duration":0.0031838119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":170,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423216+0000","last_change":"2024-08-29T15:49:45.428165+0000","last_active":"2024-08-29T15:50:52.423216+0000","last_peered":"2024-08-29T15:50:52.423216+0000","last_clean":"2024-08-29T15:50:52.423216+0000","last_became_active":"2024-08-29T15:49:45.427816+0000","last_became_peered":"2024-08-29T15:49:45.427816+0000","last_unstale":"2024-08-29T15:50:52.423216+0000","last_undegraded":"2024-08-29T15:50:52.423216+0000","last_fullsized":"2024-08-29T15:50:52.423216+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:05:12.207096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":320,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306549+0000","last_change":"2024-08-29T15:50:37.805399+0000","last_active":"2024-08-29T15:50:51.306549+0000","last_peered":"2024-08-29T15:50:51.306549+0000","last_clean":"2024-08-29T15:50:51.306549+0000","last_became_active":"2024-08-29T15:50:37.805012+0000","last_became_peered":"2024-08-29T15:50:37.805012+0000","last_unstale":"2024-08-29T15:50:51.306549+0000","last_undegraded":"2024-08-29T15:50:51.306549+0000","last_fullsized":"2024-08-29T15:50:51.306549+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:48:41.571339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":638,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.308427+0000","last_change":"2024-08-29T15:50:52.308327+0000","last_active":"2024-08-29T15:50:52.308427+0000","last_peered":"2024-08-29T15:50:52.308427+0000","last_clean":"2024-08-29T15:50:52.308427+0000","last_became_active":"2024-08-29T15:50:52.307746+0000","last_became_peered":"2024-08-29T15:50:52.307746+0000","last_unstale":"2024-08-29T15:50:52.308427+0000","last_undegraded":"2024-08-29T15:50:52.308427+0000","last_fullsized":"2024-08-29T15:50:52.308427+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:46.458367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":1082,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306921+0000","last_change":"2024-08-29T15:50:37.807890+0000","last_active":"2024-08-29T15:50:51.306921+0000","last_peered":"2024-08-29T15:50:51.306921+0000","last_clean":"2024-08-29T15:50:51.306921+0000","last_became_active":"2024-08-29T15:50:37.807557+0000","last_became_peered":"2024-08-29T15:50:37.807557+0000","last_unstale":"2024-08-29T15:50:51.306921+0000","last_undegraded":"2024-08-29T15:50:51.306921+0000","last_fullsized":"2024-08-29T15:50:51.306921+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.841106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.841106+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:36:06.253219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":168,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307241+0000","last_change":"2024-08-29T15:46:58.369628+0000","last_active":"2024-08-29T15:50:52.307241+0000","last_peered":"2024-08-29T15:50:52.307241+0000","last_clean":"2024-08-29T15:50:52.307241+0000","last_became_active":"2024-08-29T15:46:58.369315+0000","last_became_peered":"2024-08-29T15:46:58.369315+0000","last_unstale":"2024-08-29T15:50:52.307241+0000","last_undegraded":"2024-08-29T15:50:52.307241+0000","last_fullsized":"2024-08-29T15:50:52.307241+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:52:05.895354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1052,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306684+0000","last_change":"2024-08-29T15:47:42.288211+0000","last_active":"2024-08-29T15:50:52.306684+0000","last_peered":"2024-08-29T15:50:52.306684+0000","last_clean":"2024-08-29T15:50:52.306684+0000","last_became_active":"2024-08-29T15:47:42.287594+0000","last_became_peered":"2024-08-29T15:47:42.287594+0000","last_unstale":"2024-08-29T15:50:52.306684+0000","last_undegraded":"2024-08-29T15:50:52.306684+0000","last_fullsized":"2024-08-29T15:50:52.306684+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:32:18.869863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":919,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306862+0000","last_change":"2024-08-29T15:50:37.809494+0000","last_active":"2024-08-29T15:50:51.306862+0000","last_peered":"2024-08-29T15:50:51.306862+0000","last_clean":"2024-08-29T15:50:51.306862+0000","last_became_active":"2024-08-29T15:50:37.808844+0000","last_became_peered":"2024-08-29T15:50:37.808844+0000","last_unstale":"2024-08-29T15:50:51.306862+0000","last_undegraded":"2024-08-29T15:50:51.306862+0000","last_fullsized":"2024-08-29T15:50:51.306862+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:31:24.232827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":1165,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423037+0000","last_change":"2024-08-29T15:43:26.588054+0000","last_active":"2024-08-29T15:50:52.423037+0000","last_peered":"2024-08-29T15:50:52.423037+0000","last_clean":"2024-08-29T15:50:52.423037+0000","last_became_active":"2024-08-29T15:43:26.285267+0000","last_became_peered":"2024-08-29T15:43:26.285267+0000","last_unstale":"2024-08-29T15:50:52.423037+0000","last_undegraded":"2024-08-29T15:50:52.423037+0000","last_fullsized":"2024-08-29T15:50:52.423037+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.587986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.587986+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:15:21.198019+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307061+0000","last_change":"2024-08-29T15:50:31.767001+0000","last_active":"2024-08-29T15:50:51.307061+0000","last_peered":"2024-08-29T15:50:51.307061+0000","last_clean":"2024-08-29T15:50:51.307061+0000","last_became_active":"2024-08-29T15:50:31.766580+0000","last_became_peered":"2024-08-29T15:50:31.766580+0000","last_unstale":"2024-08-29T15:50:51.307061+0000","last_undegraded":"2024-08-29T15:50:51.307061+0000","last_fullsized":"2024-08-29T15:50:51.307061+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:14.735883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":440,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304786+0000","last_change":"2024-08-29T15:49:45.426244+0000","last_active":"2024-08-29T15:50:52.304786+0000","last_peered":"2024-08-29T15:50:52.304786+0000","last_clean":"2024-08-29T15:50:52.304786+0000","last_became_active":"2024-08-29T15:49:45.425976+0000","last_became_peered":"2024-08-29T15:49:45.425976+0000","last_unstale":"2024-08-29T15:50:52.304786+0000","last_undegraded":"2024-08-29T15:50:52.304786+0000","last_fullsized":"2024-08-29T15:50:52.304786+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:01:11.296031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":705,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300932+0000","last_change":"2024-08-29T15:49:48.294805+0000","last_active":"2024-08-29T15:50:51.300932+0000","last_peered":"2024-08-29T15:50:51.300932+0000","last_clean":"2024-08-29T15:50:51.300932+0000","last_became_active":"2024-08-29T15:49:48.294065+0000","last_became_peered":"2024-08-29T15:49:48.294065+0000","last_unstale":"2024-08-29T15:50:51.300932+0000","last_undegraded":"2024-08-29T15:50:51.300932+0000","last_fullsized":"2024-08-29T15:50:51.300932+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:23:34.652252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":581,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306957+0000","last_change":"2024-08-29T15:50:40.404619+0000","last_active":"2024-08-29T15:50:51.306957+0000","last_peered":"2024-08-29T15:50:51.306957+0000","last_clean":"2024-08-29T15:50:51.306957+0000","last_became_active":"2024-08-29T15:50:37.808260+0000","last_became_peered":"2024-08-29T15:50:37.808260+0000","last_unstale":"2024-08-29T15:50:51.306957+0000","last_undegraded":"2024-08-29T15:50:51.306957+0000","last_fullsized":"2024-08-29T15:50:51.306957+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:40.404512+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:40.404512+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:56:13.286215+0000","scrub_duration":0.001575943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1012,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423329+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:50:52.423329+0000","last_peered":"2024-08-29T15:50:52.423329+0000","last_clean":"2024-08-29T15:50:52.423329+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:50:52.423329+0000","last_undegraded":"2024-08-29T15:50:52.423329+0000","last_fullsized":"2024-08-29T15:50:52.423329+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":146,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304581+0000","last_change":"2024-08-29T15:47:43.258937+0000","last_active":"2024-08-29T15:50:52.304581+0000","last_peered":"2024-08-29T15:50:52.304581+0000","last_clean":"2024-08-29T15:50:52.304581+0000","last_became_active":"2024-08-29T15:47:43.258572+0000","last_became_peered":"2024-08-29T15:47:43.258572+0000","last_unstale":"2024-08-29T15:50:52.304581+0000","last_undegraded":"2024-08-29T15:50:52.304581+0000","last_fullsized":"2024-08-29T15:50:52.304581+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:58.220504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":752,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300987+0000","last_change":"2024-08-29T15:49:50.423039+0000","last_active":"2024-08-29T15:50:51.300987+0000","last_peered":"2024-08-29T15:50:51.300987+0000","last_clean":"2024-08-29T15:50:51.300987+0000","last_became_active":"2024-08-29T15:49:48.294243+0000","last_became_peered":"2024-08-29T15:49:48.294243+0000","last_unstale":"2024-08-29T15:50:51.300987+0000","last_undegraded":"2024-08-29T15:50:51.300987+0000","last_fullsized":"2024-08-29T15:50:51.300987+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:50.422956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:50.422956+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:31:25.265825+0000","scrub_duration":0.0026719690000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":604,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303659+0000","last_change":"2024-08-29T15:47:12.467689+0000","last_active":"2024-08-29T15:50:51.303659+0000","last_peered":"2024-08-29T15:50:51.303659+0000","last_clean":"2024-08-29T15:50:51.303659+0000","last_became_active":"2024-08-29T15:47:12.467400+0000","last_became_peered":"2024-08-29T15:47:12.467400+0000","last_unstale":"2024-08-29T15:50:51.303659+0000","last_undegraded":"2024-08-29T15:50:51.303659+0000","last_fullsized":"2024-08-29T15:50:51.303659+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:45:54.623271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":363,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.302970+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:50:51.302970+0000","last_peered":"2024-08-29T15:50:51.302970+0000","last_clean":"2024-08-29T15:50:51.302970+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:50:51.302970+0000","last_undegraded":"2024-08-29T15:50:51.302970+0000","last_fullsized":"2024-08-29T15:50:51.302970+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":1207,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422740+0000","last_change":"2024-08-29T15:49:48.295197+0000","last_active":"2024-08-29T15:50:52.422740+0000","last_peered":"2024-08-29T15:50:52.422740+0000","last_clean":"2024-08-29T15:50:52.422740+0000","last_became_active":"2024-08-29T15:49:48.294704+0000","last_became_peered":"2024-08-29T15:49:48.294704+0000","last_unstale":"2024-08-29T15:50:52.422740+0000","last_undegraded":"2024-08-29T15:50:52.422740+0000","last_fullsized":"2024-08-29T15:50:52.422740+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.906140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.906140+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:24:17.805166+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":310,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303596+0000","last_change":"2024-08-29T15:50:32.967030+0000","last_active":"2024-08-29T15:50:51.303596+0000","last_peered":"2024-08-29T15:50:51.303596+0000","last_clean":"2024-08-29T15:50:51.303596+0000","last_became_active":"2024-08-29T15:50:32.966651+0000","last_became_peered":"2024-08-29T15:50:32.966651+0000","last_unstale":"2024-08-29T15:50:51.303596+0000","last_undegraded":"2024-08-29T15:50:51.303596+0000","last_fullsized":"2024-08-29T15:50:51.303596+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:12:41.521139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":306,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306259+0000","last_change":"2024-08-29T15:50:31.766219+0000","last_active":"2024-08-29T15:50:51.306259+0000","last_peered":"2024-08-29T15:50:51.306259+0000","last_clean":"2024-08-29T15:50:51.306259+0000","last_became_active":"2024-08-29T15:50:31.765918+0000","last_became_peered":"2024-08-29T15:50:31.765918+0000","last_unstale":"2024-08-29T15:50:51.306259+0000","last_undegraded":"2024-08-29T15:50:51.306259+0000","last_fullsized":"2024-08-29T15:50:51.306259+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:36:38.266764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":766,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300853+0000","last_change":"2024-08-29T15:49:52.412907+0000","last_active":"2024-08-29T15:50:51.300853+0000","last_peered":"2024-08-29T15:50:51.300853+0000","last_clean":"2024-08-29T15:50:51.300853+0000","last_became_active":"2024-08-29T15:49:48.293548+0000","last_became_peered":"2024-08-29T15:49:48.293548+0000","last_unstale":"2024-08-29T15:50:51.300853+0000","last_undegraded":"2024-08-29T15:50:51.300853+0000","last_fullsized":"2024-08-29T15:50:51.300853+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:52.412798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:52.412798+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:22:05.653553+0000","scrub_duration":0.0015053149999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":301,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303217+0000","last_change":"2024-08-29T15:50:25.955631+0000","last_active":"2024-08-29T15:50:51.303217+0000","last_peered":"2024-08-29T15:50:51.303217+0000","last_clean":"2024-08-29T15:50:51.303217+0000","last_became_active":"2024-08-29T15:50:25.955245+0000","last_became_peered":"2024-08-29T15:50:25.955245+0000","last_unstale":"2024-08-29T15:50:51.303217+0000","last_undegraded":"2024-08-29T15:50:51.303217+0000","last_fullsized":"2024-08-29T15:50:51.303217+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:36:14.398486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1061,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304995+0000","last_change":"2024-08-29T15:46:58.377694+0000","last_active":"2024-08-29T15:50:52.304995+0000","last_peered":"2024-08-29T15:50:52.304995+0000","last_clean":"2024-08-29T15:50:52.304995+0000","last_became_active":"2024-08-29T15:46:58.372090+0000","last_became_peered":"2024-08-29T15:46:58.372090+0000","last_unstale":"2024-08-29T15:50:52.304995+0000","last_undegraded":"2024-08-29T15:50:52.304995+0000","last_fullsized":"2024-08-29T15:50:52.304995+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:19:50.948014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":581,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306567+0000","last_change":"2024-08-29T15:50:39.443241+0000","last_active":"2024-08-29T15:50:51.306567+0000","last_peered":"2024-08-29T15:50:51.306567+0000","last_clean":"2024-08-29T15:50:51.306567+0000","last_became_active":"2024-08-29T15:50:37.804640+0000","last_became_peered":"2024-08-29T15:50:37.804640+0000","last_unstale":"2024-08-29T15:50:51.306567+0000","last_undegraded":"2024-08-29T15:50:51.306567+0000","last_fullsized":"2024-08-29T15:50:51.306567+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:39.443139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:39.443139+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:16:35.360223+0000","scrub_duration":0.0020546499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1022,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307255+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:50:52.307255+0000","last_peered":"2024-08-29T15:50:52.307255+0000","last_clean":"2024-08-29T15:50:52.307255+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:50:52.307255+0000","last_undegraded":"2024-08-29T15:50:52.307255+0000","last_fullsized":"2024-08-29T15:50:52.307255+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300961+0000","last_change":"2024-08-29T15:49:45.425648+0000","last_active":"2024-08-29T15:50:51.300961+0000","last_peered":"2024-08-29T15:50:51.300961+0000","last_clean":"2024-08-29T15:50:51.300961+0000","last_became_active":"2024-08-29T15:49:45.424290+0000","last_became_peered":"2024-08-29T15:49:45.424290+0000","last_unstale":"2024-08-29T15:50:51.300961+0000","last_undegraded":"2024-08-29T15:50:51.300961+0000","last_fullsized":"2024-08-29T15:50:51.300961+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:36:49.085868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":875,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303423+0000","last_change":"2024-08-29T15:45:25.871113+0000","last_active":"2024-08-29T15:50:51.303423+0000","last_peered":"2024-08-29T15:50:51.303423+0000","last_clean":"2024-08-29T15:50:51.303423+0000","last_became_active":"2024-08-29T15:44:59.246190+0000","last_became_peered":"2024-08-29T15:44:59.246190+0000","last_unstale":"2024-08-29T15:50:51.303423+0000","last_undegraded":"2024-08-29T15:50:51.303423+0000","last_fullsized":"2024-08-29T15:50:51.303423+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.870329+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.870329+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:42.080608+0000","scrub_duration":0.0042968349999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":172,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306229+0000","last_change":"2024-08-29T15:50:31.765145+0000","last_active":"2024-08-29T15:50:51.306229+0000","last_peered":"2024-08-29T15:50:51.306229+0000","last_clean":"2024-08-29T15:50:51.306229+0000","last_became_active":"2024-08-29T15:50:31.764814+0000","last_became_peered":"2024-08-29T15:50:31.764814+0000","last_unstale":"2024-08-29T15:50:51.306229+0000","last_undegraded":"2024-08-29T15:50:51.306229+0000","last_fullsized":"2024-08-29T15:50:51.306229+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:57:03.810863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":103,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307499+0000","last_change":"2024-08-29T15:50:52.307381+0000","last_active":"2024-08-29T15:50:52.307499+0000","last_peered":"2024-08-29T15:50:52.307499+0000","last_clean":"2024-08-29T15:50:52.307499+0000","last_became_active":"2024-08-29T15:50:52.306942+0000","last_became_peered":"2024-08-29T15:50:52.306942+0000","last_unstale":"2024-08-29T15:50:52.307499+0000","last_undegraded":"2024-08-29T15:50:52.307499+0000","last_fullsized":"2024-08-29T15:50:52.307499+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:47:54.851578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":544,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306730+0000","last_change":"2024-08-29T15:50:32.958662+0000","last_active":"2024-08-29T15:50:51.306730+0000","last_peered":"2024-08-29T15:50:51.306730+0000","last_clean":"2024-08-29T15:50:51.306730+0000","last_became_active":"2024-08-29T15:50:31.766024+0000","last_became_peered":"2024-08-29T15:50:31.766024+0000","last_unstale":"2024-08-29T15:50:51.306730+0000","last_undegraded":"2024-08-29T15:50:51.306730+0000","last_fullsized":"2024-08-29T15:50:51.306730+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:32.958535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:32.958535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:46:32.978131+0000","scrub_duration":0.00223508,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":351,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.302961+0000","last_change":"2024-08-29T15:50:25.444614+0000","last_active":"2024-08-29T15:50:51.302961+0000","last_peered":"2024-08-29T15:50:51.302961+0000","last_clean":"2024-08-29T15:50:51.302961+0000","last_became_active":"2024-08-29T15:50:25.444264+0000","last_became_peered":"2024-08-29T15:50:25.444264+0000","last_unstale":"2024-08-29T15:50:51.302961+0000","last_undegraded":"2024-08-29T15:50:51.302961+0000","last_fullsized":"2024-08-29T15:50:51.302961+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:36.513688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":201,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307672+0000","last_change":"2024-08-29T15:50:37.806818+0000","last_active":"2024-08-29T15:50:51.307672+0000","last_peered":"2024-08-29T15:50:51.307672+0000","last_clean":"2024-08-29T15:50:51.307672+0000","last_became_active":"2024-08-29T15:50:37.806392+0000","last_became_peered":"2024-08-29T15:50:37.806392+0000","last_unstale":"2024-08-29T15:50:51.307672+0000","last_undegraded":"2024-08-29T15:50:51.307672+0000","last_fullsized":"2024-08-29T15:50:51.307672+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:46:27.096632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'149","reported_seq":1226,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305110+0000","last_change":"2024-08-29T15:44:59.240657+0000","last_active":"2024-08-29T15:50:52.305110+0000","last_peered":"2024-08-29T15:50:52.305110+0000","last_clean":"2024-08-29T15:50:52.305110+0000","last_became_active":"2024-08-29T15:44:59.240261+0000","last_became_peered":"2024-08-29T15:44:59.240261+0000","last_unstale":"2024-08-29T15:50:52.305110+0000","last_undegraded":"2024-08-29T15:50:52.305110+0000","last_fullsized":"2024-08-29T15:50:52.305110+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.834998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.834998+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:47:15.459581+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300735+0000","last_change":"2024-08-29T15:49:45.426284+0000","last_active":"2024-08-29T15:50:51.300735+0000","last_peered":"2024-08-29T15:50:51.300735+0000","last_clean":"2024-08-29T15:50:51.300735+0000","last_became_active":"2024-08-29T15:49:45.425997+0000","last_became_peered":"2024-08-29T15:49:45.425997+0000","last_unstale":"2024-08-29T15:50:51.300735+0000","last_undegraded":"2024-08-29T15:50:51.300735+0000","last_fullsized":"2024-08-29T15:50:51.300735+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:16:22.675142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":200,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307722+0000","last_change":"2024-08-29T15:50:37.808758+0000","last_active":"2024-08-29T15:50:51.307722+0000","last_peered":"2024-08-29T15:50:51.307722+0000","last_clean":"2024-08-29T15:50:51.307722+0000","last_became_active":"2024-08-29T15:50:37.808419+0000","last_became_peered":"2024-08-29T15:50:37.808419+0000","last_unstale":"2024-08-29T15:50:51.307722+0000","last_undegraded":"2024-08-29T15:50:51.307722+0000","last_fullsized":"2024-08-29T15:50:51.307722+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:18.897463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":968,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307203+0000","last_change":"2024-08-29T15:50:37.807254+0000","last_active":"2024-08-29T15:50:51.307203+0000","last_peered":"2024-08-29T15:50:51.307203+0000","last_clean":"2024-08-29T15:50:51.307203+0000","last_became_active":"2024-08-29T15:50:37.806806+0000","last_became_peered":"2024-08-29T15:50:37.806806+0000","last_unstale":"2024-08-29T15:50:51.307203+0000","last_undegraded":"2024-08-29T15:50:51.307203+0000","last_fullsized":"2024-08-29T15:50:51.307203+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.799642+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.799642+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:09:01.034776+0000","scrub_duration":0.0012023820000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":235,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306197+0000","last_change":"2024-08-29T15:50:31.763671+0000","last_active":"2024-08-29T15:50:51.306197+0000","last_peered":"2024-08-29T15:50:51.306197+0000","last_clean":"2024-08-29T15:50:51.306197+0000","last_became_active":"2024-08-29T15:50:31.763107+0000","last_became_peered":"2024-08-29T15:50:31.763107+0000","last_unstale":"2024-08-29T15:50:51.306197+0000","last_undegraded":"2024-08-29T15:50:51.306197+0000","last_fullsized":"2024-08-29T15:50:51.306197+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:57.907456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":153,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305285+0000","last_change":"2024-08-29T15:48:12.268685+0000","last_active":"2024-08-29T15:50:52.305285+0000","last_peered":"2024-08-29T15:50:52.305285+0000","last_clean":"2024-08-29T15:50:52.305285+0000","last_became_active":"2024-08-29T15:48:12.268311+0000","last_became_peered":"2024-08-29T15:48:12.268311+0000","last_unstale":"2024-08-29T15:50:52.305285+0000","last_undegraded":"2024-08-29T15:50:52.305285+0000","last_fullsized":"2024-08-29T15:50:52.305285+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:30:18.347971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":1054,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300867+0000","last_change":"2024-08-29T15:49:48.295818+0000","last_active":"2024-08-29T15:50:51.300867+0000","last_peered":"2024-08-29T15:50:51.300867+0000","last_clean":"2024-08-29T15:50:51.300867+0000","last_became_active":"2024-08-29T15:49:48.294380+0000","last_became_peered":"2024-08-29T15:49:48.294380+0000","last_unstale":"2024-08-29T15:50:51.300867+0000","last_undegraded":"2024-08-29T15:50:51.300867+0000","last_fullsized":"2024-08-29T15:50:51.300867+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.831081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.831081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:06.262402+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":203,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306795+0000","last_change":"2024-08-29T15:50:37.805588+0000","last_active":"2024-08-29T15:50:51.306795+0000","last_peered":"2024-08-29T15:50:51.306795+0000","last_clean":"2024-08-29T15:50:51.306795+0000","last_became_active":"2024-08-29T15:50:37.805121+0000","last_became_peered":"2024-08-29T15:50:37.805121+0000","last_unstale":"2024-08-29T15:50:51.306795+0000","last_undegraded":"2024-08-29T15:50:51.306795+0000","last_fullsized":"2024-08-29T15:50:51.306795+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:37:58.234694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307192+0000","last_change":"2024-08-29T15:48:12.267269+0000","last_active":"2024-08-29T15:50:52.307192+0000","last_peered":"2024-08-29T15:50:52.307192+0000","last_clean":"2024-08-29T15:50:52.307192+0000","last_became_active":"2024-08-29T15:48:12.266893+0000","last_became_peered":"2024-08-29T15:48:12.266893+0000","last_unstale":"2024-08-29T15:50:52.307192+0000","last_undegraded":"2024-08-29T15:50:52.307192+0000","last_fullsized":"2024-08-29T15:50:52.307192+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:07:31.630164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":1182,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422907+0000","last_change":"2024-08-29T15:47:46.309647+0000","last_active":"2024-08-29T15:50:52.422907+0000","last_peered":"2024-08-29T15:50:52.422907+0000","last_clean":"2024-08-29T15:50:52.422907+0000","last_became_active":"2024-08-29T15:47:46.309290+0000","last_became_peered":"2024-08-29T15:47:46.309290+0000","last_unstale":"2024-08-29T15:50:52.422907+0000","last_undegraded":"2024-08-29T15:50:52.422907+0000","last_fullsized":"2024-08-29T15:50:52.422907+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.615446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.615446+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:58.511333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":167,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306600+0000","last_change":"2024-08-29T15:46:58.374246+0000","last_active":"2024-08-29T15:50:52.306600+0000","last_peered":"2024-08-29T15:50:52.306600+0000","last_clean":"2024-08-29T15:50:52.306600+0000","last_became_active":"2024-08-29T15:46:58.373708+0000","last_became_peered":"2024-08-29T15:46:58.373708+0000","last_unstale":"2024-08-29T15:50:52.306600+0000","last_undegraded":"2024-08-29T15:50:52.306600+0000","last_fullsized":"2024-08-29T15:50:52.306600+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:27.833391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301260+0000","last_change":"2024-08-29T15:49:45.426791+0000","last_active":"2024-08-29T15:50:51.301260+0000","last_peered":"2024-08-29T15:50:51.301260+0000","last_clean":"2024-08-29T15:50:51.301260+0000","last_became_active":"2024-08-29T15:49:45.426343+0000","last_became_peered":"2024-08-29T15:49:45.426343+0000","last_unstale":"2024-08-29T15:50:51.301260+0000","last_undegraded":"2024-08-29T15:50:51.301260+0000","last_fullsized":"2024-08-29T15:50:51.301260+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:15:36.607053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":1085,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300623+0000","last_change":"2024-08-29T15:49:48.296814+0000","last_active":"2024-08-29T15:50:51.300623+0000","last_peered":"2024-08-29T15:50:51.300623+0000","last_clean":"2024-08-29T15:50:51.300623+0000","last_became_active":"2024-08-29T15:49:48.295396+0000","last_became_peered":"2024-08-29T15:49:48.295396+0000","last_unstale":"2024-08-29T15:50:51.300623+0000","last_undegraded":"2024-08-29T15:50:51.300623+0000","last_fullsized":"2024-08-29T15:50:51.300623+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.968809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.968809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:36:15.745876+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":319,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306885+0000","last_change":"2024-08-29T15:50:37.806791+0000","last_active":"2024-08-29T15:50:52.306885+0000","last_peered":"2024-08-29T15:50:52.306885+0000","last_clean":"2024-08-29T15:50:52.306885+0000","last_became_active":"2024-08-29T15:50:37.806406+0000","last_became_peered":"2024-08-29T15:50:37.806406+0000","last_unstale":"2024-08-29T15:50:52.306885+0000","last_undegraded":"2024-08-29T15:50:52.306885+0000","last_fullsized":"2024-08-29T15:50:52.306885+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:11:48.207353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1070,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305286+0000","last_change":"2024-08-29T15:47:12.481794+0000","last_active":"2024-08-29T15:50:52.305286+0000","last_peered":"2024-08-29T15:50:52.305286+0000","last_clean":"2024-08-29T15:50:52.305286+0000","last_became_active":"2024-08-29T15:47:12.480977+0000","last_became_peered":"2024-08-29T15:47:12.480977+0000","last_unstale":"2024-08-29T15:50:52.305286+0000","last_undegraded":"2024-08-29T15:50:52.305286+0000","last_fullsized":"2024-08-29T15:50:52.305286+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:29.573851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'149","reported_seq":1056,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306737+0000","last_change":"2024-08-29T15:49:48.295774+0000","last_active":"2024-08-29T15:50:52.306737+0000","last_peered":"2024-08-29T15:50:52.306737+0000","last_clean":"2024-08-29T15:50:52.306737+0000","last_became_active":"2024-08-29T15:49:48.295060+0000","last_became_peered":"2024-08-29T15:49:48.295060+0000","last_unstale":"2024-08-29T15:50:52.306737+0000","last_undegraded":"2024-08-29T15:50:52.306737+0000","last_fullsized":"2024-08-29T15:50:52.306737+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:36.699540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:36.699540+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:14:30.458306+0000","scrub_duration":0.0014941869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":8562,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":16777254,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234684,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909744,"kb_used_data":12320,"kb_used_omap":1,"kb_used_meta":897406,"kb_avail":92835408,"statfs":{"total":95995035648,"available":95063457792,"internally_reserved":0,"allocated":12615680,"data_stored":9073115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":918944203},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496315,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":707172,"kb_used_data":8276,"kb_used_omap":1,"kb_used_meta":698878,"kb_avail":93037980,"statfs":{"total":95995035648,"available":95270891520,"internally_reserved":0,"allocated":8474624,"data_stored":4943473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":715651527},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426903,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640864,"kb_used_data":8208,"kb_used_omap":8,"kb_used_meta":632631,"kb_avail":93104288,"statfs":{"total":95995035648,"available":95338790912,"internally_reserved":0,"allocated":8404992,"data_stored":4874249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":647815050},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019594,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":971812,"kb_used_data":12056,"kb_used_omap":1,"kb_used_meta":959742,"kb_avail":92773340,"statfs":{"total":95995035648,"available":94999900160,"internally_reserved":0,"allocated":12345344,"data_stored":8810154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":982776262},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435779,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004760,"kb_used_data":8264,"kb_used_omap":4,"kb_used_meta":996475,"kb_avail":92740392,"statfs":{"total":95995035648,"available":94966161408,"internally_reserved":0,"allocated":8462336,"data_stored":4939003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":1020390665},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903631,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":793812,"kb_used_data":8200,"kb_used_omap":10,"kb_used_meta":785589,"kb_avail":92951340,"statfs":{"total":95995035648,"available":95182172160,"internally_reserved":0,"allocated":8396800,"data_stored":4873357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":804443664},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762832,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":992020,"kb_used_data":7940,"kb_used_omap":7,"kb_used_meta":984056,"kb_avail":92753132,"statfs":{"total":95995035648,"available":94979207168,"internally_reserved":0,"allocated":8130560,"data_stored":4610107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":1007673698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926355,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":936716,"kb_used_data":12356,"kb_used_omap":7,"kb_used_meta":924344,"kb_avail":92808436,"statfs":{"total":95995035648,"available":95035838464,"internally_reserved":0,"allocated":12652544,"data_stored":9132380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":946528399},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:51:40.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid af03926c-6615-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-29T15:51:40.440 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/mon.a/config 2024-08-29T15:51:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:40 smithi171 ceph-mon[90031]: pgmap v1991: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:40.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:40 smithi171 ceph-mon[90031]: from='client.47558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:51:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:40 smithi027 ceph-mon[93921]: pgmap v1991: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:40 smithi027 ceph-mon[93921]: from='client.47558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:51:40.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:40 smithi027 ceph-mon[98970]: pgmap v1991: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:40.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:40 smithi027 ceph-mon[98970]: from='client.47558 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:51:42.350 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-08-29T15:51:42.351 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-08-29T15:51:42.669 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:42 smithi027 ceph-mon[98970]: pgmap v1992: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:42.669 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:42 smithi027 ceph-mon[93921]: pgmap v1992: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:43.017 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:42 smithi171 ceph-mon[90031]: pgmap v1992: 103 pgs: 103 active+clean; 4.6 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-29T15:51:43.136 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":1992,"stamp":"2024-08-29T15:51:41.237756+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":908,"num_write_kb":12625,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":204,"num_bytes_recovered":17376184,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5652,"ondisk_log_size":5652,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6956900,"kb_used_data":77620,"kb_used_omap":43,"kb_used_meta":6879124,"kb_avail":743004316,"statfs":{"total":767960285184,"available":760836419584,"internally_reserved":0,"allocated":79482880,"data_stored":51255838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44564,"internal_metadata":7044223468},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003054"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":200,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307600+0000","last_change":"2024-08-29T15:49:45.432721+0000","last_active":"2024-08-29T15:50:52.307600+0000","last_peered":"2024-08-29T15:50:52.307600+0000","last_clean":"2024-08-29T15:50:52.307600+0000","last_became_active":"2024-08-29T15:49:45.425230+0000","last_became_peered":"2024-08-29T15:49:45.425230+0000","last_unstale":"2024-08-29T15:50:52.307600+0000","last_undegraded":"2024-08-29T15:50:52.307600+0000","last_fullsized":"2024-08-29T15:50:52.307600+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:57:53.189694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3492,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":362,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306284+0000","last_change":"2024-08-29T15:19:46.805323+0000","last_active":"2024-08-29T15:50:52.306284+0000","last_peered":"2024-08-29T15:50:52.306284+0000","last_clean":"2024-08-29T15:50:52.306284+0000","last_became_active":"2024-08-29T15:19:46.805004+0000","last_became_peered":"2024-08-29T15:19:46.805004+0000","last_unstale":"2024-08-29T15:50:52.306284+0000","last_undegraded":"2024-08-29T15:50:52.306284+0000","last_fullsized":"2024-08-29T15:50:52.306284+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:54:10.947346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'149","reported_seq":1001,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423087+0000","last_change":"2024-08-29T15:47:46.316018+0000","last_active":"2024-08-29T15:50:52.423087+0000","last_peered":"2024-08-29T15:50:52.423087+0000","last_clean":"2024-08-29T15:50:52.423087+0000","last_became_active":"2024-08-29T15:47:46.315610+0000","last_became_peered":"2024-08-29T15:47:46.315610+0000","last_unstale":"2024-08-29T15:50:52.423087+0000","last_undegraded":"2024-08-29T15:50:52.423087+0000","last_fullsized":"2024-08-29T15:50:52.423087+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:24.970456+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:45:24.970456+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T03:30:31.344363+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1051,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305798+0000","last_change":"2024-08-29T15:18:34.986629+0000","last_active":"2024-08-29T15:50:52.305798+0000","last_peered":"2024-08-29T15:50:52.305798+0000","last_clean":"2024-08-29T15:50:52.305798+0000","last_became_active":"2024-08-29T15:18:34.986261+0000","last_became_peered":"2024-08-29T15:18:34.986261+0000","last_unstale":"2024-08-29T15:50:52.305798+0000","last_undegraded":"2024-08-29T15:50:52.305798+0000","last_fullsized":"2024-08-29T15:50:52.305798+0000","mapping_epoch":218,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":219,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:32:16.531082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1047,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305334+0000","last_change":"2024-08-29T15:18:33.993841+0000","last_active":"2024-08-29T15:50:52.305334+0000","last_peered":"2024-08-29T15:50:52.305334+0000","last_clean":"2024-08-29T15:50:52.305334+0000","last_became_active":"2024-08-29T15:18:33.992421+0000","last_became_peered":"2024-08-29T15:18:33.992421+0000","last_unstale":"2024-08-29T15:50:52.305334+0000","last_undegraded":"2024-08-29T15:50:52.305334+0000","last_fullsized":"2024-08-29T15:50:52.305334+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:04:01.819675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"90'149","reported_seq":581,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307882+0000","last_change":"2024-08-29T15:50:47.383348+0000","last_active":"2024-08-29T15:50:51.307882+0000","last_peered":"2024-08-29T15:50:51.307882+0000","last_clean":"2024-08-29T15:50:51.307882+0000","last_became_active":"2024-08-29T15:50:37.804811+0000","last_became_peered":"2024-08-29T15:50:37.804811+0000","last_unstale":"2024-08-29T15:50:51.307882+0000","last_undegraded":"2024-08-29T15:50:51.307882+0000","last_fullsized":"2024-08-29T15:50:51.307882+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:47.383216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:47.383216+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:54:29.268648+0000","scrub_duration":0.0017543280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":41,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301359+0000","last_change":"2024-08-29T15:49:45.433633+0000","last_active":"2024-08-29T15:50:51.301359+0000","last_peered":"2024-08-29T15:50:51.301359+0000","last_clean":"2024-08-29T15:50:51.301359+0000","last_became_active":"2024-08-29T15:49:45.424864+0000","last_became_peered":"2024-08-29T15:49:45.424864+0000","last_unstale":"2024-08-29T15:50:51.301359+0000","last_undegraded":"2024-08-29T15:50:51.301359+0000","last_fullsized":"2024-08-29T15:50:51.301359+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:37:00.817911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":300,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303194+0000","last_change":"2024-08-29T15:50:25.624548+0000","last_active":"2024-08-29T15:50:51.303194+0000","last_peered":"2024-08-29T15:50:51.303194+0000","last_clean":"2024-08-29T15:50:51.303194+0000","last_became_active":"2024-08-29T15:50:25.624279+0000","last_became_peered":"2024-08-29T15:50:25.624279+0000","last_unstale":"2024-08-29T15:50:51.303194+0000","last_undegraded":"2024-08-29T15:50:51.303194+0000","last_fullsized":"2024-08-29T15:50:51.303194+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:24:27.488131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":41,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301148+0000","last_change":"2024-08-29T15:49:45.431781+0000","last_active":"2024-08-29T15:50:51.301148+0000","last_peered":"2024-08-29T15:50:51.301148+0000","last_clean":"2024-08-29T15:50:51.301148+0000","last_became_active":"2024-08-29T15:49:45.422282+0000","last_became_peered":"2024-08-29T15:49:45.422282+0000","last_unstale":"2024-08-29T15:50:51.301148+0000","last_undegraded":"2024-08-29T15:50:51.301148+0000","last_fullsized":"2024-08-29T15:50:51.301148+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:53:33.408668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":397,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303206+0000","last_change":"2024-08-29T15:47:12.481314+0000","last_active":"2024-08-29T15:50:51.303206+0000","last_peered":"2024-08-29T15:50:51.303206+0000","last_clean":"2024-08-29T15:50:51.303206+0000","last_became_active":"2024-08-29T15:47:12.481010+0000","last_became_peered":"2024-08-29T15:47:12.481010+0000","last_unstale":"2024-08-29T15:50:51.303206+0000","last_undegraded":"2024-08-29T15:50:51.303206+0000","last_fullsized":"2024-08-29T15:50:51.303206+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:58:04.897625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"90'149","reported_seq":1030,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305251+0000","last_change":"2024-08-29T15:44:59.241503+0000","last_active":"2024-08-29T15:50:52.305251+0000","last_peered":"2024-08-29T15:50:52.305251+0000","last_clean":"2024-08-29T15:50:52.305251+0000","last_became_active":"2024-08-29T15:44:59.241088+0000","last_became_peered":"2024-08-29T15:44:59.241088+0000","last_unstale":"2024-08-29T15:50:52.305251+0000","last_undegraded":"2024-08-29T15:50:52.305251+0000","last_fullsized":"2024-08-29T15:50:52.305251+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:27.788587+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:27.788587+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:39.021866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":167,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306834+0000","last_change":"2024-08-29T15:46:58.367038+0000","last_active":"2024-08-29T15:50:52.306834+0000","last_peered":"2024-08-29T15:50:52.306834+0000","last_clean":"2024-08-29T15:50:52.306834+0000","last_became_active":"2024-08-29T15:46:58.366692+0000","last_became_peered":"2024-08-29T15:46:58.366692+0000","last_unstale":"2024-08-29T15:50:52.306834+0000","last_undegraded":"2024-08-29T15:50:52.306834+0000","last_fullsized":"2024-08-29T15:50:52.306834+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:40:02.240790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":988,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.424328+0000","last_change":"2024-08-29T15:50:52.424216+0000","last_active":"2024-08-29T15:50:52.424328+0000","last_peered":"2024-08-29T15:50:52.424328+0000","last_clean":"2024-08-29T15:50:52.424328+0000","last_became_active":"2024-08-29T15:50:52.423829+0000","last_became_peered":"2024-08-29T15:50:52.423829+0000","last_unstale":"2024-08-29T15:50:52.424328+0000","last_undegraded":"2024-08-29T15:50:52.424328+0000","last_fullsized":"2024-08-29T15:50:52.424328+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:30.219791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"90'149","reported_seq":1184,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305621+0000","last_change":"2024-08-29T15:47:01.424749+0000","last_active":"2024-08-29T15:50:52.305621+0000","last_peered":"2024-08-29T15:50:52.305621+0000","last_clean":"2024-08-29T15:50:52.305621+0000","last_became_active":"2024-08-29T15:47:01.424333+0000","last_became_peered":"2024-08-29T15:47:01.424333+0000","last_unstale":"2024-08-29T15:50:52.305621+0000","last_undegraded":"2024-08-29T15:50:52.305621+0000","last_fullsized":"2024-08-29T15:50:52.305621+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:31.851590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:31.851590+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:54:01.272901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":361,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303160+0000","last_change":"2024-08-29T15:19:46.806954+0000","last_active":"2024-08-29T15:50:51.303160+0000","last_peered":"2024-08-29T15:50:51.303160+0000","last_clean":"2024-08-29T15:50:51.303160+0000","last_became_active":"2024-08-29T15:19:46.804887+0000","last_became_peered":"2024-08-29T15:19:46.804887+0000","last_unstale":"2024-08-29T15:50:51.303160+0000","last_undegraded":"2024-08-29T15:50:51.303160+0000","last_fullsized":"2024-08-29T15:50:51.303160+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:49:00.226024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1016,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305009+0000","last_change":"2024-08-29T15:08:58.888759+0000","last_active":"2024-08-29T15:50:52.305009+0000","last_peered":"2024-08-29T15:50:52.305009+0000","last_clean":"2024-08-29T15:50:52.305009+0000","last_became_active":"2024-08-29T15:08:58.888048+0000","last_became_peered":"2024-08-29T15:08:58.888048+0000","last_unstale":"2024-08-29T15:50:52.305009+0000","last_undegraded":"2024-08-29T15:50:52.305009+0000","last_fullsized":"2024-08-29T15:50:52.305009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:46:19.764424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"90'149","reported_seq":803,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300656+0000","last_change":"2024-08-29T15:49:48.294772+0000","last_active":"2024-08-29T15:50:51.300656+0000","last_peered":"2024-08-29T15:50:51.300656+0000","last_clean":"2024-08-29T15:50:51.300656+0000","last_became_active":"2024-08-29T15:49:48.293676+0000","last_became_peered":"2024-08-29T15:49:48.293676+0000","last_unstale":"2024-08-29T15:50:51.300656+0000","last_undegraded":"2024-08-29T15:50:51.300656+0000","last_fullsized":"2024-08-29T15:50:51.300656+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:37.167690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:37.167690+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:25:36.708271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1012,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422693+0000","last_change":"2024-08-29T15:19:46.805852+0000","last_active":"2024-08-29T15:50:52.422693+0000","last_peered":"2024-08-29T15:50:52.422693+0000","last_clean":"2024-08-29T15:50:52.422693+0000","last_became_active":"2024-08-29T15:19:46.805518+0000","last_became_peered":"2024-08-29T15:19:46.805518+0000","last_unstale":"2024-08-29T15:50:52.422693+0000","last_undegraded":"2024-08-29T15:50:52.422693+0000","last_fullsized":"2024-08-29T15:50:52.422693+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:19:52.688184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307439+0000","last_change":"2024-08-29T15:48:12.267373+0000","last_active":"2024-08-29T15:50:52.307439+0000","last_peered":"2024-08-29T15:50:52.307439+0000","last_clean":"2024-08-29T15:50:52.307439+0000","last_became_active":"2024-08-29T15:48:12.267059+0000","last_became_peered":"2024-08-29T15:48:12.267059+0000","last_unstale":"2024-08-29T15:50:52.307439+0000","last_undegraded":"2024-08-29T15:50:52.307439+0000","last_fullsized":"2024-08-29T15:50:52.307439+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:25:55.282346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"90'149","reported_seq":1047,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300909+0000","last_change":"2024-08-29T15:49:48.295500+0000","last_active":"2024-08-29T15:50:51.300909+0000","last_peered":"2024-08-29T15:50:51.300909+0000","last_clean":"2024-08-29T15:50:51.300909+0000","last_became_active":"2024-08-29T15:49:48.295131+0000","last_became_peered":"2024-08-29T15:49:48.295131+0000","last_unstale":"2024-08-29T15:50:51.300909+0000","last_undegraded":"2024-08-29T15:50:51.300909+0000","last_fullsized":"2024-08-29T15:50:51.300909+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:40.396348+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:40.396348+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:59:09.899452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":191,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.424352+0000","last_change":"2024-08-29T15:50:52.424253+0000","last_active":"2024-08-29T15:50:52.424352+0000","last_peered":"2024-08-29T15:50:52.424352+0000","last_clean":"2024-08-29T15:50:52.424352+0000","last_became_active":"2024-08-29T15:50:52.423932+0000","last_became_peered":"2024-08-29T15:50:52.423932+0000","last_unstale":"2024-08-29T15:50:52.424352+0000","last_undegraded":"2024-08-29T15:50:52.424352+0000","last_fullsized":"2024-08-29T15:50:52.424352+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:22.076998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":191,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306757+0000","last_change":"2024-08-29T15:50:31.763839+0000","last_active":"2024-08-29T15:50:51.306757+0000","last_peered":"2024-08-29T15:50:51.306757+0000","last_clean":"2024-08-29T15:50:51.306757+0000","last_became_active":"2024-08-29T15:50:31.763276+0000","last_became_peered":"2024-08-29T15:50:31.763276+0000","last_unstale":"2024-08-29T15:50:51.306757+0000","last_undegraded":"2024-08-29T15:50:51.306757+0000","last_fullsized":"2024-08-29T15:50:51.306757+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:21:46.127572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"90'149","reported_seq":817,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.302937+0000","last_change":"2024-08-29T15:45:26.912257+0000","last_active":"2024-08-29T15:50:51.302937+0000","last_peered":"2024-08-29T15:50:51.302937+0000","last_clean":"2024-08-29T15:50:51.302937+0000","last_became_active":"2024-08-29T15:44:59.244250+0000","last_became_peered":"2024-08-29T15:44:59.244250+0000","last_unstale":"2024-08-29T15:50:51.302937+0000","last_undegraded":"2024-08-29T15:50:51.302937+0000","last_fullsized":"2024-08-29T15:50:51.302937+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.912179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.912179+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:24:51.014992+0000","scrub_duration":0.0012045789999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1032,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304779+0000","last_change":"2024-08-29T15:16:59.030342+0000","last_active":"2024-08-29T15:50:52.304779+0000","last_peered":"2024-08-29T15:50:52.304779+0000","last_clean":"2024-08-29T15:50:52.304779+0000","last_became_active":"2024-08-29T15:16:59.029831+0000","last_became_peered":"2024-08-29T15:16:59.029831+0000","last_unstale":"2024-08-29T15:50:52.304779+0000","last_undegraded":"2024-08-29T15:50:52.304779+0000","last_fullsized":"2024-08-29T15:50:52.304779+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:39:10.871247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":181,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307671+0000","last_change":"2024-08-29T15:49:45.428868+0000","last_active":"2024-08-29T15:50:52.307671+0000","last_peered":"2024-08-29T15:50:52.307671+0000","last_clean":"2024-08-29T15:50:52.307671+0000","last_became_active":"2024-08-29T15:49:45.428396+0000","last_became_peered":"2024-08-29T15:49:45.428396+0000","last_unstale":"2024-08-29T15:50:52.307671+0000","last_undegraded":"2024-08-29T15:50:52.307671+0000","last_fullsized":"2024-08-29T15:50:52.307671+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:43:59.655509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"90'149","reported_seq":919,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307532+0000","last_change":"2024-08-29T15:49:48.296397+0000","last_active":"2024-08-29T15:50:51.307532+0000","last_peered":"2024-08-29T15:50:51.307532+0000","last_clean":"2024-08-29T15:50:51.307532+0000","last_became_active":"2024-08-29T15:49:48.296044+0000","last_became_peered":"2024-08-29T15:49:48.296044+0000","last_unstale":"2024-08-29T15:50:51.307532+0000","last_undegraded":"2024-08-29T15:50:51.307532+0000","last_fullsized":"2024-08-29T15:50:51.307532+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:26.882638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:26.882638+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:03:49.621431+0000","scrub_duration":0.001489043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":980,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304936+0000","last_change":"2024-08-29T15:16:59.041240+0000","last_active":"2024-08-29T15:50:52.304936+0000","last_peered":"2024-08-29T15:50:52.304936+0000","last_clean":"2024-08-29T15:50:52.304936+0000","last_became_active":"2024-08-29T15:16:59.040918+0000","last_became_peered":"2024-08-29T15:16:59.040918+0000","last_unstale":"2024-08-29T15:50:52.304936+0000","last_undegraded":"2024-08-29T15:50:52.304936+0000","last_fullsized":"2024-08-29T15:50:52.304936+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:29:07.166848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301244+0000","last_change":"2024-08-29T15:49:45.424186+0000","last_active":"2024-08-29T15:50:51.301244+0000","last_peered":"2024-08-29T15:50:51.301244+0000","last_clean":"2024-08-29T15:50:51.301244+0000","last_became_active":"2024-08-29T15:49:45.423882+0000","last_became_peered":"2024-08-29T15:49:45.423882+0000","last_unstale":"2024-08-29T15:50:51.301244+0000","last_undegraded":"2024-08-29T15:50:51.301244+0000","last_fullsized":"2024-08-29T15:50:51.301244+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:52:36.923638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"90'149","reported_seq":1148,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.308659+0000","last_change":"2024-08-29T15:50:52.308570+0000","last_active":"2024-08-29T15:50:52.308659+0000","last_peered":"2024-08-29T15:50:52.308659+0000","last_clean":"2024-08-29T15:50:52.308659+0000","last_became_active":"2024-08-29T15:50:52.308209+0000","last_became_peered":"2024-08-29T15:50:52.308209+0000","last_unstale":"2024-08-29T15:50:52.308659+0000","last_undegraded":"2024-08-29T15:50:52.308659+0000","last_fullsized":"2024-08-29T15:50:52.308659+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:44.368136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:44.368136+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:12:10.878495+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":201,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307390+0000","last_change":"2024-08-29T15:50:37.806204+0000","last_active":"2024-08-29T15:50:51.307390+0000","last_peered":"2024-08-29T15:50:51.307390+0000","last_clean":"2024-08-29T15:50:51.307390+0000","last_became_active":"2024-08-29T15:50:37.805868+0000","last_became_peered":"2024-08-29T15:50:37.805868+0000","last_unstale":"2024-08-29T15:50:51.307390+0000","last_undegraded":"2024-08-29T15:50:51.307390+0000","last_fullsized":"2024-08-29T15:50:51.307390+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:21:01.177942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":155,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422660+0000","last_change":"2024-08-29T15:47:12.486814+0000","last_active":"2024-08-29T15:50:52.422660+0000","last_peered":"2024-08-29T15:50:52.422660+0000","last_clean":"2024-08-29T15:50:52.422660+0000","last_became_active":"2024-08-29T15:47:12.486372+0000","last_became_peered":"2024-08-29T15:47:12.486372+0000","last_unstale":"2024-08-29T15:50:52.422660+0000","last_undegraded":"2024-08-29T15:50:52.422660+0000","last_fullsized":"2024-08-29T15:50:52.422660+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:13:38.593835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"90'149","reported_seq":1024,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301368+0000","last_change":"2024-08-29T15:49:48.295362+0000","last_active":"2024-08-29T15:50:51.301368+0000","last_peered":"2024-08-29T15:50:51.301368+0000","last_clean":"2024-08-29T15:50:51.301368+0000","last_became_active":"2024-08-29T15:49:48.295053+0000","last_became_peered":"2024-08-29T15:49:48.295053+0000","last_unstale":"2024-08-29T15:50:51.301368+0000","last_undegraded":"2024-08-29T15:50:51.301368+0000","last_fullsized":"2024-08-29T15:50:51.301368+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:28.337426+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:28.337426+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:40:14.219256+0000","scrub_duration":0.001450535,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"90'23","reported_seq":372,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303344+0000","last_change":"2024-08-29T15:28:35.529883+0000","last_active":"2024-08-29T15:50:51.303344+0000","last_peered":"2024-08-29T15:50:51.303344+0000","last_clean":"2024-08-29T15:50:51.303344+0000","last_became_active":"2024-08-29T15:19:46.806088+0000","last_became_peered":"2024-08-29T15:19:46.806088+0000","last_unstale":"2024-08-29T15:50:51.303344+0000","last_undegraded":"2024-08-29T15:50:51.303344+0000","last_fullsized":"2024-08-29T15:50:51.303344+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:00:30.764607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":362,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303535+0000","last_change":"2024-08-29T15:19:46.875690+0000","last_active":"2024-08-29T15:50:51.303535+0000","last_peered":"2024-08-29T15:50:51.303535+0000","last_clean":"2024-08-29T15:50:51.303535+0000","last_became_active":"2024-08-29T15:19:46.875394+0000","last_became_peered":"2024-08-29T15:19:46.875394+0000","last_unstale":"2024-08-29T15:50:51.303535+0000","last_undegraded":"2024-08-29T15:50:51.303535+0000","last_fullsized":"2024-08-29T15:50:51.303535+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:51:48.735441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"90'149","reported_seq":1024,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423052+0000","last_change":"2024-08-29T15:44:59.239507+0000","last_active":"2024-08-29T15:50:52.423052+0000","last_peered":"2024-08-29T15:50:52.423052+0000","last_clean":"2024-08-29T15:50:52.423052+0000","last_became_active":"2024-08-29T15:44:59.239049+0000","last_became_peered":"2024-08-29T15:44:59.239049+0000","last_unstale":"2024-08-29T15:50:52.423052+0000","last_undegraded":"2024-08-29T15:50:52.423052+0000","last_fullsized":"2024-08-29T15:50:52.423052+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:28.746327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:28.746327+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:38:22.918512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":302,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303075+0000","last_change":"2024-08-29T15:50:25.955660+0000","last_active":"2024-08-29T15:50:51.303075+0000","last_peered":"2024-08-29T15:50:51.303075+0000","last_clean":"2024-08-29T15:50:51.303075+0000","last_became_active":"2024-08-29T15:50:25.955251+0000","last_became_peered":"2024-08-29T15:50:25.955251+0000","last_unstale":"2024-08-29T15:50:51.303075+0000","last_undegraded":"2024-08-29T15:50:51.303075+0000","last_fullsized":"2024-08-29T15:50:51.303075+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:08:35.207019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"86'66","reported_seq":377,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306684+0000","last_change":"2024-08-29T15:48:12.274839+0000","last_active":"2024-08-29T15:50:51.306684+0000","last_peered":"2024-08-29T15:50:51.306684+0000","last_clean":"2024-08-29T15:50:51.306684+0000","last_became_active":"2024-08-29T15:48:12.268376+0000","last_became_peered":"2024-08-29T15:48:12.268376+0000","last_unstale":"2024-08-29T15:50:51.306684+0000","last_undegraded":"2024-08-29T15:50:51.306684+0000","last_fullsized":"2024-08-29T15:50:51.306684+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:49:19.911889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024726499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":453,"num_read_kb":858,"num_write":64,"num_write_kb":266,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":172,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307347+0000","last_change":"2024-08-29T15:46:58.380711+0000","last_active":"2024-08-29T15:50:52.307347+0000","last_peered":"2024-08-29T15:50:52.307347+0000","last_clean":"2024-08-29T15:50:52.307347+0000","last_became_active":"2024-08-29T15:46:58.369572+0000","last_became_peered":"2024-08-29T15:46:58.369572+0000","last_unstale":"2024-08-29T15:50:52.307347+0000","last_undegraded":"2024-08-29T15:50:52.307347+0000","last_fullsized":"2024-08-29T15:50:52.307347+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:49:53.912727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"90'149","reported_seq":1216,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304981+0000","last_change":"2024-08-29T15:49:48.293377+0000","last_active":"2024-08-29T15:50:52.304981+0000","last_peered":"2024-08-29T15:50:52.304981+0000","last_clean":"2024-08-29T15:50:52.304981+0000","last_became_active":"2024-08-29T15:49:48.292925+0000","last_became_peered":"2024-08-29T15:49:48.292925+0000","last_unstale":"2024-08-29T15:50:52.304981+0000","last_undegraded":"2024-08-29T15:50:52.304981+0000","last_fullsized":"2024-08-29T15:50:52.304981+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.774861+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.774861+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:07:53.726096+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":370,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304685+0000","last_change":"2024-08-29T15:48:12.266718+0000","last_active":"2024-08-29T15:50:52.304685+0000","last_peered":"2024-08-29T15:50:52.304685+0000","last_clean":"2024-08-29T15:50:52.304685+0000","last_became_active":"2024-08-29T15:48:12.266363+0000","last_became_peered":"2024-08-29T15:48:12.266363+0000","last_unstale":"2024-08-29T15:50:52.304685+0000","last_undegraded":"2024-08-29T15:50:52.304685+0000","last_fullsized":"2024-08-29T15:50:52.304685+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:32:53.550145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'139","reported_seq":1669,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423349+0000","last_change":"2024-08-29T15:28:40.836889+0000","last_active":"2024-08-29T15:50:52.423349+0000","last_peered":"2024-08-29T15:50:52.423349+0000","last_clean":"2024-08-29T15:50:52.423349+0000","last_became_active":"2024-08-29T15:28:40.836423+0000","last_became_peered":"2024-08-29T15:28:40.836423+0000","last_unstale":"2024-08-29T15:50:52.423349+0000","last_undegraded":"2024-08-29T15:50:52.423349+0000","last_fullsized":"2024-08-29T15:50:52.423349+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:35:56.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097086299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":935,"num_read_kb":2190,"num_write":105,"num_write_kb":1273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1017,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305644+0000","last_change":"2024-08-29T15:08:58.888442+0000","last_active":"2024-08-29T15:50:52.305644+0000","last_peered":"2024-08-29T15:50:52.305644+0000","last_clean":"2024-08-29T15:50:52.305644+0000","last_became_active":"2024-08-29T15:08:58.887809+0000","last_became_peered":"2024-08-29T15:08:58.887809+0000","last_unstale":"2024-08-29T15:50:52.305644+0000","last_undegraded":"2024-08-29T15:50:52.305644+0000","last_fullsized":"2024-08-29T15:50:52.305644+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:10:55.204150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'149","reported_seq":991,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.308121+0000","last_change":"2024-08-29T15:44:22.861452+0000","last_active":"2024-08-29T15:50:52.308121+0000","last_peered":"2024-08-29T15:50:52.308121+0000","last_clean":"2024-08-29T15:50:52.308121+0000","last_became_active":"2024-08-29T15:44:22.611166+0000","last_became_peered":"2024-08-29T15:44:22.611166+0000","last_unstale":"2024-08-29T15:50:52.308121+0000","last_undegraded":"2024-08-29T15:50:52.308121+0000","last_fullsized":"2024-08-29T15:50:52.308121+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":471,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.861280+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.861280+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:57:45.254970+0000","scrub_duration":0.0040975610000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1038,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307194+0000","last_change":"2024-08-29T15:49:45.424973+0000","last_active":"2024-08-29T15:50:52.307194+0000","last_peered":"2024-08-29T15:50:52.307194+0000","last_clean":"2024-08-29T15:50:52.307194+0000","last_became_active":"2024-08-29T15:49:45.424624+0000","last_became_peered":"2024-08-29T15:49:45.424624+0000","last_unstale":"2024-08-29T15:50:52.307194+0000","last_undegraded":"2024-08-29T15:50:52.307194+0000","last_fullsized":"2024-08-29T15:50:52.307194+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:46.192350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"86'92","reported_seq":1093,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307574+0000","last_change":"2024-08-29T15:46:58.399147+0000","last_active":"2024-08-29T15:50:51.307574+0000","last_peered":"2024-08-29T15:50:51.307574+0000","last_clean":"2024-08-29T15:50:51.307574+0000","last_became_active":"2024-08-29T15:46:58.371595+0000","last_became_peered":"2024-08-29T15:46:58.371595+0000","last_unstale":"2024-08-29T15:50:51.307574+0000","last_undegraded":"2024-08-29T15:50:51.307574+0000","last_fullsized":"2024-08-29T15:50:51.307574+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":92,"log_dups_size":0,"ondisk_log_size":92,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:46:46.943596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021517899999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":689,"num_read_kb":897,"num_write":66,"num_write_kb":4283,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1046,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423114+0000","last_change":"2024-08-29T15:49:45.428512+0000","last_active":"2024-08-29T15:50:52.423114+0000","last_peered":"2024-08-29T15:50:52.423114+0000","last_clean":"2024-08-29T15:50:52.423114+0000","last_became_active":"2024-08-29T15:49:45.428171+0000","last_became_peered":"2024-08-29T15:49:45.428171+0000","last_unstale":"2024-08-29T15:50:52.423114+0000","last_undegraded":"2024-08-29T15:50:52.423114+0000","last_fullsized":"2024-08-29T15:50:52.423114+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:44:00.494227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'149","reported_seq":544,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306434+0000","last_change":"2024-08-29T15:50:31.977180+0000","last_active":"2024-08-29T15:50:51.306434+0000","last_peered":"2024-08-29T15:50:51.306434+0000","last_clean":"2024-08-29T15:50:51.306434+0000","last_became_active":"2024-08-29T15:50:31.763055+0000","last_became_peered":"2024-08-29T15:50:31.763055+0000","last_unstale":"2024-08-29T15:50:51.306434+0000","last_undegraded":"2024-08-29T15:50:51.306434+0000","last_fullsized":"2024-08-29T15:50:51.306434+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:31.977082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:31.977082+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:45:00.646384+0000","scrub_duration":0.001873375,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":170,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422791+0000","last_change":"2024-08-29T15:49:45.428070+0000","last_active":"2024-08-29T15:50:52.422791+0000","last_peered":"2024-08-29T15:50:52.422791+0000","last_clean":"2024-08-29T15:50:52.422791+0000","last_became_active":"2024-08-29T15:49:45.427658+0000","last_became_peered":"2024-08-29T15:49:45.427658+0000","last_unstale":"2024-08-29T15:50:52.422791+0000","last_undegraded":"2024-08-29T15:50:52.422791+0000","last_fullsized":"2024-08-29T15:50:52.422791+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:51:32.357331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"86'130","reported_seq":1450,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423231+0000","last_change":"2024-08-29T15:07:52.138092+0000","last_active":"2024-08-29T15:50:52.423231+0000","last_peered":"2024-08-29T15:50:52.423231+0000","last_clean":"2024-08-29T15:50:52.423231+0000","last_became_active":"2024-08-29T15:07:52.137096+0000","last_became_peered":"2024-08-29T15:07:52.137096+0000","last_unstale":"2024-08-29T15:50:52.423231+0000","last_undegraded":"2024-08-29T15:50:52.423231+0000","last_fullsized":"2024-08-29T15:50:52.423231+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:50:06.762548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021578,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":585,"num_read_kb":1791,"num_write":134,"num_write_kb":1217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301241+0000","last_change":"2024-08-29T15:49:45.426855+0000","last_active":"2024-08-29T15:50:51.301241+0000","last_peered":"2024-08-29T15:50:51.301241+0000","last_clean":"2024-08-29T15:50:51.301241+0000","last_became_active":"2024-08-29T15:49:45.425903+0000","last_became_peered":"2024-08-29T15:49:45.425903+0000","last_unstale":"2024-08-29T15:50:51.301241+0000","last_undegraded":"2024-08-29T15:50:51.301241+0000","last_fullsized":"2024-08-29T15:50:51.301241+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:16:10.531855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'149","reported_seq":544,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306481+0000","last_change":"2024-08-29T15:50:33.921689+0000","last_active":"2024-08-29T15:50:51.306481+0000","last_peered":"2024-08-29T15:50:51.306481+0000","last_clean":"2024-08-29T15:50:51.306481+0000","last_became_active":"2024-08-29T15:50:31.762574+0000","last_became_peered":"2024-08-29T15:50:31.762574+0000","last_unstale":"2024-08-29T15:50:51.306481+0000","last_undegraded":"2024-08-29T15:50:51.306481+0000","last_fullsized":"2024-08-29T15:50:51.306481+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:33.921562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:33.921562+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:53:21.195720+0000","scrub_duration":0.0033730650000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422871+0000","last_change":"2024-08-29T15:49:23.300350+0000","last_active":"2024-08-29T15:50:52.422871+0000","last_peered":"2024-08-29T15:50:52.422871+0000","last_clean":"2024-08-29T15:50:52.422871+0000","last_became_active":"2024-08-29T15:49:23.295545+0000","last_became_peered":"2024-08-29T15:49:23.295545+0000","last_unstale":"2024-08-29T15:50:52.422871+0000","last_undegraded":"2024-08-29T15:50:52.422871+0000","last_fullsized":"2024-08-29T15:50:52.422871+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:03:11.367343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":364,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307656+0000","last_change":"2024-08-29T15:19:46.875664+0000","last_active":"2024-08-29T15:50:52.307656+0000","last_peered":"2024-08-29T15:50:52.307656+0000","last_clean":"2024-08-29T15:50:52.307656+0000","last_became_active":"2024-08-29T15:19:46.875392+0000","last_became_peered":"2024-08-29T15:19:46.875392+0000","last_unstale":"2024-08-29T15:50:52.307656+0000","last_undegraded":"2024-08-29T15:50:52.307656+0000","last_fullsized":"2024-08-29T15:50:52.307656+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:19:16.421308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"86'54","reported_seq":55,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300714+0000","last_change":"2024-08-29T15:49:45.445767+0000","last_active":"2024-08-29T15:50:51.300714+0000","last_peered":"2024-08-29T15:50:51.300714+0000","last_clean":"2024-08-29T15:50:51.300714+0000","last_became_active":"2024-08-29T15:49:45.426097+0000","last_became_peered":"2024-08-29T15:49:45.426097+0000","last_unstale":"2024-08-29T15:50:51.300714+0000","last_undegraded":"2024-08-29T15:50:51.300714+0000","last_fullsized":"2024-08-29T15:50:51.300714+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:19:04.260523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000385311,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":568,"num_read_kb":2571,"num_write":39,"num_write_kb":2071,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'149","reported_seq":967,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300688+0000","last_change":"2024-08-29T15:49:48.295920+0000","last_active":"2024-08-29T15:50:51.300688+0000","last_peered":"2024-08-29T15:50:51.300688+0000","last_clean":"2024-08-29T15:50:51.300688+0000","last_became_active":"2024-08-29T15:49:48.294838+0000","last_became_peered":"2024-08-29T15:49:48.294838+0000","last_unstale":"2024-08-29T15:50:51.300688+0000","last_undegraded":"2024-08-29T15:50:51.300688+0000","last_fullsized":"2024-08-29T15:50:51.300688+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:44:26.709569+0000","last_clean_scrub_stamp":"2024-08-29T15:44:26.709569+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:48:16.461251+0000","scrub_duration":0.0031838119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":170,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423216+0000","last_change":"2024-08-29T15:49:45.428165+0000","last_active":"2024-08-29T15:50:52.423216+0000","last_peered":"2024-08-29T15:50:52.423216+0000","last_clean":"2024-08-29T15:50:52.423216+0000","last_became_active":"2024-08-29T15:49:45.427816+0000","last_became_peered":"2024-08-29T15:49:45.427816+0000","last_unstale":"2024-08-29T15:50:52.423216+0000","last_undegraded":"2024-08-29T15:50:52.423216+0000","last_fullsized":"2024-08-29T15:50:52.423216+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:05:12.207096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":320,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306549+0000","last_change":"2024-08-29T15:50:37.805399+0000","last_active":"2024-08-29T15:50:51.306549+0000","last_peered":"2024-08-29T15:50:51.306549+0000","last_clean":"2024-08-29T15:50:51.306549+0000","last_became_active":"2024-08-29T15:50:37.805012+0000","last_became_peered":"2024-08-29T15:50:37.805012+0000","last_unstale":"2024-08-29T15:50:51.306549+0000","last_undegraded":"2024-08-29T15:50:51.306549+0000","last_fullsized":"2024-08-29T15:50:51.306549+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:48:41.571339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'117","reported_seq":638,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.308427+0000","last_change":"2024-08-29T15:50:52.308327+0000","last_active":"2024-08-29T15:50:52.308427+0000","last_peered":"2024-08-29T15:50:52.308427+0000","last_clean":"2024-08-29T15:50:52.308427+0000","last_became_active":"2024-08-29T15:50:52.307746+0000","last_became_peered":"2024-08-29T15:50:52.307746+0000","last_unstale":"2024-08-29T15:50:52.308427+0000","last_undegraded":"2024-08-29T15:50:52.308427+0000","last_fullsized":"2024-08-29T15:50:52.308427+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:07:46.458367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032875599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":737,"num_read_kb":693,"num_write":110,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'149","reported_seq":1082,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306921+0000","last_change":"2024-08-29T15:50:37.807890+0000","last_active":"2024-08-29T15:50:51.306921+0000","last_peered":"2024-08-29T15:50:51.306921+0000","last_clean":"2024-08-29T15:50:51.306921+0000","last_became_active":"2024-08-29T15:50:37.807557+0000","last_became_peered":"2024-08-29T15:50:37.807557+0000","last_unstale":"2024-08-29T15:50:51.306921+0000","last_undegraded":"2024-08-29T15:50:51.306921+0000","last_fullsized":"2024-08-29T15:50:51.306921+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.841106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.841106+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:36:06.253219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":168,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307241+0000","last_change":"2024-08-29T15:46:58.369628+0000","last_active":"2024-08-29T15:50:52.307241+0000","last_peered":"2024-08-29T15:50:52.307241+0000","last_clean":"2024-08-29T15:50:52.307241+0000","last_became_active":"2024-08-29T15:46:58.369315+0000","last_became_peered":"2024-08-29T15:46:58.369315+0000","last_unstale":"2024-08-29T15:50:52.307241+0000","last_undegraded":"2024-08-29T15:50:52.307241+0000","last_fullsized":"2024-08-29T15:50:52.307241+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:52:05.895354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1052,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306684+0000","last_change":"2024-08-29T15:47:42.288211+0000","last_active":"2024-08-29T15:50:52.306684+0000","last_peered":"2024-08-29T15:50:52.306684+0000","last_clean":"2024-08-29T15:50:52.306684+0000","last_became_active":"2024-08-29T15:47:42.287594+0000","last_became_peered":"2024-08-29T15:47:42.287594+0000","last_unstale":"2024-08-29T15:50:52.306684+0000","last_undegraded":"2024-08-29T15:50:52.306684+0000","last_fullsized":"2024-08-29T15:50:52.306684+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:32:18.869863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'180","reported_seq":919,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306862+0000","last_change":"2024-08-29T15:50:37.809494+0000","last_active":"2024-08-29T15:50:51.306862+0000","last_peered":"2024-08-29T15:50:51.306862+0000","last_clean":"2024-08-29T15:50:51.306862+0000","last_became_active":"2024-08-29T15:50:37.808844+0000","last_became_peered":"2024-08-29T15:50:37.808844+0000","last_unstale":"2024-08-29T15:50:51.306862+0000","last_undegraded":"2024-08-29T15:50:51.306862+0000","last_fullsized":"2024-08-29T15:50:51.306862+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":180,"log_dups_size":0,"ondisk_log_size":180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:31:24.232827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036460399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1280,"num_read_kb":2184,"num_write":140,"num_write_kb":1114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'149","reported_seq":1165,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423037+0000","last_change":"2024-08-29T15:43:26.588054+0000","last_active":"2024-08-29T15:50:52.423037+0000","last_peered":"2024-08-29T15:50:52.423037+0000","last_clean":"2024-08-29T15:50:52.423037+0000","last_became_active":"2024-08-29T15:43:26.285267+0000","last_became_peered":"2024-08-29T15:43:26.285267+0000","last_unstale":"2024-08-29T15:50:52.423037+0000","last_undegraded":"2024-08-29T15:50:52.423037+0000","last_fullsized":"2024-08-29T15:50:52.423037+0000","mapping_epoch":365,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.587986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.587986+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:15:21.198019+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307061+0000","last_change":"2024-08-29T15:50:31.767001+0000","last_active":"2024-08-29T15:50:51.307061+0000","last_peered":"2024-08-29T15:50:51.307061+0000","last_clean":"2024-08-29T15:50:51.307061+0000","last_became_active":"2024-08-29T15:50:31.766580+0000","last_became_peered":"2024-08-29T15:50:31.766580+0000","last_unstale":"2024-08-29T15:50:51.307061+0000","last_undegraded":"2024-08-29T15:50:51.307061+0000","last_fullsized":"2024-08-29T15:50:51.307061+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:01:14.735883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":440,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304786+0000","last_change":"2024-08-29T15:49:45.426244+0000","last_active":"2024-08-29T15:50:52.304786+0000","last_peered":"2024-08-29T15:50:52.304786+0000","last_clean":"2024-08-29T15:50:52.304786+0000","last_became_active":"2024-08-29T15:49:45.425976+0000","last_became_peered":"2024-08-29T15:49:45.425976+0000","last_unstale":"2024-08-29T15:50:52.304786+0000","last_undegraded":"2024-08-29T15:50:52.304786+0000","last_fullsized":"2024-08-29T15:50:52.304786+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:01:11.296031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'199","reported_seq":705,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300932+0000","last_change":"2024-08-29T15:49:48.294805+0000","last_active":"2024-08-29T15:50:51.300932+0000","last_peered":"2024-08-29T15:50:51.300932+0000","last_clean":"2024-08-29T15:50:51.300932+0000","last_became_active":"2024-08-29T15:49:48.294065+0000","last_became_peered":"2024-08-29T15:49:48.294065+0000","last_unstale":"2024-08-29T15:50:51.300932+0000","last_undegraded":"2024-08-29T15:50:51.300932+0000","last_fullsized":"2024-08-29T15:50:51.300932+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:54:17.546833+0000","last_clean_scrub_stamp":"2024-08-29T14:54:17.546833+0000","objects_scrubbed":0,"log_size":199,"log_dups_size":0,"ondisk_log_size":199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:23:34.652252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010541159999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":740,"num_read_kb":1816,"num_write":188,"num_write_kb":1225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'149","reported_seq":581,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306957+0000","last_change":"2024-08-29T15:50:40.404619+0000","last_active":"2024-08-29T15:50:51.306957+0000","last_peered":"2024-08-29T15:50:51.306957+0000","last_clean":"2024-08-29T15:50:51.306957+0000","last_became_active":"2024-08-29T15:50:37.808260+0000","last_became_peered":"2024-08-29T15:50:37.808260+0000","last_unstale":"2024-08-29T15:50:51.306957+0000","last_undegraded":"2024-08-29T15:50:51.306957+0000","last_fullsized":"2024-08-29T15:50:51.306957+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:40.404512+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:40.404512+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:56:13.286215+0000","scrub_duration":0.001575943,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1012,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.423329+0000","last_change":"2024-08-29T15:19:46.805577+0000","last_active":"2024-08-29T15:50:52.423329+0000","last_peered":"2024-08-29T15:50:52.423329+0000","last_clean":"2024-08-29T15:50:52.423329+0000","last_became_active":"2024-08-29T15:19:46.805230+0000","last_became_peered":"2024-08-29T15:19:46.805230+0000","last_unstale":"2024-08-29T15:50:52.423329+0000","last_undegraded":"2024-08-29T15:50:52.423329+0000","last_fullsized":"2024-08-29T15:50:52.423329+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:41:07.437806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":146,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304581+0000","last_change":"2024-08-29T15:47:43.258937+0000","last_active":"2024-08-29T15:50:52.304581+0000","last_peered":"2024-08-29T15:50:52.304581+0000","last_clean":"2024-08-29T15:50:52.304581+0000","last_became_active":"2024-08-29T15:47:43.258572+0000","last_became_peered":"2024-08-29T15:47:43.258572+0000","last_unstale":"2024-08-29T15:50:52.304581+0000","last_undegraded":"2024-08-29T15:50:52.304581+0000","last_fullsized":"2024-08-29T15:50:52.304581+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:15:58.220504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'149","reported_seq":752,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300987+0000","last_change":"2024-08-29T15:49:50.423039+0000","last_active":"2024-08-29T15:50:51.300987+0000","last_peered":"2024-08-29T15:50:51.300987+0000","last_clean":"2024-08-29T15:50:51.300987+0000","last_became_active":"2024-08-29T15:49:48.294243+0000","last_became_peered":"2024-08-29T15:49:48.294243+0000","last_unstale":"2024-08-29T15:50:51.300987+0000","last_undegraded":"2024-08-29T15:50:51.300987+0000","last_fullsized":"2024-08-29T15:50:51.300987+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:50.422956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:50.422956+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:31:25.265825+0000","scrub_duration":0.0026719690000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":604,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303659+0000","last_change":"2024-08-29T15:47:12.467689+0000","last_active":"2024-08-29T15:50:51.303659+0000","last_peered":"2024-08-29T15:50:51.303659+0000","last_clean":"2024-08-29T15:50:51.303659+0000","last_became_active":"2024-08-29T15:47:12.467400+0000","last_became_peered":"2024-08-29T15:47:12.467400+0000","last_unstale":"2024-08-29T15:50:51.303659+0000","last_undegraded":"2024-08-29T15:50:51.303659+0000","last_fullsized":"2024-08-29T15:50:51.303659+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:45:54.623271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":363,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.302970+0000","last_change":"2024-08-29T15:19:46.802968+0000","last_active":"2024-08-29T15:50:51.302970+0000","last_peered":"2024-08-29T15:50:51.302970+0000","last_clean":"2024-08-29T15:50:51.302970+0000","last_became_active":"2024-08-29T15:19:46.802642+0000","last_became_peered":"2024-08-29T15:19:46.802642+0000","last_unstale":"2024-08-29T15:50:51.302970+0000","last_undegraded":"2024-08-29T15:50:51.302970+0000","last_fullsized":"2024-08-29T15:50:51.302970+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T23:21:29.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'149","reported_seq":1207,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422740+0000","last_change":"2024-08-29T15:49:48.295197+0000","last_active":"2024-08-29T15:50:52.422740+0000","last_peered":"2024-08-29T15:50:52.422740+0000","last_clean":"2024-08-29T15:50:52.422740+0000","last_became_active":"2024-08-29T15:49:48.294704+0000","last_became_peered":"2024-08-29T15:49:48.294704+0000","last_unstale":"2024-08-29T15:50:52.422740+0000","last_undegraded":"2024-08-29T15:50:52.422740+0000","last_fullsized":"2024-08-29T15:50:52.422740+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:22.906140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:22.906140+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:24:17.805166+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":310,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303596+0000","last_change":"2024-08-29T15:50:32.967030+0000","last_active":"2024-08-29T15:50:51.303596+0000","last_peered":"2024-08-29T15:50:51.303596+0000","last_clean":"2024-08-29T15:50:51.303596+0000","last_became_active":"2024-08-29T15:50:32.966651+0000","last_became_peered":"2024-08-29T15:50:32.966651+0000","last_unstale":"2024-08-29T15:50:51.303596+0000","last_undegraded":"2024-08-29T15:50:51.303596+0000","last_fullsized":"2024-08-29T15:50:51.303596+0000","mapping_epoch":564,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:12:41.521139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":306,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306259+0000","last_change":"2024-08-29T15:50:31.766219+0000","last_active":"2024-08-29T15:50:51.306259+0000","last_peered":"2024-08-29T15:50:51.306259+0000","last_clean":"2024-08-29T15:50:51.306259+0000","last_became_active":"2024-08-29T15:50:31.765918+0000","last_became_peered":"2024-08-29T15:50:31.765918+0000","last_unstale":"2024-08-29T15:50:51.306259+0000","last_undegraded":"2024-08-29T15:50:51.306259+0000","last_fullsized":"2024-08-29T15:50:51.306259+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:36:38.266764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"90'149","reported_seq":766,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300853+0000","last_change":"2024-08-29T15:49:52.412907+0000","last_active":"2024-08-29T15:50:51.300853+0000","last_peered":"2024-08-29T15:50:51.300853+0000","last_clean":"2024-08-29T15:50:51.300853+0000","last_became_active":"2024-08-29T15:49:48.293548+0000","last_became_peered":"2024-08-29T15:49:48.293548+0000","last_unstale":"2024-08-29T15:50:51.300853+0000","last_undegraded":"2024-08-29T15:50:51.300853+0000","last_fullsized":"2024-08-29T15:50:51.300853+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:49:52.412798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:49:52.412798+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:22:05.653553+0000","scrub_duration":0.0015053149999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":301,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303217+0000","last_change":"2024-08-29T15:50:25.955631+0000","last_active":"2024-08-29T15:50:51.303217+0000","last_peered":"2024-08-29T15:50:51.303217+0000","last_clean":"2024-08-29T15:50:51.303217+0000","last_became_active":"2024-08-29T15:50:25.955245+0000","last_became_peered":"2024-08-29T15:50:25.955245+0000","last_unstale":"2024-08-29T15:50:51.303217+0000","last_undegraded":"2024-08-29T15:50:51.303217+0000","last_fullsized":"2024-08-29T15:50:51.303217+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T19:36:14.398486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1061,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.304995+0000","last_change":"2024-08-29T15:46:58.377694+0000","last_active":"2024-08-29T15:50:52.304995+0000","last_peered":"2024-08-29T15:50:52.304995+0000","last_clean":"2024-08-29T15:50:52.304995+0000","last_became_active":"2024-08-29T15:46:58.372090+0000","last_became_peered":"2024-08-29T15:46:58.372090+0000","last_unstale":"2024-08-29T15:50:52.304995+0000","last_undegraded":"2024-08-29T15:50:52.304995+0000","last_fullsized":"2024-08-29T15:50:52.304995+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:19:50.948014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'149","reported_seq":581,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306567+0000","last_change":"2024-08-29T15:50:39.443241+0000","last_active":"2024-08-29T15:50:51.306567+0000","last_peered":"2024-08-29T15:50:51.306567+0000","last_clean":"2024-08-29T15:50:51.306567+0000","last_became_active":"2024-08-29T15:50:37.804640+0000","last_became_peered":"2024-08-29T15:50:37.804640+0000","last_unstale":"2024-08-29T15:50:51.306567+0000","last_undegraded":"2024-08-29T15:50:51.306567+0000","last_fullsized":"2024-08-29T15:50:51.306567+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":359,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:39.443139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:39.443139+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:16:35.360223+0000","scrub_duration":0.0020546499999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1022,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307255+0000","last_change":"2024-08-29T15:19:46.801332+0000","last_active":"2024-08-29T15:50:52.307255+0000","last_peered":"2024-08-29T15:50:52.307255+0000","last_clean":"2024-08-29T15:50:52.307255+0000","last_became_active":"2024-08-29T15:19:46.800980+0000","last_became_peered":"2024-08-29T15:19:46.800980+0000","last_unstale":"2024-08-29T15:50:52.307255+0000","last_undegraded":"2024-08-29T15:50:52.307255+0000","last_fullsized":"2024-08-29T15:50:52.307255+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:46:37.039827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300961+0000","last_change":"2024-08-29T15:49:45.425648+0000","last_active":"2024-08-29T15:50:51.300961+0000","last_peered":"2024-08-29T15:50:51.300961+0000","last_clean":"2024-08-29T15:50:51.300961+0000","last_became_active":"2024-08-29T15:49:45.424290+0000","last_became_peered":"2024-08-29T15:49:45.424290+0000","last_unstale":"2024-08-29T15:50:51.300961+0000","last_undegraded":"2024-08-29T15:50:51.300961+0000","last_fullsized":"2024-08-29T15:50:51.300961+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T01:36:49.085868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'149","reported_seq":875,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.303423+0000","last_change":"2024-08-29T15:45:25.871113+0000","last_active":"2024-08-29T15:50:51.303423+0000","last_peered":"2024-08-29T15:50:51.303423+0000","last_clean":"2024-08-29T15:50:51.303423+0000","last_became_active":"2024-08-29T15:44:59.246190+0000","last_became_peered":"2024-08-29T15:44:59.246190+0000","last_unstale":"2024-08-29T15:50:51.303423+0000","last_undegraded":"2024-08-29T15:50:51.303423+0000","last_fullsized":"2024-08-29T15:50:51.303423+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:45:25.870329+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:45:25.870329+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:42.080608+0000","scrub_duration":0.0042968349999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":172,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306229+0000","last_change":"2024-08-29T15:50:31.765145+0000","last_active":"2024-08-29T15:50:51.306229+0000","last_peered":"2024-08-29T15:50:51.306229+0000","last_clean":"2024-08-29T15:50:51.306229+0000","last_became_active":"2024-08-29T15:50:31.764814+0000","last_became_peered":"2024-08-29T15:50:31.764814+0000","last_unstale":"2024-08-29T15:50:51.306229+0000","last_undegraded":"2024-08-29T15:50:51.306229+0000","last_fullsized":"2024-08-29T15:50:51.306229+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:57:03.810863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":103,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307499+0000","last_change":"2024-08-29T15:50:52.307381+0000","last_active":"2024-08-29T15:50:52.307499+0000","last_peered":"2024-08-29T15:50:52.307499+0000","last_clean":"2024-08-29T15:50:52.307499+0000","last_became_active":"2024-08-29T15:50:52.306942+0000","last_became_peered":"2024-08-29T15:50:52.306942+0000","last_unstale":"2024-08-29T15:50:52.307499+0000","last_undegraded":"2024-08-29T15:50:52.307499+0000","last_fullsized":"2024-08-29T15:50:52.307499+0000","mapping_epoch":572,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":573,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:47:54.851578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"90'149","reported_seq":544,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306730+0000","last_change":"2024-08-29T15:50:32.958662+0000","last_active":"2024-08-29T15:50:51.306730+0000","last_peered":"2024-08-29T15:50:51.306730+0000","last_clean":"2024-08-29T15:50:51.306730+0000","last_became_active":"2024-08-29T15:50:31.766024+0000","last_became_peered":"2024-08-29T15:50:31.766024+0000","last_unstale":"2024-08-29T15:50:51.306730+0000","last_undegraded":"2024-08-29T15:50:51.306730+0000","last_fullsized":"2024-08-29T15:50:51.306730+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:50:32.958535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:50:32.958535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:46:32.978131+0000","scrub_duration":0.00223508,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":351,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.302961+0000","last_change":"2024-08-29T15:50:25.444614+0000","last_active":"2024-08-29T15:50:51.302961+0000","last_peered":"2024-08-29T15:50:51.302961+0000","last_clean":"2024-08-29T15:50:51.302961+0000","last_became_active":"2024-08-29T15:50:25.444264+0000","last_became_peered":"2024-08-29T15:50:25.444264+0000","last_unstale":"2024-08-29T15:50:51.302961+0000","last_undegraded":"2024-08-29T15:50:51.302961+0000","last_fullsized":"2024-08-29T15:50:51.302961+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:11:36.513688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":201,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307672+0000","last_change":"2024-08-29T15:50:37.806818+0000","last_active":"2024-08-29T15:50:51.307672+0000","last_peered":"2024-08-29T15:50:51.307672+0000","last_clean":"2024-08-29T15:50:51.307672+0000","last_became_active":"2024-08-29T15:50:37.806392+0000","last_became_peered":"2024-08-29T15:50:37.806392+0000","last_unstale":"2024-08-29T15:50:51.307672+0000","last_undegraded":"2024-08-29T15:50:51.307672+0000","last_fullsized":"2024-08-29T15:50:51.307672+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T20:46:27.096632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'149","reported_seq":1226,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305110+0000","last_change":"2024-08-29T15:44:59.240657+0000","last_active":"2024-08-29T15:50:52.305110+0000","last_peered":"2024-08-29T15:50:52.305110+0000","last_clean":"2024-08-29T15:50:52.305110+0000","last_became_active":"2024-08-29T15:44:59.240261+0000","last_became_peered":"2024-08-29T15:44:59.240261+0000","last_unstale":"2024-08-29T15:50:52.305110+0000","last_undegraded":"2024-08-29T15:50:52.305110+0000","last_fullsized":"2024-08-29T15:50:52.305110+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.834998+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.834998+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T15:47:15.459581+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300735+0000","last_change":"2024-08-29T15:49:45.426284+0000","last_active":"2024-08-29T15:50:51.300735+0000","last_peered":"2024-08-29T15:50:51.300735+0000","last_clean":"2024-08-29T15:50:51.300735+0000","last_became_active":"2024-08-29T15:49:45.425997+0000","last_became_peered":"2024-08-29T15:49:45.425997+0000","last_unstale":"2024-08-29T15:50:51.300735+0000","last_undegraded":"2024-08-29T15:50:51.300735+0000","last_fullsized":"2024-08-29T15:50:51.300735+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:16:22.675142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":200,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307722+0000","last_change":"2024-08-29T15:50:37.808758+0000","last_active":"2024-08-29T15:50:51.307722+0000","last_peered":"2024-08-29T15:50:51.307722+0000","last_clean":"2024-08-29T15:50:51.307722+0000","last_became_active":"2024-08-29T15:50:37.808419+0000","last_became_peered":"2024-08-29T15:50:37.808419+0000","last_unstale":"2024-08-29T15:50:51.307722+0000","last_undegraded":"2024-08-29T15:50:51.307722+0000","last_fullsized":"2024-08-29T15:50:51.307722+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:22:18.897463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"90'149","reported_seq":968,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.307203+0000","last_change":"2024-08-29T15:50:37.807254+0000","last_active":"2024-08-29T15:50:51.307203+0000","last_peered":"2024-08-29T15:50:51.307203+0000","last_clean":"2024-08-29T15:50:51.307203+0000","last_became_active":"2024-08-29T15:50:37.806806+0000","last_became_peered":"2024-08-29T15:50:37.806806+0000","last_unstale":"2024-08-29T15:50:51.307203+0000","last_undegraded":"2024-08-29T15:50:51.307203+0000","last_fullsized":"2024-08-29T15:50:51.307203+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:24.799642+0000","last_deep_scrub":"90'149","last_deep_scrub_stamp":"2024-08-29T15:43:29.760194+0000","last_clean_scrub_stamp":"2024-08-29T15:44:24.799642+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:09:01.034776+0000","scrub_duration":0.0012023820000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":235,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306197+0000","last_change":"2024-08-29T15:50:31.763671+0000","last_active":"2024-08-29T15:50:51.306197+0000","last_peered":"2024-08-29T15:50:51.306197+0000","last_clean":"2024-08-29T15:50:51.306197+0000","last_became_active":"2024-08-29T15:50:31.763107+0000","last_became_peered":"2024-08-29T15:50:31.763107+0000","last_unstale":"2024-08-29T15:50:51.306197+0000","last_undegraded":"2024-08-29T15:50:51.306197+0000","last_fullsized":"2024-08-29T15:50:51.306197+0000","mapping_epoch":563,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:59:57.907456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":153,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305285+0000","last_change":"2024-08-29T15:48:12.268685+0000","last_active":"2024-08-29T15:50:52.305285+0000","last_peered":"2024-08-29T15:50:52.305285+0000","last_clean":"2024-08-29T15:50:52.305285+0000","last_became_active":"2024-08-29T15:48:12.268311+0000","last_became_peered":"2024-08-29T15:48:12.268311+0000","last_unstale":"2024-08-29T15:50:52.305285+0000","last_undegraded":"2024-08-29T15:50:52.305285+0000","last_fullsized":"2024-08-29T15:50:52.305285+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T02:30:18.347971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'149","reported_seq":1054,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300867+0000","last_change":"2024-08-29T15:49:48.295818+0000","last_active":"2024-08-29T15:50:51.300867+0000","last_peered":"2024-08-29T15:50:51.300867+0000","last_clean":"2024-08-29T15:50:51.300867+0000","last_became_active":"2024-08-29T15:49:48.294380+0000","last_became_peered":"2024-08-29T15:49:48.294380+0000","last_unstale":"2024-08-29T15:50:51.300867+0000","last_undegraded":"2024-08-29T15:50:51.300867+0000","last_fullsized":"2024-08-29T15:50:51.300867+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:26.831081+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:26.831081+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:23:06.262402+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":203,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.306795+0000","last_change":"2024-08-29T15:50:37.805588+0000","last_active":"2024-08-29T15:50:51.306795+0000","last_peered":"2024-08-29T15:50:51.306795+0000","last_clean":"2024-08-29T15:50:51.306795+0000","last_became_active":"2024-08-29T15:50:37.805121+0000","last_became_peered":"2024-08-29T15:50:37.805121+0000","last_unstale":"2024-08-29T15:50:51.306795+0000","last_undegraded":"2024-08-29T15:50:51.306795+0000","last_fullsized":"2024-08-29T15:50:51.306795+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:37:58.234694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.307192+0000","last_change":"2024-08-29T15:48:12.267269+0000","last_active":"2024-08-29T15:50:52.307192+0000","last_peered":"2024-08-29T15:50:52.307192+0000","last_clean":"2024-08-29T15:50:52.307192+0000","last_became_active":"2024-08-29T15:48:12.266893+0000","last_became_peered":"2024-08-29T15:48:12.266893+0000","last_unstale":"2024-08-29T15:50:52.307192+0000","last_undegraded":"2024-08-29T15:50:52.307192+0000","last_fullsized":"2024-08-29T15:50:52.307192+0000","mapping_epoch":515,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":516,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T22:07:31.630164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'149","reported_seq":1182,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.422907+0000","last_change":"2024-08-29T15:47:46.309647+0000","last_active":"2024-08-29T15:50:52.422907+0000","last_peered":"2024-08-29T15:50:52.422907+0000","last_clean":"2024-08-29T15:50:52.422907+0000","last_became_active":"2024-08-29T15:47:46.309290+0000","last_became_peered":"2024-08-29T15:47:46.309290+0000","last_unstale":"2024-08-29T15:50:52.422907+0000","last_undegraded":"2024-08-29T15:50:52.422907+0000","last_fullsized":"2024-08-29T15:50:52.422907+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:27.615446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:27.615446+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:41:58.511333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306671+0000","last_change":"2024-08-29T15:46:58.374246+0000","last_active":"2024-08-29T15:50:52.306671+0000","last_peered":"2024-08-29T15:50:52.306671+0000","last_clean":"2024-08-29T15:50:52.306671+0000","last_became_active":"2024-08-29T15:46:58.373708+0000","last_became_peered":"2024-08-29T15:46:58.373708+0000","last_unstale":"2024-08-29T15:50:52.306671+0000","last_undegraded":"2024-08-29T15:50:52.306671+0000","last_fullsized":"2024-08-29T15:50:52.306671+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:17:27.833391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":37,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.301260+0000","last_change":"2024-08-29T15:49:45.426791+0000","last_active":"2024-08-29T15:50:51.301260+0000","last_peered":"2024-08-29T15:50:51.301260+0000","last_clean":"2024-08-29T15:50:51.301260+0000","last_became_active":"2024-08-29T15:49:45.426343+0000","last_became_peered":"2024-08-29T15:49:45.426343+0000","last_unstale":"2024-08-29T15:50:51.301260+0000","last_undegraded":"2024-08-29T15:50:51.301260+0000","last_fullsized":"2024-08-29T15:50:51.301260+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T17:15:36.607053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'149","reported_seq":1085,"reported_epoch":572,"state":"active+clean","last_fresh":"2024-08-29T15:50:51.300623+0000","last_change":"2024-08-29T15:49:48.296814+0000","last_active":"2024-08-29T15:50:51.300623+0000","last_peered":"2024-08-29T15:50:51.300623+0000","last_clean":"2024-08-29T15:50:51.300623+0000","last_became_active":"2024-08-29T15:49:48.295396+0000","last_became_peered":"2024-08-29T15:49:48.295396+0000","last_unstale":"2024-08-29T15:50:51.300623+0000","last_undegraded":"2024-08-29T15:50:51.300623+0000","last_fullsized":"2024-08-29T15:50:51.300623+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:43:31.968809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:43:31.968809+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T16:36:15.745876+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":319,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306885+0000","last_change":"2024-08-29T15:50:37.806791+0000","last_active":"2024-08-29T15:50:52.306885+0000","last_peered":"2024-08-29T15:50:52.306885+0000","last_clean":"2024-08-29T15:50:52.306885+0000","last_became_active":"2024-08-29T15:50:37.806406+0000","last_became_peered":"2024-08-29T15:50:37.806406+0000","last_unstale":"2024-08-29T15:50:52.306885+0000","last_undegraded":"2024-08-29T15:50:52.306885+0000","last_fullsized":"2024-08-29T15:50:52.306885+0000","mapping_epoch":567,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":568,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:31.386831+0000","last_clean_scrub_stamp":"2024-08-29T14:55:31.386831+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T21:11:48.207353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1070,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.305286+0000","last_change":"2024-08-29T15:47:12.481794+0000","last_active":"2024-08-29T15:50:52.305286+0000","last_peered":"2024-08-29T15:50:52.305286+0000","last_clean":"2024-08-29T15:50:52.305286+0000","last_became_active":"2024-08-29T15:47:12.480977+0000","last_became_peered":"2024-08-29T15:47:12.480977+0000","last_unstale":"2024-08-29T15:50:52.305286+0000","last_undegraded":"2024-08-29T15:50:52.305286+0000","last_fullsized":"2024-08-29T15:50:52.305286+0000","mapping_epoch":501,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:55:32.389086+0000","last_clean_scrub_stamp":"2024-08-29T14:55:32.389086+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-31T00:28:29.573851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'149","reported_seq":1057,"reported_epoch":573,"state":"active+clean","last_fresh":"2024-08-29T15:50:52.306790+0000","last_change":"2024-08-29T15:49:48.295774+0000","last_active":"2024-08-29T15:50:52.306790+0000","last_peered":"2024-08-29T15:50:52.306790+0000","last_clean":"2024-08-29T15:50:52.306790+0000","last_became_active":"2024-08-29T15:49:48.295060+0000","last_became_peered":"2024-08-29T15:49:48.295060+0000","last_unstale":"2024-08-29T15:50:52.306790+0000","last_undegraded":"2024-08-29T15:50:52.306790+0000","last_fullsized":"2024-08-29T15:50:52.306790+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":457,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'149","last_scrub_stamp":"2024-08-29T15:44:36.699540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-29T14:51:49.042786+0000","last_clean_scrub_stamp":"2024-08-29T15:44:36.699540+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-30T18:14:30.458306+0000","scrub_duration":0.0014941869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":8562,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26245,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5987,"num_read_kb":13000,"num_write":846,"num_write_kb":12594,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":16777254,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234685,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909744,"kb_used_data":12320,"kb_used_omap":1,"kb_used_meta":897406,"kb_avail":92835408,"statfs":{"total":95995035648,"available":95063457792,"internally_reserved":0,"allocated":12615680,"data_stored":9073115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":918944203},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496316,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":707172,"kb_used_data":8276,"kb_used_omap":1,"kb_used_meta":698878,"kb_avail":93037980,"statfs":{"total":95995035648,"available":95270891520,"internally_reserved":0,"allocated":8474624,"data_stored":4943473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":715651527},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":201,"seq":863288426903,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":640864,"kb_used_data":8208,"kb_used_omap":8,"kb_used_meta":632631,"kb_avail":93104288,"statfs":{"total":95995035648,"available":95338790912,"internally_reserved":0,"allocated":8404992,"data_stored":4874249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8310,"internal_metadata":647815050},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019595,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":971812,"kb_used_data":12056,"kb_used_omap":1,"kb_used_meta":959742,"kb_avail":92773340,"statfs":{"total":95995035648,"available":94999900160,"internally_reserved":0,"allocated":12345344,"data_stored":8810154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":982776262},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":109,"seq":468151435779,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004760,"kb_used_data":8264,"kb_used_omap":4,"kb_used_meta":996475,"kb_avail":92740392,"statfs":{"total":95995035648,"available":94966161408,"internally_reserved":0,"allocated":8462336,"data_stored":4939003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4855,"internal_metadata":1020390665},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":217,"seq":932007903631,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":793812,"kb_used_data":8200,"kb_used_omap":10,"kb_used_meta":785589,"kb_avail":92951340,"statfs":{"total":95995035648,"available":95182172160,"internally_reserved":0,"allocated":8396800,"data_stored":4873357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10736,"internal_metadata":804443664},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":99,"seq":425201762832,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":992020,"kb_used_data":7940,"kb_used_omap":7,"kb_used_meta":984056,"kb_avail":92753132,"statfs":{"total":95995035648,"available":94979207168,"internally_reserved":0,"allocated":8130560,"data_stored":4610107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7838,"internal_metadata":1007673698},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926356,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":936716,"kb_used_data":12356,"kb_used_omap":7,"kb_used_meta":924344,"kb_avail":92808436,"statfs":{"total":95995035648,"available":95035838464,"internally_reserved":0,"allocated":12652544,"data_stored":9132380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8049,"internal_metadata":946528399},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3922,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6455,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6244,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3567,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3263,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6716,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-29T15:51:43.140 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-29T15:51:43.140 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_1f4056d968930e12926c25a8f13ee8ed6047c791/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_1f4056d968930e12926c25a8f13ee8ed6047c791/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi027 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-29T15:51:43.320 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=d184d9e03cae4a048b9c71d252c83578 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_1f4056d968930e12926c25a8f13ee8ed6047c791/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_1f4056d968930e12926c25a8f13ee8ed6047c791/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi027 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-29T15:51:43.323 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-08-29T15:51:43.336 INFO:tasks.cephadm:Teardown begin 2024-08-29T15:51:43.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-29T15:51:43.381 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-29T15:51:43.430 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-08-29T15:51:43.430 DEBUG:teuthology.orchestra.run.smithi027:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-29T15:51:43.448 DEBUG:teuthology.orchestra.run.smithi171:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-29T15:51:43.475 INFO:tasks.cephadm:Stopping all daemons... 2024-08-29T15:51:43.476 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-08-29T15:51:43.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a 2024-08-29T15:51:43.767 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:43 smithi171 ceph-mon[90031]: from='client.47564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:51:43.818 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:43 smithi027 ceph-mon[98970]: from='client.47564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:51:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:43 smithi027 ceph-mon[93921]: from='client.47564 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-29T15:51:43.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:43 smithi027 systemd[1]: Stopping Ceph mon.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:51:44.123 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:51:43.823+0000 7f3f1298e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:51:44.123 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a[93917]: 2024-08-29T15:51:43.823+0000 7f3f1298e640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-08-29T15:51:44.123 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:43 smithi027 podman[410141]: 2024-08-29 15:51:43.875017487 +0000 UTC m=+0.175985513 container died 1ad77d7a522185b89efc5670afff46beacfb3567443cad40629c75249de4aaf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-08-29T15:51:44.124 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:43 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:43] ENGINE Bus STOPPING 2024-08-29T15:51:44.124 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:44 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T15:51:44.124 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:44 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:44] ENGINE Bus STOPPED 2024-08-29T15:51:44.124 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:44 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:44] ENGINE Bus STARTING 2024-08-29T15:51:44.124 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:44 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:44] ENGINE Serving on http://:::9283 2024-08-29T15:51:44.124 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:44 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:44] ENGINE Bus STARTED 2024-08-29T15:51:44.784 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:44 smithi027 podman[410141]: 2024-08-29 15:51:44.465878444 +0000 UTC m=+0.766846469 container cleanup 1ad77d7a522185b89efc5670afff46beacfb3567443cad40629c75249de4aaf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-08-29T15:51:44.785 INFO:journalctl@ceph.mon.a.smithi027.stdout:Aug 29 15:51:44 smithi027 bash[410141]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-a 2024-08-29T15:51:44.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.a.service' 2024-08-29T15:51:44.950 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:51:44.950 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-08-29T15:51:44.950 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-08-29T15:51:44.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.c 2024-08-29T15:51:45.297 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:45 smithi027 systemd[1]: Stopping Ceph mon.c for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:51:45.568 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:45] ENGINE Bus STOPPING 2024-08-29T15:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c[98966]: 2024-08-29T15:51:45.295+0000 7f21411f3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c[98966]: 2024-08-29T15:51:45.295+0000 7f21411f3640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-08-29T15:51:45.569 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:45 smithi027 podman[410275]: 2024-08-29 15:51:45.479976159 +0000 UTC m=+0.296548319 container died 78f3772b32b4604923da25350d8abc84dbb958fe1e8b2a9f1fad68ea3800e4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:51:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:45 smithi027 podman[410275]: 2024-08-29 15:51:45.846448892 +0000 UTC m=+0.663021054 container cleanup 78f3772b32b4604923da25350d8abc84dbb958fe1e8b2a9f1fad68ea3800e4ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:51:46.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Aug 29 15:51:45 smithi027 bash[410275]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-c 2024-08-29T15:51:46.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-29T15:51:46.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:45] ENGINE Bus STOPPED 2024-08-29T15:51:46.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:45] ENGINE Bus STARTING 2024-08-29T15:51:46.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:45] ENGINE Serving on http://:::9283 2024-08-29T15:51:46.069 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:45 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: [29/Aug/2024:15:51:45] ENGINE Bus STARTED 2024-08-29T15:51:46.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.c.service' 2024-08-29T15:51:46.286 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:51:46.286 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-08-29T15:51:46.286 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-08-29T15:51:46.286 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.b 2024-08-29T15:51:46.603 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:46 smithi171 systemd[1]: Stopping Ceph mon.b for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:51:46.858 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:46 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b[90027]: 2024-08-29T15:51:46.600+0000 7f667260b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:51:46.858 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:46 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b[90027]: 2024-08-29T15:51:46.601+0000 7f667260b640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-08-29T15:51:47.158 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:46 smithi171 podman[292287]: 2024-08-29 15:51:46.857350322 +0000 UTC m=+0.359692715 container died e4fb79954e664eba897c32842e8ee136a53dbdfd876669e42455bbaf0aca0714 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-29T15:51:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:47 smithi171 podman[292287]: 2024-08-29 15:51:47.352912071 +0000 UTC m=+0.855254451 container cleanup e4fb79954e664eba897c32842e8ee136a53dbdfd876669e42455bbaf0aca0714 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-08-29T15:51:47.517 INFO:journalctl@ceph.mon.b.smithi171.stdout:Aug 29 15:51:47 smithi171 bash[292287]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mon-b 2024-08-29T15:51:47.750 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mon.b.service' 2024-08-29T15:51:47.787 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:51:47.787 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-08-29T15:51:47.787 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-08-29T15:51:47.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y 2024-08-29T15:51:48.068 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:47 smithi027 systemd[1]: Stopping Ceph mgr.y for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:51:48.419 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:48 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:51:48.101+0000 7f7e4859c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:51:48.419 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:48 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y[75520]: 2024-08-29T15:51:48.101+0000 7f7e4859c640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-08-29T15:51:48.419 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:48 smithi027 podman[410407]: 2024-08-29 15:51:48.14487151 +0000 UTC m=+0.159272879 container died f766ab17b7e23ead1473deba26bb25300675dc92fa7dae5b45955b1b858ec196 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-08-29T15:51:48.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:48 smithi027 podman[410407]: 2024-08-29 15:51:48.643223632 +0000 UTC m=+0.657624954 container cleanup f766ab17b7e23ead1473deba26bb25300675dc92fa7dae5b45955b1b858ec196 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=) 2024-08-29T15:51:48.818 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Aug 29 15:51:48 smithi027 bash[410407]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-y 2024-08-29T15:51:49.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.y.service' 2024-08-29T15:51:49.203 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:51:49.204 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-08-29T15:51:49.204 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-08-29T15:51:49.204 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x 2024-08-29T15:51:49.517 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:51:49 smithi171 systemd[1]: Stopping Ceph mgr.x for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:51:49.828 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:51:49 smithi171 podman[292408]: 2024-08-29 15:51:49.538495755 +0000 UTC m=+0.138753170 container died 85b20cdc09553c648a2259b3a7d00b55b90fcf6653e89cc622e6f7c67c8a386d (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d) 2024-08-29T15:51:50.127 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:51:49 smithi171 podman[292408]: 2024-08-29 15:51:49.907734755 +0000 UTC m=+0.507992159 container cleanup 85b20cdc09553c648a2259b3a7d00b55b90fcf6653e89cc622e6f7c67c8a386d (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:51:50.127 INFO:journalctl@ceph.mgr.x.smithi171.stdout:Aug 29 15:51:49 smithi171 bash[292408]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-mgr-x 2024-08-29T15:51:50.349 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@mgr.x.service' 2024-08-29T15:51:50.384 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:51:50.385 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-08-29T15:51:50.385 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-08-29T15:51:50.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.0 2024-08-29T15:51:50.568 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:50 smithi027 systemd[1]: Stopping Ceph osd.0 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:51:51.068 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[109907]: 2024-08-29T15:51:50.705+0000 7fe72401d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:51:51.068 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[109907]: 2024-08-29T15:51:50.705+0000 7fe72401d640 -1 osd.0 573 *** Got signal Terminated *** 2024-08-29T15:51:51.069 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:50 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0[109907]: 2024-08-29T15:51:50.705+0000 7fe72401d640 -1 osd.0 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:51:56.534 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:56 smithi027 podman[410531]: 2024-08-29 15:51:56.231519911 +0000 UTC m=+5.655291779 container died 13a2862581674ca9107c716f566f20e62e1f40f49de6c7a6fc733c18228c2771 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:51:56.818 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:56 smithi027 podman[410531]: 2024-08-29 15:51:56.707676478 +0000 UTC m=+6.131448348 container cleanup 13a2862581674ca9107c716f566f20e62e1f40f49de6c7a6fc733c18228c2771 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:51:56.818 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:56 smithi027 bash[410531]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0 2024-08-29T15:51:57.318 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:56 smithi027 podman[410547]: 2024-08-29 15:51:56.976184875 +0000 UTC m=+0.739961497 container remove 13a2862581674ca9107c716f566f20e62e1f40f49de6c7a6fc733c18228c2771 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-29T15:51:57.747 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:57 smithi027 podman[410636]: 2024-08-29 15:51:57.243709416 +0000 UTC m=+0.016589041 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:51:57.748 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:57 smithi027 podman[410636]: 2024-08-29 15:51:57.343615934 +0000 UTC m=+0.116495546 container create c2397b5e99deddf99057475f67cf2d5ad1920a529067b92ffa09b610bef3f545 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d) 2024-08-29T15:51:57.748 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:57 smithi027 podman[410636]: 2024-08-29 15:51:57.563022926 +0000 UTC m=+0.335902538 container init c2397b5e99deddf99057475f67cf2d5ad1920a529067b92ffa09b610bef3f545 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-29T15:51:57.748 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:57 smithi027 podman[410636]: 2024-08-29 15:51:57.566502313 +0000 UTC m=+0.339381932 container start c2397b5e99deddf99057475f67cf2d5ad1920a529067b92ffa09b610bef3f545 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:51:58.068 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:57 smithi027 podman[410636]: 2024-08-29 15:51:57.746477291 +0000 UTC m=+0.519357015 container attach c2397b5e99deddf99057475f67cf2d5ad1920a529067b92ffa09b610bef3f545 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:51:58.934 INFO:journalctl@ceph.osd.0.smithi027.stdout:Aug 29 15:51:58 smithi027 podman[410636]: 2024-08-29 15:51:58.599193515 +0000 UTC m=+1.372073146 container died c2397b5e99deddf99057475f67cf2d5ad1920a529067b92ffa09b610bef3f545 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS) 2024-08-29T15:51:59.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.0.service' 2024-08-29T15:51:59.267 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:51:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:51:58.863Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T15:51:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:51:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:51:58.867Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T15:51:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:51:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:51:58.867Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T15:51:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:51:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:51:58.867Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T15:51:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:51:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:51:58.869Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T15:51:59.268 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:51:58 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:51:58.869Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-08-29T15:51:59.306 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:51:59.307 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-08-29T15:51:59.307 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-08-29T15:51:59.307 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.1 2024-08-29T15:51:59.568 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:51:59 smithi027 systemd[1]: Stopping Ceph osd.1 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:00.068 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:51:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[114482]: 2024-08-29T15:51:59.661+0000 7f1a033fb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:52:00.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:51:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[114482]: 2024-08-29T15:51:59.661+0000 7f1a033fb640 -1 osd.1 573 *** Got signal Terminated *** 2024-08-29T15:52:00.069 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:51:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1[114482]: 2024-08-29T15:51:59.661+0000 7f1a033fb640 -1 osd.1 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:52:05.432 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:05 smithi027 podman[410741]: 2024-08-29 15:52:05.134274604 +0000 UTC m=+5.596725562 container died fe9dd18c3ce6ed073e91ddd0660c23bd2c9f9cbd9d2ecbbf989823852e2ad4e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-29T15:52:05.818 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:05 smithi027 podman[410741]: 2024-08-29 15:52:05.603901689 +0000 UTC m=+6.066352643 container cleanup fe9dd18c3ce6ed073e91ddd0660c23bd2c9f9cbd9d2ecbbf989823852e2ad4e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-29T15:52:05.818 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:05 smithi027 bash[410741]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1 2024-08-29T15:52:06.173 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:05 smithi027 podman[410756]: 2024-08-29 15:52:05.855406329 +0000 UTC m=+0.716318104 container remove fe9dd18c3ce6ed073e91ddd0660c23bd2c9f9cbd9d2ecbbf989823852e2ad4e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-29T15:52:06.442 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:06 smithi027 podman[410842]: 2024-08-29 15:52:06.17307699 +0000 UTC m=+0.105484131 container create 08c530d2775894e4a6e18b39c8c51d7e3173734455bedbd8f2ea549878548989 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:52:06.443 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:06 smithi027 podman[410842]: 2024-08-29 15:52:06.084153367 +0000 UTC m=+0.016560481 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:52:06.443 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:06 smithi027 podman[410842]: 2024-08-29 15:52:06.36705802 +0000 UTC m=+0.299465128 container init 08c530d2775894e4a6e18b39c8c51d7e3173734455bedbd8f2ea549878548989 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-29T15:52:06.443 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:06 smithi027 podman[410842]: 2024-08-29 15:52:06.37033344 +0000 UTC m=+0.302740540 container start 08c530d2775894e4a6e18b39c8c51d7e3173734455bedbd8f2ea549878548989 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-08-29T15:52:06.443 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:06 smithi027 podman[410842]: 2024-08-29 15:52:06.44203235 +0000 UTC m=+0.374439450 container attach 08c530d2775894e4a6e18b39c8c51d7e3173734455bedbd8f2ea549878548989 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=) 2024-08-29T15:52:06.818 INFO:journalctl@ceph.osd.1.smithi027.stdout:Aug 29 15:52:06 smithi027 podman[410842]: 2024-08-29 15:52:06.564697818 +0000 UTC m=+0.497104918 container died 08c530d2775894e4a6e18b39c8c51d7e3173734455bedbd8f2ea549878548989 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-29T15:52:07.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.1.service' 2024-08-29T15:52:07.344 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:07.344 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-08-29T15:52:07.345 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-29T15:52:07.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2 2024-08-29T15:52:07.568 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:07 smithi027 systemd[1]: Stopping Ceph osd.2 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:08.068 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[188540]: 2024-08-29T15:52:07.666+0000 7fe5ec732640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:52:08.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[188540]: 2024-08-29T15:52:07.666+0000 7fe5ec732640 -1 osd.2 573 *** Got signal Terminated *** 2024-08-29T15:52:08.069 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:07 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2[188540]: 2024-08-29T15:52:07.666+0000 7fe5ec732640 -1 osd.2 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:52:13.543 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:13 smithi027 podman[410944]: 2024-08-29 15:52:13.268119508 +0000 UTC m=+5.727638002 container died 61ba5f8296d5ccbb0506a2576c8daa8f01e7e9753c7a2127d8c934f28fcecac1 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-29T15:52:13.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:13 smithi027 podman[410944]: 2024-08-29 15:52:13.733144473 +0000 UTC m=+6.192662973 container cleanup 61ba5f8296d5ccbb0506a2576c8daa8f01e7e9753c7a2127d8c934f28fcecac1 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-08-29T15:52:13.818 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:13 smithi027 bash[410944]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2 2024-08-29T15:52:14.318 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:14 smithi027 podman[410971]: 2024-08-29 15:52:14.055994038 +0000 UTC m=+0.783124414 container remove 61ba5f8296d5ccbb0506a2576c8daa8f01e7e9753c7a2127d8c934f28fcecac1 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-08-29T15:52:14.647 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:14 smithi027 podman[411057]: 2024-08-29 15:52:14.274418939 +0000 UTC m=+0.016468484 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:52:14.647 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:14 smithi027 podman[411057]: 2024-08-29 15:52:14.394207007 +0000 UTC m=+0.136256535 container create 3830e08044525907368500ddef5ec21cb38ca03e1f0ea1b6e35c0682685733ba (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-29T15:52:14.647 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:14 smithi027 podman[411057]: 2024-08-29 15:52:14.646539917 +0000 UTC m=+0.388589456 container init 3830e08044525907368500ddef5ec21cb38ca03e1f0ea1b6e35c0682685733ba (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-08-29T15:52:15.068 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:14 smithi027 podman[411057]: 2024-08-29 15:52:14.650512216 +0000 UTC m=+0.392561752 container start 3830e08044525907368500ddef5ec21cb38ca03e1f0ea1b6e35c0682685733ba (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:52:15.068 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:14 smithi027 podman[411057]: 2024-08-29 15:52:14.763404549 +0000 UTC m=+0.505454097 container attach 3830e08044525907368500ddef5ec21cb38ca03e1f0ea1b6e35c0682685733ba (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-29T15:52:15.068 INFO:journalctl@ceph.osd.2.smithi027.stdout:Aug 29 15:52:14 smithi027 podman[411057]: 2024-08-29 15:52:14.848762382 +0000 UTC m=+0.590811911 container died 3830e08044525907368500ddef5ec21cb38ca03e1f0ea1b6e35c0682685733ba (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-08-29T15:52:15.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.2.service' 2024-08-29T15:52:15.517 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:15.517 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-29T15:52:15.518 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-08-29T15:52:15.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.3 2024-08-29T15:52:15.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:15 smithi027 systemd[1]: Stopping Ceph osd.3 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:16.318 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:52:15.870+0000 7f6006ff8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:52:16.318 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:52:15.870+0000 7f6006ff8640 -1 osd.3 573 *** Got signal Terminated *** 2024-08-29T15:52:16.319 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:15 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:52:15.870+0000 7f6006ff8640 -1 osd.3 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:52:17.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:16 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:16.689+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:17.282 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:17.247+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:18.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:17 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:17.713+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:18.318 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:17 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:52:17.976+0000 7f5fff605640 -1 osd.3 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.748788+0000 front 2024-08-29T15:51:51.748936+0000 (oldest deadline 2024-08-29T15:52:17.648889+0000) 2024-08-29T15:52:18.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:18 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:18.241+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:19.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:18 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:18.761+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:19.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:18 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:18.990+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:19.318 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:18 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:52:18.998+0000 7f5fff605640 -1 osd.3 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.748788+0000 front 2024-08-29T15:51:51.748936+0000 (oldest deadline 2024-08-29T15:52:17.648889+0000) 2024-08-29T15:52:19.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:19 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:19.195+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:20.232 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:19 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:19.789+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:20.233 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:19 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:19.947+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:20.233 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:19 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:19.868+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:20.318 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:20 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3[123792]: 2024-08-29T15:52:20.034+0000 7f5fff605640 -1 osd.3 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.748788+0000 front 2024-08-29T15:51:51.748936+0000 (oldest deadline 2024-08-29T15:52:17.648889+0000) 2024-08-29T15:52:20.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:20.231+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:21.191 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:20.906+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:21.191 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:20.882+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:21.192 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:20 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:20.789+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:21.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:21.190+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:21.807 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:21 smithi027 podman[411158]: 2024-08-29 15:52:21.484439635 +0000 UTC m=+5.739563792 container died 14455322d13f1ebb461d9cdf90f07876d6a0a4a25bd4b2515e568ceda9cf1a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-08-29T15:52:22.017 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:21.914+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:22.017 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:21.845+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:22.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:21 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:21.759+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:22.068 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:21 smithi027 podman[411158]: 2024-08-29 15:52:21.97041443 +0000 UTC m=+6.225538589 container cleanup 14455322d13f1ebb461d9cdf90f07876d6a0a4a25bd4b2515e568ceda9cf1a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:52:22.068 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:21 smithi027 bash[411158]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3 2024-08-29T15:52:22.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:22 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:22.209+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:22.563 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:22 smithi027 podman[411174]: 2024-08-29 15:52:22.246644875 +0000 UTC m=+0.763064430 container remove 14455322d13f1ebb461d9cdf90f07876d6a0a4a25bd4b2515e568ceda9cf1a02 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-29T15:52:22.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:22 smithi027 podman[411267]: 2024-08-29 15:52:22.465744846 +0000 UTC m=+0.016136443 image pull d17246d170ec2b241f76e4089352ee25d88095b9137d452d079ccea04ae575d3 quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc 2024-08-29T15:52:22.818 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:22 smithi027 podman[411267]: 2024-08-29 15:52:22.56396172 +0000 UTC m=+0.114353337 container create c836cf08f534c587fb5fd954cccd9d5f56ad0a47b1688635fe9b6435950ddf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-29T15:52:22.819 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:22 smithi027 podman[411267]: 2024-08-29 15:52:22.783296733 +0000 UTC m=+0.333688316 container init c836cf08f534c587fb5fd954cccd9d5f56ad0a47b1688635fe9b6435950ddf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-29T15:52:22.819 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:22 smithi027 podman[411267]: 2024-08-29 15:52:22.786736493 +0000 UTC m=+0.337128064 container start c836cf08f534c587fb5fd954cccd9d5f56ad0a47b1688635fe9b6435950ddf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:52:23.017 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:22 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:22.872+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:23.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:22 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:22.731+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:23.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:22 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:22.853+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:23.278 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:22 smithi027 podman[411267]: 2024-08-29 15:52:22.87498573 +0000 UTC m=+0.425377316 container attach c836cf08f534c587fb5fd954cccd9d5f56ad0a47b1688635fe9b6435950ddf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-29T15:52:23.278 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:22 smithi027 podman[411267]: 2024-08-29 15:52:22.981748149 +0000 UTC m=+0.532139721 container died c836cf08f534c587fb5fd954cccd9d5f56ad0a47b1688635fe9b6435950ddf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:52:23.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:23.243+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:23.531 INFO:journalctl@ceph.osd.3.smithi027.stdout:Aug 29 15:52:23 smithi027 podman[411291]: 2024-08-29 15:52:23.511795842 +0000 UTC m=+0.520029749 container remove c836cf08f534c587fb5fd954cccd9d5f56ad0a47b1688635fe9b6435950ddf84 (image=quay.ceph.io/ceph-ci/ceph@sha256:a5f4fb9c4b095b4987d7731cc0a7a19fe76076fcf949594b647a426124f476bc, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-yuri4-testing-2024-08-28-1359-1f4056d, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-29T15:52:23.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.3.service' 2024-08-29T15:52:23.580 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:23.580 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-08-29T15:52:23.580 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-08-29T15:52:23.581 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.4 2024-08-29T15:52:23.911 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:23 smithi171 systemd[1]: Stopping Ceph osd.4 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:23.911 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:23.909+0000 7fe9d366e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:52:23.912 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:23.841+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:23.912 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:23.822+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:23.912 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:23.698+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:24.219 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:23.909+0000 7fe9d366e640 -1 osd.4 573 *** Got signal Terminated *** 2024-08-29T15:52:24.219 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:23 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:23.909+0000 7fe9d366e640 -1 osd.4 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:52:24.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:24 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:24.217+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:25.017 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:24 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:24.843+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:25.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:24 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:24.685+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:25.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:24 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:24.855+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:25.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:25 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:25.191+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:26.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:25 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:25.683+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:26.017 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:25 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:25.822+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:26.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:25 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:25.882+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:26.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:26 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:26.169+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:26.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:26 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:26.169+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:03.050984+0000 front 2024-08-29T15:52:03.050968+0000 (oldest deadline 2024-08-29T15:52:25.350918+0000) 2024-08-29T15:52:27.017 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:26 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:26.824+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:27.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:26 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:26.906+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:27.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:26 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:26.715+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:27.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:26 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:26.715+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:27.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:27.125+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:27.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:27.125+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:03.050984+0000 front 2024-08-29T15:52:03.050968+0000 (oldest deadline 2024-08-29T15:52:25.350918+0000) 2024-08-29T15:52:28.017 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:27.890+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:28.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:27.688+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:28.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:27.688+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:28.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:27.868+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:28.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:27 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:27.868+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:28.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:28.147+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:51.048382+0000 front 2024-08-29T15:51:51.048430+0000 (oldest deadline 2024-08-29T15:52:16.347899+0000) 2024-08-29T15:52:28.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4[27854]: 2024-08-29T15:52:28.147+0000 7fe9cbc7a640 -1 osd.4 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:03.050984+0000 front 2024-08-29T15:52:03.050968+0000 (oldest deadline 2024-08-29T15:52:25.350918+0000) 2024-08-29T15:52:29.017 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:28.874+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:29.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:28.699+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:29.018 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:28.699+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:29.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:28.869+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:29.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:28 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:28.869+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:29.834 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:29 smithi171 podman[292547]: 2024-08-29 15:52:29.56698655 +0000 UTC m=+5.788134950 container died f00fb58eb4c2b3705aa472620ddb22fcfde2d253d8ea15fe9f146359fa8c8a80 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-29T15:52:29.834 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:29 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:29.728+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:29.834 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:29 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:29.728+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:30.254 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:29 smithi171 podman[292547]: 2024-08-29 15:52:29.952346702 +0000 UTC m=+6.173495101 container cleanup f00fb58eb4c2b3705aa472620ddb22fcfde2d253d8ea15fe9f146359fa8c8a80 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c) 2024-08-29T15:52:30.255 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:29 smithi171 bash[292547]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4 2024-08-29T15:52:30.255 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:29 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:29.832+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:30.256 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:29 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:29.832+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:30.256 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:29 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:29.838+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:30.256 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:29 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:29.838+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:30.517 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:30 smithi171 podman[292562]: 2024-08-29 15:52:30.253982846 +0000 UTC m=+0.682579645 container remove f00fb58eb4c2b3705aa472620ddb22fcfde2d253d8ea15fe9f146359fa8c8a80 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4, CEPH_POINT_RELEASE=, RELEASE=reef-fe43f1c, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-08-29T15:52:30.883 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:30.757+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:30.883 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:30.757+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:30.883 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:30.787+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:30.883 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:30.787+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:30.884 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:30 smithi171 podman[292647]: 2024-08-29 15:52:30.519406167 +0000 UTC m=+0.016065762 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T15:52:30.884 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:30 smithi171 podman[292647]: 2024-08-29 15:52:30.629867921 +0000 UTC m=+0.126527507 container create 3c3b7a67cef2d67ec73bf44dd7aca47d7d6c61247c54918d2916002c55303615 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.1, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c) 2024-08-29T15:52:30.884 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:30.858+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:30.884 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:30 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:30.858+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:31.190 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:30 smithi171 podman[292647]: 2024-08-29 15:52:30.882287146 +0000 UTC m=+0.378946730 container init 3c3b7a67cef2d67ec73bf44dd7aca47d7d6c61247c54918d2916002c55303615 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-fe43f1c, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-29T15:52:31.191 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:30 smithi171 podman[292647]: 2024-08-29 15:52:30.88592186 +0000 UTC m=+0.382581450 container start 3c3b7a67cef2d67ec73bf44dd7aca47d7d6c61247c54918d2916002c55303615 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.1) 2024-08-29T15:52:31.191 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:30 smithi171 podman[292647]: 2024-08-29 15:52:30.990683975 +0000 UTC m=+0.487343562 container attach 3c3b7a67cef2d67ec73bf44dd7aca47d7d6c61247c54918d2916002c55303615 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-29T15:52:31.466 INFO:journalctl@ceph.osd.4.smithi171.stdout:Aug 29 15:52:31 smithi171 podman[292647]: 2024-08-29 15:52:31.190237473 +0000 UTC m=+0.686897062 container died 3c3b7a67cef2d67ec73bf44dd7aca47d7d6c61247c54918d2916002c55303615 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=) 2024-08-29T15:52:31.903 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.4.service' 2024-08-29T15:52:31.942 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:31.943 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-08-29T15:52:31.943 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-29T15:52:31.943 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.5 2024-08-29T15:52:32.004 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:31 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:31.797+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:32.004 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:31 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:31.797+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:32.005 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:31 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:31.771+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:32.005 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:31 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:31.771+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:32.006 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:31 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:31.856+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:32.006 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:31 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:31.856+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:32.267 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:32 smithi171 systemd[1]: Stopping Ceph osd.5 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:32.767 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:32.281+0000 7f0624b2f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:52:32.768 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:32.281+0000 7f0624b2f640 -1 osd.5 573 *** Got signal Terminated *** 2024-08-29T15:52:32.768 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:32.281+0000 7f0624b2f640 -1 osd.5 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:52:33.267 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:32.836+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:33.267 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:32.836+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:33.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:32.788+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:33.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:32.788+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:33.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:32.878+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:33.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:32 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:32.878+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:34.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:33.772+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:34.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:33.772+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:34.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:33.912+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:34.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:33.912+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:34.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:33.885+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:34.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:33.885+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:34.269 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:33 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:33.885+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.733360+0000 front 2024-08-29T15:52:09.733415+0000 (oldest deadline 2024-08-29T15:52:33.833275+0000) 2024-08-29T15:52:35.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:34.793+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:35.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:34.793+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:35.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:34.868+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:35.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:34.868+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:35.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:34.868+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.733360+0000 front 2024-08-29T15:52:09.733415+0000 (oldest deadline 2024-08-29T15:52:33.833275+0000) 2024-08-29T15:52:35.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:34.910+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:35.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:34.910+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:35.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:34 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:34.910+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:36.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:35.825+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:36.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:35.825+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:36.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:35.825+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:36.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:35.906+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:36.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:35.906+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:36.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:35.906+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.733360+0000 front 2024-08-29T15:52:09.733415+0000 (oldest deadline 2024-08-29T15:52:33.833275+0000) 2024-08-29T15:52:36.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:35.923+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:36.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:35.923+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:36.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:35 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:35.923+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:37.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:36.783+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:37.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:36.783+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:37.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:36.783+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:37.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:36.935+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:52.230922+0000 front 2024-08-29T15:51:52.231082+0000 (oldest deadline 2024-08-29T15:52:16.330966+0000) 2024-08-29T15:52:37.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:36.935+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:01.032070+0000 front 2024-08-29T15:52:01.031970+0000 (oldest deadline 2024-08-29T15:52:25.732005+0000) 2024-08-29T15:52:37.268 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5[131619]: 2024-08-29T15:52:36.935+0000 7f0621143640 -1 osd.5 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.733360+0000 front 2024-08-29T15:52:09.733415+0000 (oldest deadline 2024-08-29T15:52:33.833275+0000) 2024-08-29T15:52:37.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:36.947+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:37.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:36.947+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:37.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:36 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:36.947+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:38.117 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:37 smithi171 podman[292759]: 2024-08-29 15:52:37.807241714 +0000 UTC m=+5.632705517 container died db94cc15457ad714d1f99f9dc40d3c58479465ff4d188b709c06a72c3a04ad23 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c) 2024-08-29T15:52:38.117 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:37.786+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:38.118 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:37.786+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:38.118 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:37.786+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:38.118 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:37.934+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:38.118 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:37.934+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:38.118 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:37 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:37.934+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:38.459 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:38 smithi171 podman[292759]: 2024-08-29 15:52:38.223389745 +0000 UTC m=+6.048853548 container cleanup db94cc15457ad714d1f99f9dc40d3c58479465ff4d188b709c06a72c3a04ad23 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.1, RELEASE=reef-fe43f1c, org.label-schema.schema-version=1.0) 2024-08-29T15:52:38.460 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:38 smithi171 bash[292759]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5 2024-08-29T15:52:38.767 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:38 smithi171 podman[292775]: 2024-08-29 15:52:38.459075296 +0000 UTC m=+0.649503315 container remove db94cc15457ad714d1f99f9dc40d3c58479465ff4d188b709c06a72c3a04ad23 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5, io.buildah.version=1.37.1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-29T15:52:39.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:38 smithi171 podman[292866]: 2024-08-29 15:52:38.802380814 +0000 UTC m=+0.015954138 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T15:52:39.017 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:38 smithi171 podman[292866]: 2024-08-29 15:52:38.918571969 +0000 UTC m=+0.132145292 container create fb2e37c46ac1a8e70114ab151caae168cbed5e4646cd85469df20b7612cd4cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-fe43f1c, ceph=True, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-29T15:52:39.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:38.765+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:39.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:38.765+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:39.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:38.765+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:39.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:38.892+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:39.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:38.892+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:39.019 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:38 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:38.892+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:39.311 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:39 smithi171 podman[292866]: 2024-08-29 15:52:39.120973592 +0000 UTC m=+0.334546916 container init fb2e37c46ac1a8e70114ab151caae168cbed5e4646cd85469df20b7612cd4cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-29T15:52:39.311 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:39 smithi171 podman[292866]: 2024-08-29 15:52:39.124488407 +0000 UTC m=+0.338061742 container start fb2e37c46ac1a8e70114ab151caae168cbed5e4646cd85469df20b7612cd4cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:52:39.312 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:39 smithi171 podman[292866]: 2024-08-29 15:52:39.196494673 +0000 UTC m=+0.410068011 container attach fb2e37c46ac1a8e70114ab151caae168cbed5e4646cd85469df20b7612cd4cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.1, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-29T15:52:39.652 INFO:journalctl@ceph.osd.5.smithi171.stdout:Aug 29 15:52:39 smithi171 podman[292866]: 2024-08-29 15:52:39.311141366 +0000 UTC m=+0.524714695 container died fb2e37c46ac1a8e70114ab151caae168cbed5e4646cd85469df20b7612cd4cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-fe43f1c, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-29T15:52:39.924 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:39.776+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:39.924 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:39.776+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:39.924 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:39.777+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:39.959 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.5.service' 2024-08-29T15:52:40.038 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:40.038 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-29T15:52:40.039 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-29T15:52:40.039 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.6 2024-08-29T15:52:40.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:40 smithi171 systemd[1]: Stopping Ceph osd.6 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:40.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:39.923+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:40.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:39.923+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:40.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:39 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:39.923+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:40.767 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:40.386+0000 7f4bec0ea640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:52:40.768 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:40.386+0000 7f4bec0ea640 -1 osd.6 573 *** Got signal Terminated *** 2024-08-29T15:52:40.768 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:40.386+0000 7f4bec0ea640 -1 osd.6 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:52:41.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:40.791+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:41.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:40.791+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:41.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:40.791+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:41.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:40.909+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:41.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:40.909+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:41.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:40 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:40.909+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:42.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:41.822+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:42.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:41.822+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:42.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:41.822+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:42.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:41.858+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:42.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:41.858+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:42.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:41.858+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:42.269 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:41 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:41.858+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:43.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:42.871+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:43.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:42.871+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:43.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:42.871+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:43.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:42.871+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:43.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:42.787+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:43.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:42.787+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:43.268 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:42 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:42.787+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:44.017 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:43.905+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:44.017 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:43.905+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:44.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:43.905+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:44.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:43.905+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:44.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:43.742+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:44.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:43.742+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:44.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:43.742+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:44.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:43 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:43.742+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:18.510864+0000 front 2024-08-29T15:52:18.510856+0000 (oldest deadline 2024-08-29T15:52:43.711286+0000) 2024-08-29T15:52:45.017 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:44.727+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:55.207792+0000 front 2024-08-29T15:51:55.207986+0000 (oldest deadline 2024-08-29T15:52:18.707637+0000) 2024-08-29T15:52:45.017 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:44.727+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.008675+0000 front 2024-08-29T15:52:04.008833+0000 (oldest deadline 2024-08-29T15:52:26.908611+0000) 2024-08-29T15:52:45.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:44.727+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:11.510035+0000 front 2024-08-29T15:52:11.509972+0000 (oldest deadline 2024-08-29T15:52:35.009949+0000) 2024-08-29T15:52:45.018 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6[39262]: 2024-08-29T15:52:44.727+0000 7f4be86fe640 -1 osd.6 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:18.510864+0000 front 2024-08-29T15:52:18.510856+0000 (oldest deadline 2024-08-29T15:52:43.711286+0000) 2024-08-29T15:52:45.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:44.918+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:45.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:44.918+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:45.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:44.918+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:45.018 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:44 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:44.918+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:46.267 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:46 smithi171 podman[292970]: 2024-08-29 15:52:46.029041539 +0000 UTC m=+5.765273208 container died 228ad60cc49253d99d462b81c480fd91409ce1a3d913f85109fcf13ee07000ef (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-08-29T15:52:46.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:45 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:45.966+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:46.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:45 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:45.966+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:46.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:45 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:45.966+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:46.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:45 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:45.966+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:46.688 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:46 smithi171 podman[292970]: 2024-08-29 15:52:46.394651026 +0000 UTC m=+6.130882687 container cleanup 228ad60cc49253d99d462b81c480fd91409ce1a3d913f85109fcf13ee07000ef (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=reef-fe43f1c, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-08-29T15:52:46.688 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:46 smithi171 bash[292970]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6 2024-08-29T15:52:46.972 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:46 smithi171 podman[292985]: 2024-08-29 15:52:46.687588032 +0000 UTC m=+0.645546225 container remove 228ad60cc49253d99d462b81c480fd91409ce1a3d913f85109fcf13ee07000ef (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-29T15:52:47.226 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:47 smithi171 podman[293073]: 2024-08-29 15:52:46.904885667 +0000 UTC m=+0.016264836 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T15:52:47.226 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:47 smithi171 podman[293073]: 2024-08-29 15:52:47.022620736 +0000 UTC m=+0.133999905 container create 7731952a7309c11dcf611d185f7762e97e2d7a65bbab1a9fb82050a66d7b6259 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-fe43f1c, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-29T15:52:47.226 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:47 smithi171 podman[293073]: 2024-08-29 15:52:47.225406564 +0000 UTC m=+0.336785741 container init 7731952a7309c11dcf611d185f7762e97e2d7a65bbab1a9fb82050a66d7b6259 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-fe43f1c, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-29T15:52:47.226 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:46 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:46.970+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:47.227 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:46 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:46.970+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:47.227 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:46 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:46.970+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:47.227 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:46 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:46.970+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:47.517 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:47 smithi171 podman[293073]: 2024-08-29 15:52:47.229221012 +0000 UTC m=+0.340600179 container start 7731952a7309c11dcf611d185f7762e97e2d7a65bbab1a9fb82050a66d7b6259 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.1, RELEASE=reef-fe43f1c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-29T15:52:47.518 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:47 smithi171 podman[293073]: 2024-08-29 15:52:47.317119698 +0000 UTC m=+0.428498879 container attach 7731952a7309c11dcf611d185f7762e97e2d7a65bbab1a9fb82050a66d7b6259 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=reef-fe43f1c, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-08-29T15:52:47.518 INFO:journalctl@ceph.osd.6.smithi171.stdout:Aug 29 15:52:47 smithi171 podman[293073]: 2024-08-29 15:52:47.413069198 +0000 UTC m=+0.524448377 container died 7731952a7309c11dcf611d185f7762e97e2d7a65bbab1a9fb82050a66d7b6259 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-fe43f1c, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-08-29T15:52:48.018 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.6.service' 2024-08-29T15:52:48.059 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:48.059 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-29T15:52:48.059 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-29T15:52:48.059 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.7 2024-08-29T15:52:48.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:47 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:47.969+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:48.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:47 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:47.969+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:48.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:47 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:47.969+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:48.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:47 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:47.969+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:48.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 systemd[1]: Stopping Ceph osd.7 for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:48.768 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:48.447+0000 7f5376971640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-29T15:52:48.768 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:48.447+0000 7f5376971640 -1 osd.7 573 *** Got signal Terminated *** 2024-08-29T15:52:48.768 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:48.447+0000 7f5376971640 -1 osd.7 573 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-29T15:52:49.248 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:48.926+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:49.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:48.926+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:49.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:48.926+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:49.249 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:48 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:48.926+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:50.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:49 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:49.925+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:50.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:49 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:49.925+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:50.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:49 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:49.925+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:50.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:49 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:49.925+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:51.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:50 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:50.971+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:51.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:50 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:50.971+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:51.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:50 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:50.971+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:51.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:50 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:50.971+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:52.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:51 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:51.976+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:52.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:51 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:51.976+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:52.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:51 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:51.976+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:52.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:51 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:51.976+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:52.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:51 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:51.976+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.171:6804 osd.4 since back 2024-08-29T15:52:27.281444+0000 front 2024-08-29T15:52:27.281508+0000 (oldest deadline 2024-08-29T15:52:51.381443+0000) 2024-08-29T15:52:53.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:52 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:52.938+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-08-29T15:51:53.878206+0000 front 2024-08-29T15:51:53.878147+0000 (oldest deadline 2024-08-29T15:52:19.778125+0000) 2024-08-29T15:52:53.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:52 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:52.938+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-08-29T15:52:04.479034+0000 front 2024-08-29T15:52:04.479109+0000 (oldest deadline 2024-08-29T15:52:29.179028+0000) 2024-08-29T15:52:53.267 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:52 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:52.938+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-08-29T15:52:09.179645+0000 front 2024-08-29T15:52:09.179715+0000 (oldest deadline 2024-08-29T15:52:34.479591+0000) 2024-08-29T15:52:53.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:52 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:52.938+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-08-29T15:52:17.980421+0000 front 2024-08-29T15:52:17.980390+0000 (oldest deadline 2024-08-29T15:52:41.480328+0000) 2024-08-29T15:52:53.268 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:52 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7[45373]: 2024-08-29T15:52:52.938+0000 7f5372f85640 -1 osd.7 573 heartbeat_check: no reply from 172.21.15.171:6804 osd.4 since back 2024-08-29T15:52:27.281444+0000 front 2024-08-29T15:52:27.281508+0000 (oldest deadline 2024-08-29T15:52:51.381443+0000) 2024-08-29T15:52:54.400 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:54 smithi171 podman[293177]: 2024-08-29 15:52:54.11064842 +0000 UTC m=+5.798124487 container died c43a0db5d0af3e4ba648e7af598b39099236e2dbd98e30c28d8ef29f31d9a74d (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, CEPH_POINT_RELEASE=, RELEASE=reef-fe43f1c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-08-29T15:52:54.750 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:54 smithi171 podman[293177]: 2024-08-29 15:52:54.49010673 +0000 UTC m=+6.177582790 container cleanup c43a0db5d0af3e4ba648e7af598b39099236e2dbd98e30c28d8ef29f31d9a74d (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.1, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-fe43f1c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-29T15:52:54.750 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:54 smithi171 bash[293177]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7 2024-08-29T15:52:55.017 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:54 smithi171 podman[293203]: 2024-08-29 15:52:54.749637477 +0000 UTC m=+0.636284180 container remove c43a0db5d0af3e4ba648e7af598b39099236e2dbd98e30c28d8ef29f31d9a74d (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7, io.buildah.version=1.37.1, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-fe43f1c, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-29T15:52:55.420 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:55 smithi171 podman[293293]: 2024-08-29 15:52:54.999375597 +0000 UTC m=+0.016124553 image pull 96384990183a90e11fa6fcb78b6a44500ccf55da778999360ad8f92318176892 quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee 2024-08-29T15:52:55.420 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:55 smithi171 podman[293293]: 2024-08-29 15:52:55.108976536 +0000 UTC m=+0.125725551 container create 7f3c867926a3b45b015c355edb0ae69f2ecb75ace0c668732ffcfdd5a1b52bf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, RELEASE=reef-fe43f1c, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-29T15:52:55.421 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:55 smithi171 podman[293293]: 2024-08-29 15:52:55.327906201 +0000 UTC m=+0.344655158 container init 7f3c867926a3b45b015c355edb0ae69f2ecb75ace0c668732ffcfdd5a1b52bf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.1, ceph=True, RELEASE=reef-fe43f1c) 2024-08-29T15:52:55.421 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:55 smithi171 podman[293293]: 2024-08-29 15:52:55.331138288 +0000 UTC m=+0.347887245 container start 7f3c867926a3b45b015c355edb0ae69f2ecb75ace0c668732ffcfdd5a1b52bf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.1, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-29T15:52:55.421 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:55 smithi171 podman[293293]: 2024-08-29 15:52:55.41959937 +0000 UTC m=+0.436348351 container attach 7f3c867926a3b45b015c355edb0ae69f2ecb75ace0c668732ffcfdd5a1b52bf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-08-29T15:52:55.768 INFO:journalctl@ceph.osd.7.smithi171.stdout:Aug 29 15:52:55 smithi171 podman[293293]: 2024-08-29 15:52:55.52077053 +0000 UTC m=+0.537519492 container died 7f3c867926a3b45b015c355edb0ae69f2ecb75ace0c668732ffcfdd5a1b52bf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5108c7cbd104eb247639ca437f56fde756f773f0867466a7d6597908f67824ee, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=reef-fe43f1c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-29T15:52:56.082 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@osd.7.service' 2024-08-29T15:52:56.121 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:56.121 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-29T15:52:56.121 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-08-29T15:52:56.121 DEBUG:teuthology.orchestra.run.smithi171:> sudo systemctl stop ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a 2024-08-29T15:52:56.452 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 systemd[1]: Stopping Ceph prometheus.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:56.730 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-29T15:52:56.730 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-08-29T15:52:56.730 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-08-29T15:52:56.730 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-29T15:52:56.730 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-08-29T15:52:56.730 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-08-29T15:52:56.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-08-29T15:52:56.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-08-29T15:52:56.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.451Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-08-29T15:52:56.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.453Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-08-29T15:52:56.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.453Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-08-29T15:52:56.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a[76213]: ts=2024-08-29T15:52:56.453Z caller=main.go:1273 level=info msg="See you next time!" 2024-08-29T15:52:56.731 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 podman[293395]: 2024-08-29 15:52:56.461857318 +0000 UTC m=+0.113754543 container died b0e73daa625e9cce05ef1ecc82bc8de739d30539b6dfed3708bd0ccf79711f32 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T15:52:57.017 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 podman[293395]: 2024-08-29 15:52:56.809134675 +0000 UTC m=+0.461031899 container cleanup b0e73daa625e9cce05ef1ecc82bc8de739d30539b6dfed3708bd0ccf79711f32 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-29T15:52:57.017 INFO:journalctl@ceph.prometheus.a.smithi171.stdout:Aug 29 15:52:56 smithi171 bash[293395]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-prometheus-a 2024-08-29T15:52:57.229 DEBUG:teuthology.orchestra.run.smithi171:> sudo pkill -f 'journalctl -f -n 0 -u ceph-af03926c-6615-11ef-bcd4-c7b262605968@prometheus.a.service' 2024-08-29T15:52:57.263 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-29T15:52:57.263 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-08-29T15:52:57.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af03926c-6615-11ef-bcd4-c7b262605968 --force --keep-logs 2024-08-29T15:52:57.427 INFO:teuthology.orchestra.run.smithi027.stdout:Deleting cluster with fsid: af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:52:59.179 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:52:58 smithi027 systemd[1]: Stopping Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:52:59.180 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:52:59 smithi027 ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a[88677]: ts=2024-08-29T15:52:59.138Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-29T15:52:59.180 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:52:59 smithi027 podman[411672]: 2024-08-29 15:52:59.178957236 +0000 UTC m=+0.136445517 container died 4ac5fe286fabd1fa04328fcd588accc8ffc9edbf3a0f16a31b3a7cb0108c3090 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:52:59.818 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:52:59 smithi027 podman[411672]: 2024-08-29 15:52:59.646713294 +0000 UTC m=+0.604201574 container cleanup 4ac5fe286fabd1fa04328fcd588accc8ffc9edbf3a0f16a31b3a7cb0108c3090 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:52:59.819 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:52:59 smithi027 bash[411672]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-29T15:53:00.227 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:52:59 smithi027 podman[411683]: 2024-08-29 15:52:59.948124253 +0000 UTC m=+0.763618460 container remove 4ac5fe286fabd1fa04328fcd588accc8ffc9edbf3a0f16a31b3a7cb0108c3090 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-29T15:53:00.227 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:53:00 smithi027 podman[411683]: 2024-08-29 15:53:00.032212507 +0000 UTC m=+0.847706727 volume remove 77a52f9a4ea478a51208bb4a310620c74e2b3b7947bff3c77e1cf1d9568b1404 2024-08-29T15:53:00.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:53:00 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-29T15:53:00.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:53:00 smithi027 systemd[1]: Stopped Ceph alertmanager.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:53:00.568 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Aug 29 15:53:00 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@alertmanager.a.service: Consumed 3.596s CPU time. 2024-08-29T15:53:00.569 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:00 smithi027 systemd[1]: Stopping Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:53:00.953 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:00 smithi027 podman[411793]: 2024-08-29 15:53:00.678758003 +0000 UTC m=+0.096754457 container died e53886849ab5c6733f39c394787ad02242e49fd5a0e8394a79b396924225ccbd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T15:53:01.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:01 smithi027 podman[411793]: 2024-08-29 15:53:01.051455988 +0000 UTC m=+0.469452430 container cleanup e53886849ab5c6733f39c394787ad02242e49fd5a0e8394a79b396924225ccbd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T15:53:01.294 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:01 smithi027 bash[411793]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a 2024-08-29T15:53:01.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:01 smithi027 podman[411805]: 2024-08-29 15:53:01.294146034 +0000 UTC m=+0.619153904 container remove e53886849ab5c6733f39c394787ad02242e49fd5a0e8394a79b396924225ccbd (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-29T15:53:01.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:01 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-29T15:53:01.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:01 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-29T15:53:01.568 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:01 smithi027 systemd[1]: Stopped Ceph node-exporter.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:53:01.569 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Aug 29 15:53:01 smithi027 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.a.service: Consumed 9.084s CPU time. 2024-08-29T15:53:13.147 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af03926c-6615-11ef-bcd4-c7b262605968 --force --keep-logs 2024-08-29T15:53:13.303 INFO:teuthology.orchestra.run.smithi171.stdout:Deleting cluster with fsid: af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T15:53:14.921 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:14 smithi171 systemd[1]: Stopping Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:53:15.214 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=server t=2024-08-29T15:53:14.920578183Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-08-29T15:53:15.214 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=tracing t=2024-08-29T15:53:14.920906839Z level=info msg="Closing tracing" 2024-08-29T15:53:15.214 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=ticker t=2024-08-29T15:53:14.920925507Z level=info msg=stopped last_tick=2024-08-29T15:53:10Z 2024-08-29T15:53:15.215 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:14 smithi171 ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a[80824]: logger=grafana-apiserver t=2024-08-29T15:53:14.920954355Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-08-29T15:53:15.215 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:14 smithi171 podman[293805]: 2024-08-29 15:53:14.941078968 +0000 UTC m=+0.125608124 container died 99f2571ddaab8f90df9586a9a708d8d258bae46bfb214f0683167bbe01e14e57 (image=quay.io/ceph/grafana:10.4.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-29T15:53:15.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:15 smithi171 podman[293805]: 2024-08-29 15:53:15.320253916 +0000 UTC m=+0.504783074 container cleanup 99f2571ddaab8f90df9586a9a708d8d258bae46bfb214f0683167bbe01e14e57 (image=quay.io/ceph/grafana:10.4.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-29T15:53:15.517 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:15 smithi171 bash[293805]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a 2024-08-29T15:53:15.906 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:15 smithi171 podman[293817]: 2024-08-29 15:53:15.604830517 +0000 UTC m=+0.663309679 container remove 99f2571ddaab8f90df9586a9a708d8d258bae46bfb214f0683167bbe01e14e57 (image=quay.io/ceph/grafana:10.4.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-29T15:53:15.906 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:15 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@grafana.a.service: Deactivated successfully. 2024-08-29T15:53:15.906 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:15 smithi171 systemd[1]: Stopped Ceph grafana.a for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:53:15.907 INFO:journalctl@ceph.grafana.a.smithi171.stdout:Aug 29 15:53:15 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@grafana.a.service: Consumed 34.382s CPU time. 2024-08-29T15:53:16.160 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:15 smithi171 systemd[1]: Stopping Ceph node-exporter.b for af03926c-6615-11ef-bcd4-c7b262605968... 2024-08-29T15:53:16.482 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 podman[293923]: 2024-08-29 15:53:16.160110122 +0000 UTC m=+0.117448763 container died 46afc3b5440459faa591cf8c229ac73dd2f4ad45ec5b27aa0194a78db2df1c4c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T15:53:16.483 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 podman[293923]: 2024-08-29 15:53:16.482375065 +0000 UTC m=+0.439713705 container cleanup 46afc3b5440459faa591cf8c229ac73dd2f4ad45ec5b27aa0194a78db2df1c4c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T15:53:16.767 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 bash[293923]: ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b 2024-08-29T15:53:16.767 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 podman[293935]: 2024-08-29 15:53:16.725132466 +0000 UTC m=+0.565679823 container remove 46afc3b5440459faa591cf8c229ac73dd2f4ad45ec5b27aa0194a78db2df1c4c (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-af03926c-6615-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-29T15:53:16.768 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-29T15:53:17.267 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-29T15:53:17.267 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 systemd[1]: Stopped Ceph node-exporter.b for af03926c-6615-11ef-bcd4-c7b262605968. 2024-08-29T15:53:17.268 INFO:journalctl@ceph.node-exporter.b.smithi171.stdout:Aug 29 15:53:16 smithi171 systemd[1]: ceph-af03926c-6615-11ef-bcd4-c7b262605968@node-exporter.b.service: Consumed 7.879s CPU time. 2024-08-29T15:53:20.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-29T15:53:20.924 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-29T15:53:20.952 INFO:tasks.cephadm:Archiving crash dumps... 2024-08-29T15:53:20.954 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/crash to /home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940/remote/smithi027/crash 2024-08-29T15:53:20.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/crash -- . 2024-08-29T15:53:21.007 INFO:teuthology.orchestra.run.smithi027.stderr:tar: /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/crash: Cannot open: No such file or directory 2024-08-29T15:53:21.007 INFO:teuthology.orchestra.run.smithi027.stderr:tar: Error is not recoverable: exiting now 2024-08-29T15:53:21.009 DEBUG:teuthology.misc:Transferring archived files from smithi171:/var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/crash to /home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940/remote/smithi171/crash 2024-08-29T15:53:21.009 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar c -f - -C /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/crash -- . 2024-08-29T15:53:21.049 INFO:teuthology.orchestra.run.smithi171.stderr:tar: /var/lib/ceph/af03926c-6615-11ef-bcd4-c7b262605968/crash: Cannot open: No such file or directory 2024-08-29T15:53:21.050 INFO:teuthology.orchestra.run.smithi171.stderr:tar: Error is not recoverable: exiting now 2024-08-29T15:53:21.051 INFO:tasks.cephadm:Checking cluster log for badness... 2024-08-29T15:53:21.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-08-29T15:53:21.148 INFO:teuthology.orchestra.run.smithi027.stdout:2024-08-29T14:54:20.234632+0000 mon.a (mon.0) 692 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-29T15:53:21.153 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-08-29T15:53:21.153 INFO:tasks.cephadm:Compressing logs... 2024-08-29T15:53:21.154 DEBUG:teuthology.orchestra.run.smithi027:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-29T15:53:21.156 DEBUG:teuthology.orchestra.run.smithi171:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-29T15:53:21.193 INFO:teuthology.orchestra.run.smithi027.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.0.log 2024-08-29T15:53:21.193 INFO:teuthology.orchestra.run.smithi027.stderr:: No such file or directory 2024-08-29T15:53:21.193 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.1.log 2024-08-29T15:53:21.194 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-volume.log 2024-08-29T15:53:21.194 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.0.log: /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.audit.log 2024-08-29T15:53:21.195 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.c.log 2024-08-29T15:53:21.195 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.cephadm.log 2024-08-29T15:53:21.197 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.a.log 2024-08-29T15:53:21.197 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.2.log 2024-08-29T15:53:21.197 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mgr.y.log 2024-08-29T15:53:21.197 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.log 2024-08-29T15:53:21.197 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mds.foo.smithi027.afjjno.log 2024-08-29T15:53:21.197 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.3.log 2024-08-29T15:53:21.198 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mds.foo.smithi027.afjjno.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-29T15:53:21.198 INFO:teuthology.orchestra.run.smithi027.stderr: 86.0% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mds.foo.smithi027.afjjno.log.gz 2024-08-29T15:53:21.199 INFO:teuthology.orchestra.run.smithi171.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.7.log: No such file or directory 2024-08-29T15:53:21.199 INFO:teuthology.orchestra.run.smithi171.stderr: 2024-08-29T15:53:21.199 INFO:teuthology.orchestra.run.smithi171.stderr:gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-volume.log 2024-08-29T15:53:21.199 INFO:teuthology.orchestra.run.smithi171.stderr:gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.4.log 2024-08-29T15:53:21.200 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.7.log: /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.audit.log 2024-08-29T15:53:21.200 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.cephadm.log 2024-08-29T15:53:21.202 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mds.foo.smithi171.grbaeh.log 2024-08-29T15:53:21.202 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.6.log 2024-08-29T15:53:21.202 INFO:teuthology.orchestra.run.smithi171.stderr:gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.5.log 2024-08-29T15:53:21.202 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mds.foo.smithi171.grbaeh.log: 84.0% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mds.foo.smithi171.grbaeh.log.gz 2024-08-29T15:53:21.202 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.log 2024-08-29T15:53:21.202 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mgr.x.log 2024-08-29T15:53:21.203 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.b.log 2024-08-29T15:53:21.203 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-29T15:53:21.208 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 93.3% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.log.gz 2024-08-29T15:53:21.537 INFO:teuthology.orchestra.run.smithi171.stderr:/var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-volume.log.gz 2024-08-29T15:53:21.586 INFO:teuthology.orchestra.run.smithi171.stderr: 90.4% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.audit.log.gz 2024-08-29T15:53:21.672 INFO:teuthology.orchestra.run.smithi027.stderr: 91.2% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-volume.log.gz 2024-08-29T15:53:21.678 INFO:teuthology.orchestra.run.smithi171.stderr: 85.8% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.cephadm.log.gz 2024-08-29T15:53:21.752 INFO:teuthology.orchestra.run.smithi171.stderr: 90.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-29T15:53:21.764 INFO:teuthology.orchestra.run.smithi171.stderr: 89.9% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mgr.x.log.gz 2024-08-29T15:53:21.808 INFO:teuthology.orchestra.run.smithi171.stderr: 88.0% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.log.gz 2024-08-29T15:53:21.848 INFO:teuthology.orchestra.run.smithi027.stderr: 95.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-29T15:53:21.929 INFO:teuthology.orchestra.run.smithi027.stderr: 92.0% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.cephadm.log.gz 2024-08-29T15:53:22.513 INFO:teuthology.orchestra.run.smithi027.stderr: 94.2% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph.audit.log.gz 2024-08-29T15:53:24.823 INFO:teuthology.orchestra.run.smithi027.stderr: 89.8% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mgr.y.log.gz 2024-08-29T15:53:42.726 INFO:teuthology.orchestra.run.smithi027.stderr: 91.8% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.c.log.gz 2024-08-29T15:53:44.047 INFO:teuthology.orchestra.run.smithi171.stderr: 92.3% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.b.log.gz 2024-08-29T15:54:26.021 INFO:teuthology.orchestra.run.smithi027.stderr: 90.2% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-mon.a.log.gz 2024-08-29T16:04:12.276 INFO:teuthology.orchestra.run.smithi171.stderr: 94.0% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.6.log.gz 2024-08-29T16:06:45.341 INFO:teuthology.orchestra.run.smithi171.stderr: 92.1% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.5.log.gz 2024-08-29T16:07:40.153 INFO:teuthology.orchestra.run.smithi171.stderr: 93.3% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.4.log.gz 2024-08-29T16:07:51.595 INFO:teuthology.orchestra.run.smithi171.stderr: 91.9% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.7.log.gz 2024-08-29T16:07:51.598 INFO:teuthology.orchestra.run.smithi171.stderr: 2024-08-29T16:07:51.598 INFO:teuthology.orchestra.run.smithi171.stderr:real 14m30.426s 2024-08-29T16:07:51.598 INFO:teuthology.orchestra.run.smithi171.stderr:user 6m54.199s 2024-08-29T16:07:51.598 INFO:teuthology.orchestra.run.smithi171.stderr:sys 0m19.934s 2024-08-29T16:17:04.126 INFO:teuthology.orchestra.run.smithi027.stderr: 92.7% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.2.log.gz 2024-08-29T16:21:28.817 INFO:teuthology.orchestra.run.smithi027.stderr: 92.7% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.0.log.gz 2024-08-29T16:21:28.878 INFO:teuthology.orchestra.run.smithi027.stderr: 92.6% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.1.log.gz 2024-08-29T16:21:45.577 INFO:teuthology.orchestra.run.smithi027.stderr: 92.5% -- replaced with /var/log/ceph/af03926c-6615-11ef-bcd4-c7b262605968/ceph-osd.3.log.gz 2024-08-29T16:21:45.647 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-08-29T16:21:45.648 INFO:teuthology.orchestra.run.smithi027.stderr:real 28m24.476s 2024-08-29T16:21:45.648 INFO:teuthology.orchestra.run.smithi027.stderr:user 10m17.626s 2024-08-29T16:21:45.648 INFO:teuthology.orchestra.run.smithi027.stderr:sys 0m29.090s 2024-08-29T16:21:45.648 INFO:tasks.cephadm:Archiving logs... 2024-08-29T16:21:45.650 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/log/ceph to /home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940/remote/smithi027/log 2024-08-29T16:21:45.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-29T16:23:16.346 DEBUG:teuthology.misc:Transferring archived files from smithi171:/var/log/ceph to /home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940/remote/smithi171/log 2024-08-29T16:23:16.348 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-29T16:24:22.938 INFO:tasks.cephadm:Removing cluster... 2024-08-29T16:24:22.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af03926c-6615-11ef-bcd4-c7b262605968 --force 2024-08-29T16:24:23.583 INFO:teuthology.orchestra.run.smithi027.stdout:Deleting cluster with fsid: af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T16:24:24.522 DEBUG:teuthology.orchestra.run.smithi171:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid af03926c-6615-11ef-bcd4-c7b262605968 --force 2024-08-29T16:24:24.736 INFO:teuthology.orchestra.run.smithi171.stdout:Deleting cluster with fsid: af03926c-6615-11ef-bcd4-c7b262605968 2024-08-29T16:24:25.474 INFO:tasks.cephadm:Removing cephadm ... 2024-08-29T16:24:25.474 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-29T16:24:25.499 DEBUG:teuthology.orchestra.run.smithi171:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-29T16:24:25.515 INFO:tasks.cephadm:Teardown complete 2024-08-29T16:24:25.515 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-08-29T16:24:25.526 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-08-29T16:24:25.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-29T16:24:25.543 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-29T16:24:25.713 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-08-29T16:24:25.723 INFO:teuthology.task.clock:Checking final clock skew... 2024-08-29T16:24:25.723 DEBUG:teuthology.orchestra.run.smithi027:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-29T16:24:25.726 DEBUG:teuthology.orchestra.run.smithi171:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-29T16:24:25.741 INFO:teuthology.orchestra.run.smithi171.stderr:bash: line 1: ntpq: command not found 2024-08-29T16:24:25.741 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-08-29T16:24:25.779 INFO:teuthology.orchestra.run.smithi171.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-29T16:24:25.779 INFO:teuthology.orchestra.run.smithi171.stdout:=============================================================================== 2024-08-29T16:24:25.779 INFO:teuthology.orchestra.run.smithi171.stdout:^* hv01.front.sepia.ceph.com 3 6 377 24 +201us[ +202us] +/- 40ms 2024-08-29T16:24:25.780 INFO:teuthology.orchestra.run.smithi171.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 26 -943us[ -941us] +/- 63ms 2024-08-29T16:24:25.780 INFO:teuthology.orchestra.run.smithi171.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 30 +769us[ +771us] +/- 71ms 2024-08-29T16:24:25.780 INFO:teuthology.orchestra.run.smithi171.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T16:24:25.798 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-29T16:24:25.798 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-08-29T16:24:25.798 INFO:teuthology.orchestra.run.smithi027.stdout:^* hv01.front.sepia.ceph.com 3 7 377 97 +210us[ +221us] +/- 39ms 2024-08-29T16:24:25.799 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 30 -957us[ -957us] +/- 63ms 2024-08-29T16:24:25.799 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 34 +768us[ +768us] +/- 71ms 2024-08-29T16:24:25.799 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-29T16:24:25.810 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-08-29T16:24:25.824 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-08-29T16:24:25.825 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-08-29T16:24:25.835 DEBUG:teuthology.orchestra.run.smithi027:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-29T16:24:25.957 DEBUG:teuthology.orchestra.run.smithi171:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-29T16:24:26.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-29T16:24:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T16:24:26.112 DEBUG:teuthology.orchestra.run.smithi171:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-08-29T16:24:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T16:24:26.142 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-08-29T16:24:26.153 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-08-29T16:24:26.163 INFO:teuthology.task.internal:Duration was 6027.795537 seconds 2024-08-29T16:24:26.163 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-08-29T16:24:26.172 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-08-29T16:24:26.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-29T16:24:26.175 DEBUG:teuthology.orchestra.run.smithi171:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-29T16:24:26.237 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-29T16:24:26.248 INFO:teuthology.orchestra.run.smithi171.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-29T16:24:26.660 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-08-29T16:24:26.661 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi027.front.sepia.ceph.com 2024-08-29T16:24:26.661 DEBUG:teuthology.orchestra.run.smithi027:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-29T16:24:26.690 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi171.front.sepia.ceph.com 2024-08-29T16:24:26.691 DEBUG:teuthology.orchestra.run.smithi171:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-29T16:24:26.717 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-08-29T16:24:26.717 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-29T16:24:26.733 DEBUG:teuthology.orchestra.run.smithi171:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-29T16:24:26.785 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-08-29T16:24:26.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-29T16:24:26.813 DEBUG:teuthology.orchestra.run.smithi171:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-29T16:24:29.236 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-08-29T16:24:29.246 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-08-29T16:24:29.246 DEBUG:teuthology.orchestra.run.smithi027:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-29T16:24:29.274 DEBUG:teuthology.orchestra.run.smithi171:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-29T16:24:29.316 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-08-29T16:24:29.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-29T16:24:29.329 DEBUG:teuthology.orchestra.run.smithi171:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-29T16:24:29.368 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = core 2024-08-29T16:24:29.390 INFO:teuthology.orchestra.run.smithi171.stdout:kernel.core_pattern = core 2024-08-29T16:24:29.413 DEBUG:teuthology.orchestra.run.smithi027:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-29T16:24:29.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T16:24:29.458 DEBUG:teuthology.orchestra.run.smithi171:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-29T16:24:29.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T16:24:29.475 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-08-29T16:24:29.486 INFO:teuthology.task.internal:Transferring archived files... 2024-08-29T16:24:29.487 DEBUG:teuthology.misc:Transferring archived files from smithi027:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940/remote/smithi027 2024-08-29T16:24:29.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-29T16:24:29.581 DEBUG:teuthology.misc:Transferring archived files from smithi171:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2024-08-29_14:13:22-upgrade-wip-yuri4-testing-2024-08-28-1359-distro-default-smithi/7879940/remote/smithi171 2024-08-29T16:24:29.582 DEBUG:teuthology.orchestra.run.smithi171:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-29T16:24:29.631 INFO:teuthology.task.internal:Removing archive directory... 2024-08-29T16:24:29.631 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-29T16:24:29.634 DEBUG:teuthology.orchestra.run.smithi171:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-29T16:24:29.677 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-08-29T16:24:29.688 INFO:teuthology.task.internal:Not uploading archives. 2024-08-29T16:24:29.688 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-08-29T16:24:29.698 INFO:teuthology.task.internal:Tidying up after the test... 2024-08-29T16:24:29.698 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-29T16:24:29.700 DEBUG:teuthology.orchestra.run.smithi171:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-29T16:24:29.726 INFO:teuthology.orchestra.run.smithi027.stdout: 262217 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 29 16:24 /home/ubuntu/cephtest 2024-08-29T16:24:29.726 INFO:teuthology.orchestra.run.smithi027.stdout: 4980763 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 29 15:49 /home/ubuntu/cephtest/mnt.0 2024-08-29T16:24:29.726 INFO:teuthology.orchestra.run.smithi027.stdout: 4980764 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 29 15:49 /home/ubuntu/cephtest/mnt.0/client.0 2024-08-29T16:24:29.726 INFO:teuthology.orchestra.run.smithi027.stdout: 4982790 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 29 15:49 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-29T16:24:29.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-29T16:24:29.727 INFO:teuthology.orchestra.run.smithi027.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-08-29T16:24:29.728 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi027 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-29T16:24:29.734 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-08-29T16:24:29.744 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-08-29T16:24:29.756 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi027 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-29T16:24:29.759 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 6027.795536518097 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi027 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_yuriw@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=d184d9e03cae4a048b9c71d252c83578 status: fail success: false 2024-08-29T16:24:29.760 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-29T16:24:29.783 INFO:teuthology.orchestra.run.smithi171.stdout: 262217 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 29 16:24 /home/ubuntu/cephtest 2024-08-29T16:24:29.864 INFO:teuthology.run:FAIL